Skip to content

vietnamese-engineer-in-japan/twitter-kafka-streaming

 
 

Folders and files

NameName
Last commit message
Last commit date

Latest commit

 

History

11 Commits
 
 
 
 
 
 
 
 

Repository files navigation

Twitter data streaming with Kafka

Preparations

  1. Install Kafka on your local environment
  2. Create the new topic named twitter_tweets:
    kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 6 --topic twitter_tweets
    
  3. Start kafka service
    brew services start kafka
    
  4. Setup Twitter KEY, SECRET_KEY, TOKEN and SECRET_TOKEN
  • Open development.properties and set the value:
    CONSUMER_KEY=
    CONSUMER_SECRET=
    TOKEN=
    TOKEN_SECRET=
    

How to run

  • Open Bootstrap class and run the main method
    public class Bootstrap {
    
        public static void main(String[] args) {
            new Thread(new TwitterProducer(KafkaProperty.properties())).start();
            new Thread(new TwitterConsumer(KafkaProperty.properties())).start();
        }
    }

Check result

  • You can confirm your output through console like:

    twitter_tweets [5] offset=99, key=null, value="{"created_at":"Sun Jun 23 15:08:48 +0000 2019","id":1142811752532467712,"id_str":"1142811752532467712","text":"RT @SparrowExchange: \ud83d\ude80Just 3 days to launch! Soon everyone will be able to trade #digitalassets options easily and securely. \n\n#TradeatSpar\u2026","source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1062575421064167425,"id_str":"1062575421064167425","name":"sergYan","screen_name":"sergYanu","location":"\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u044c","url":null,"description":"#FOLLOWBACK #RUFOLLOWBACK #TEAMFOLLOWBACK #\u0412\u0417\u0410\u0418\u041c\u041d\u041e #\u0427\u0418\u0422\u0410\u042e\u0412\u0417\u0410\u0418\u041c\u041d\u041e #\u0412\u0417\u0410\u0418\u041c\u041d\u042b\u0419\u0424\u041e\u041b\u041b\u041e\u0412\u0418\u041d\u0413 #BOUNTY #Airdrop #ICO #\u043a\u0440\u0438\u043f\u0442\u043e\u0432\u0430\u043b\u044e\u0442\u044b #bitcoin #\u043a\u0430\u043a_\u0437\u0430\u0440\u0430\u0431\u043e\u0442\u0430\u0442\u044c_\u0432_\u0438\u043d\u0442\u0435\u0440\u043d\u0435\u0442\u0435","translator_type":"none","protected":false,"verified":false,"followers_count":4318,"friends_count":4962,"listed_count":3,"favourites_count":1001,"statuses_count":1979,"created_at":"Wed Nov 14 05:18:16 +0000 2018","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1063121944822890499\/hq7ADEPj_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1063121944822890499\/hq7ADEPj_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1062575421064167425\/1542175416","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Tue Jun 04 06:28:31 +0000 2019","id":1135795448277422080,"id_str":"1135795448277422080","text":"\ud83d\ude80Just 3 days to launch! Soon everyone will be able to trade #digitalassets options easily and securely.\u2026 https:\/\/t.co\/03hHlxqm2y","display_text_range":[0,140],"source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":930958016110669824,"id_str":"930958016110669824","name":"Sparrow - The Leading Options Trading Platform.","screen_name":"SparrowExchange","location":"Singapore","url":"http:\/\/www.sparrowexchange.com","description":"Take Control. Trade Confidently. http:\/\/t.me\/SparrowExchange","translator_type":"none","protected":false,"verified":false,"followers_count":609,"friends_count":502,"listed_count":9,"favourites_count":289,"statuses_count":215,"created_at":"Thu Nov 16 00:37:42 +0000 2017","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/936602856886493190\/BpNsFpTr_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/936602856886493190\/BpNsFpTr_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/930958016110669824\/1558058483","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"\ud83d\ude80Just 3 days to launch! Soon everyone will be able to trade #digitalassets options easily and securely. \n\n#TradeatSparrow #Tradeoptions #crypto #bitcoin #cryptotrading #optionstrading #tradecrypto #btc #eth #launch #watchthisspace https:\/\/t.co\/wMGsQh9msc","display_text_range":[0,230],"entities":{"hashtags":[{"text":"digitalassets","indices":[60,74]},{"text":"TradeatSparrow","indices":[106,121]},{"text":"Tradeoptions","indices":[122,135]},{"text":"crypto","indices":[136,143]},{"text":"bitcoin","indices":[144,152]},{"text":"cryptotrading","indices":[153,167]},{"text":"optionstrading","indices":[168,183]},{"text":"tradecrypto","indices":[184,196]},{"text":"btc","indices":[197,201]},{"text":"eth","indices":[202,206]},{"text":"launch","indices":[207,214]},{"text":"watchthisspace","indices":[215,230]}],"urls":[],"user_mentions":[],"symbols":[],"media":[{"id":1135791212391161856,"id_str":"1135791212391161856","indices":[231,254],"media_url":"http:\/\/pbs.twimg.com\/media\/D8MkFwmV4AAUnXu.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/D8MkFwmV4AAUnXu.jpg","url":"https:\/\/t.co\/wMGsQh9msc","display_url":"pic.twitter.com\/wMGsQh9msc","expanded_url":"https:\/\/twitter.com\/SparrowExchange\/status\/1135795448277422080\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":512,"resize":"fit"},"medium":{"w":1024,"h":512,"resize":"fit"},"small":{"w":680,"h":340,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1135791212391161856,"id_str":"1135791212391161856","indices":[231,254],"media_url":"http:\/\/pbs.twimg.com\/media\/D8MkFwmV4AAUnXu.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/D8MkFwmV4AAUnXu.jpg","url":"https:\/\/t.co\/wMGsQh9msc","display_url":"pic.twitter.com\/wMGsQh9msc","expanded_url":"https:\/\/twitter.com\/SparrowExchange\/status\/1135795448277422080\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":512,"resize":"fit"},"medium":{"w":1024,"h":512,"resize":"fit"},"small":{"w":680,"h":340,"resize":"fit"}}}]}},"quote_count":28,"reply_count":21,"retweet_count":52,"favorite_count":82,"entities":{"hashtags":[{"text":"digitalassets","indices":[60,74]}],"urls":[{"url":"https:\/\/t.co\/03hHlxqm2y","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1135795448277422080","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[105,128]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"digitalassets","indices":[81,95]}],"urls":[],"user_mentions":[{"screen_name":"SparrowExchange","name":"Sparrow - The Leading Options Trading Platform.","id":930958016110669824,"id_str":"930958016110669824","indices":[3,19]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en","timestamp_ms":"1561302528876"}
    "
    twitter_tweets [4] offset=97, key=null, value="{"created_at":"Sun Jun 23 15:08:50 +0000 2019","id":1142811757821476865,"id_str":"1142811757821476865","text":"RT @MPCXPlatform: \ud83d\udcc8With #Bitcoin skyrocketing to $10,788, it seems that we are heading towards a new a all time high surpussing the previou\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1583934276,"id_str":"1583934276","name":"\u0415\u0433\u0435\u0440","screen_name":"egor_rickun","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":41,"friends_count":8,"listed_count":0,"favourites_count":90,"statuses_count":1444,"created_at":"Wed Jul 10 20:27:59 +0000 2013","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"211A1C","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":true,"profile_link_color":"FF0839","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000334057609\/9ad39e6ebf887ad3fb733f5fa9cbad1e_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000334057609\/9ad39e6ebf887ad3fb733f5fa9cbad1e_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1583934276\/1455655513","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sat Jun 22 14:35:53 +0000 2019","id":1142441079851102208,"id_str":"1142441079851102208","text":"\ud83d\udcc8With #Bitcoin skyrocketing to $10,788, it seems that we are heading towards a new a all time high surpussing the p\u2026 https:\/\/t.co\/rCEm2FEQtC","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":967011488476082176,"id_str":"967011488476082176","name":"MPCX Platform","screen_name":"MPCXPlatform","location":"City of London, London","url":"http:\/\/mpcx.co","description":"Digital Wealth Management Platform","translator_type":"none","protected":false,"verified":false,"followers_count":15766,"friends_count":59,"listed_count":45,"favourites_count":1723,"statuses_count":729,"created_at":"Fri Feb 23 12:21:19 +0000 2018","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"000000","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"1B95E0","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"000000","profile_text_color":"000000","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1008799970567315457\/f4YUChV-_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1008799970567315457\/f4YUChV-_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/967011488476082176\/1529351817","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"\ud83d\udcc8With #Bitcoin skyrocketing to $10,788, it seems that we are heading towards a new a all time high surpussing the previous $19,895. \n\n\ud83d\udcccDo you think it is going to happen within 2019? \n\n#cryptocurrency #blockchain #bull","display_text_range":[0,218],"entities":{"hashtags":[{"text":"Bitcoin","indices":[6,14]},{"text":"cryptocurrency","indices":[185,200]},{"text":"blockchain","indices":[201,212]},{"text":"bull","indices":[213,218]}],"urls":[],"user_mentions":[],"symbols":[]}},"quote_count":2,"reply_count":10,"retweet_count":9,"favorite_count":24,"entities":{"hashtags":[{"text":"Bitcoin","indices":[6,14]}],"urls":[{"url":"https:\/\/t.co\/rCEm2FEQtC","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1142441079851102208","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"Bitcoin","indices":[24,32]}],"urls":[],"user_mentions":[{"screen_name":"MPCXPlatform","name":"MPCX Platform","id":967011488476082176,"id_str":"967011488476082176","indices":[3,16]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en","timestamp_ms":"1561302530137"}
    "
    twitter_tweets [3] offset=97, key=null, value="{"created_at":"Sun Jun 23 15:08:47 +0000 2019","id":1142811744747831301,"id_str":"1142811744747831301","text":"Buy\/Sell Bitcoin & altcoin moves with up to 100x Leverage at PrimeXBT! \ud83e\udd11\ud83d\udc51\n\nSign up now and convert your $75 into $1\u2026 https:\/\/t.co\/qyJgc6QMGp","display_text_range":[0,140],"source":"\u003ca href=\"https:\/\/mobile.twitter.com\" rel=\"nofollow\"\u003eTwitter Web App\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1141188516493348864,"id_str":"1141188516493348864","name":"Alliyah Patipat","screen_name":"alliyahpatipat","location":"Chesterton","url":null,"description":"Model Makers, Grain oilseed or pasture grower, Business machine mechanic","translator_type":"none","protected":false,"verified":false,"followers_count":3,"friends_count":56,"listed_count":0,"favourites_count":0,"statuses_count":2,"created_at":"Wed Jun 19 03:38:39 +0000 2019","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"000000","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"BFFCBC","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"000000","profile_text_color":"000000","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1141191800759226368\/78afSgUI_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1141191800759226368\/78afSgUI_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1141188516493348864\/1560916301","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"Buy\/Sell Bitcoin & altcoin moves with up to 100x Leverage at PrimeXBT! \ud83e\udd11\ud83d\udc51\n\nSign up now and convert your $75 into $12500:\n\n\u2705 https:\/\/t.co\/PxZmy3Nkcd \u2705\n\nMake money even when it is dropping! \ud83d\udcc9\ud83d\udcc9\n\n$RVN - $MONA - $QNT - $BNB - $PPT - $ENJ - $ADA - $HEDG - $ETP - $PPT https:\/\/t.co\/lmMHFp2UEL","display_text_range":[0,265],"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/PxZmy3Nkcd","expanded_url":"http:\/\/bit.ly\/01PrimeXBT","display_url":"bit.ly\/01PrimeXBT","indices":[128,151]}],"user_mentions":[],"symbols":[{"text":"RVN","indices":[196,200]},{"text":"MONA","indices":[203,208]},{"text":"QNT","indices":[211,215]},{"text":"BNB","indices":[218,222]},{"text":"PPT","indices":[225,229]},{"text":"ENJ","indices":[232,236]},{"text":"ADA","indices":[239,243]},{"text":"HEDG","indices":[246,251]},{"text":"ETP","indices":[254,258]},{"text":"PPT","indices":[261,265]}],"media":[{"id":1142811738326413312,"id_str":"1142811738326413312","indices":[266,289],"media_url":"http:\/\/pbs.twimg.com\/media\/D9wVOJwXsAAHiss.png","media_url_https":"https:\/\/pbs.twimg.com\/media\/D9wVOJwXsAAHiss.png","url":"https:\/\/t.co\/lmMHFp2UEL","display_url":"pic.twitter.com\/lmMHFp2UEL","expanded_url":"https:\/\/twitter.com\/alliyahpatipat\/status\/1142811744747831301\/photo\/1","type":"photo","sizes":{"medium":{"w":1200,"h":630,"resize":"fit"},"small":{"w":680,"h":357,"resize":"fit"},"large":{"w":1200,"h":630,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"}}}]},"extended_entities":{"media":[{"id":1142811738326413312,"id_str":"1142811738326413312","indices":[266,289],"media_url":"http:\/\/pbs.twimg.com\/media\/D9wVOJwXsAAHiss.png","media_url_https":"https:\/\/pbs.twimg.com\/media\/D9wVOJwXsAAHiss.png","url":"https:\/\/t.co\/lmMHFp2UEL","display_url":"pic.twitter.com\/lmMHFp2UEL","expanded_url":"https:\/\/twitter.com\/alliyahpatipat\/status\/1142811744747831301\/photo\/1","type":"photo","sizes":{"medium":{"w":1200,"h":630,"resize":"fit"},"small":{"w":680,"h":357,"resize":"fit"},"large":{"w":1200,"h":630,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"}}}]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/qyJgc6QMGp","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1142811744747831301","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[121,144]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1561302527020"}
    "
    twitter_tweets [2] offset=100, key=null, value="{"created_at":"Sun Jun 23 15:08:50 +0000 2019","id":1142811760279166977,"id_str":"1142811760279166977","text":"RT @gravitywave2: The simple case for a $250,000 bitcoin price before 2023. This model takes into account diminishing returns for each new\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":985050417061969921,"id_str":"985050417061969921","name":"Gary.Ma","screen_name":"GaryMa_Zhu","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":11,"friends_count":80,"listed_count":0,"favourites_count":33,"statuses_count":668,"created_at":"Sat Apr 14 07:01:35 +0000 2018","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sat Jun 22 12:03:55 +0000 2019","id":1142402835583311873,"id_str":"1142402835583311873","text":"The simple case for a $250,000 bitcoin price before 2023. This model takes into account diminishing returns for eac\u2026 https:\/\/t.co\/PJsvlh8kY1","display_text_range":[0,140],"source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1004591710243090433,"id_str":"1004591710243090433","name":"GravityWave","screen_name":"gravitywave2","location":null,"url":"https:\/\/bitcoin.org","description":"Bitcoin macro cycle analyst and buyer of last resort.","translator_type":"none","protected":false,"verified":false,"followers_count":3053,"friends_count":194,"listed_count":69,"favourites_count":3041,"statuses_count":686,"created_at":"Thu Jun 07 05:11:42 +0000 2018","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"000000","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"ABB8C2","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"000000","profile_text_color":"000000","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1004595148922277889\/GmAPZkrT_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1004595148922277889\/GmAPZkrT_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1004591710243090433\/1547715569","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"The simple case for a $250,000 bitcoin price before 2023. This model takes into account diminishing returns for each new cycle, and increasing cycle length. https:\/\/t.co\/nyfbu2l09p","display_text_range":[0,156],"entities":{"hashtags":[],"urls":[],"user_mentions":[],"symbols":[],"media":[{"id":1142401669642248197,"id_str":"1142401669642248197","indices":[157,180],"media_url":"http:\/\/pbs.twimg.com\/media\/D9qgRA3W4AUTHDp.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/D9qgRA3W4AUTHDp.jpg","url":"https:\/\/t.co\/nyfbu2l09p","display_url":"pic.twitter.com\/nyfbu2l09p","expanded_url":"https:\/\/twitter.com\/gravitywave2\/status\/1142402835583311873\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":419,"resize":"fit"},"medium":{"w":1200,"h":739,"resize":"fit"},"large":{"w":2048,"h":1261,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1142401669642248197,"id_str":"1142401669642248197","indices":[157,180],"media_url":"http:\/\/pbs.twimg.com\/media\/D9qgRA3W4AUTHDp.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/D9qgRA3W4AUTHDp.jpg","url":"https:\/\/t.co\/nyfbu2l09p","display_url":"pic.twitter.com\/nyfbu2l09p","expanded_url":"https:\/\/twitter.com\/gravitywave2\/status\/1142402835583311873\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":419,"resize":"fit"},"medium":{"w":1200,"h":739,"resize":"fit"},"large":{"w":2048,"h":1261,"resize":"fit"}}}]}},"quote_count":19,"reply_count":27,"retweet_count":208,"favorite_count":607,"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/PJsvlh8kY1","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1142402835583311873","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"gravitywave2","name":"GravityWave","id":1004591710243090433,"id_str":"1004591710243090433","indices":[3,16]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en","timestamp_ms":"1561302530723"}
    "
    twitter_tweets [1] offset=99, key=null, value="{"created_at":"Sun Jun 23 15:08:50 +0000 2019","id":1142811759612272641,"id_str":"1142811759612272641","text":"Pre-Halving Phase: Bitcoin is Just Getting Started; Huge Gains Expected - https:\/\/t.co\/leTvqAH8FP $BTC #news #finance #investing","source":"\u003ca href=\"http:\/\/cryptonewswire.com\/\" rel=\"nofollow\"\u003eCryptoNewswire\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":771502019471220736,"id_str":"771502019471220736","name":"CryptoNewswire \ud83c\udf10","screen_name":"CryptoNewswire","location":null,"url":"https:\/\/cryptonewswire.com","description":"Follow the latest cryptocurrency news and events. #bitcoin #altcoins #crypto #blockchain #fintech","translator_type":"none","protected":false,"verified":false,"followers_count":9155,"friends_count":181,"listed_count":518,"favourites_count":0,"statuses_count":148447,"created_at":"Fri Sep 02 00:16:30 +0000 2016","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/794271147399335936\/wP92-Lk9_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/794271147399335936\/wP92-Lk9_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/771502019471220736\/1476028784","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"news","indices":[103,108]},{"text":"finance","indices":[109,117]},{"text":"investing","indices":[118,128]}],"urls":[{"url":"https:\/\/t.co\/leTvqAH8FP","expanded_url":"https:\/\/cryptonewswire.com\/pre-halving-phase-bitcoin-is-just-getting-started-huge-gains-expected","display_url":"cryptonewswire.com\/pre-halving-ph\u2026","indices":[74,97]}],"user_mentions":[],"symbols":[{"text":"BTC","indices":[98,102]}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1561302530564"}
    "
    twitter_tweets [0] offset=100, key=null, value="{"created_at":"Sun Jun 23 15:08:49 +0000 2019","id":1142811753207799811,"id_str":"1142811753207799811","text":"Precio del #bitcoin alcanz\u00f3 su m\u00e1ximo tope de los \u00faltimos 15 meses              -_              https:\/\/t.co\/2SyC6EWWZy","source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":171299971,"id_str":"171299971","name":"CaraotaDigital","screen_name":"CaraotaDigital","location":"Caracas - Venezuela","url":"http:\/\/caraotalibre.cf","description":"\u00a1Tu portal de #noticias venezolano, donde encontrar\u00e1s la informaci\u00f3n directa al grano!","translator_type":"none","protected":false,"verified":true,"followers_count":1381102,"friends_count":356,"listed_count":3284,"favourites_count":16054,"statuses_count":5221831,"created_at":"Tue Jul 27 01:32:25 +0000 2010","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_tile":true,"profile_link_color":"336633","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1135926035432845314\/jhVmQbND_normal.png","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1135926035432845314\/jhVmQbND_normal.png","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/171299971\/1559660813","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"bitcoin","indices":[11,19]}],"urls":[{"url":"https:\/\/t.co\/2SyC6EWWZy","expanded_url":"http:\/\/bit.ly\/2ZFCifz","display_url":"bit.ly\/2ZFCifz","indices":[96,119]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"es","timestamp_ms":"1561302529037"}
    "
    
  • Above data is the real-time data from twitter which related to bitcoin.

  • You can change the topic to get other tweets by changing:

    public class TwitterTopic implements Topic {
    
        @Override
        public List<String> topics() {
            return Lists.newArrayList("bitcoin");
        }
    }

About

Using Twitter API data and pass it through Kafka

Resources

Stars

Watchers

Forks

Releases

No releases published

Packages

No packages published

Languages

  • Java 100.0%