From 13bcb55f6e9128476f8c57d08d0c634d94c1aacf Mon Sep 17 00:00:00 2001 From: Monil Patel Date: Tue, 12 Nov 2024 13:58:54 -0800 Subject: [PATCH 1/4] [1/2] Added Poll Creation via V2 API with examples --- .env.example | 11 +++++++- README.md | 65 +++++++++++++++++++++++++++++++++++++++-------- SampleAgent.js | 43 +++++++++++++++++++++++++++++++ package-lock.json | 29 +++++++++++++++++---- package.json | 8 +++--- src/auth-user.ts | 7 +++++ src/auth.ts | 37 +++++++++++++++++++++++++++ src/scraper.ts | 42 ++++++++++++++++++++++++++++-- src/tweets.ts | 52 +++++++++++++++++++++++++++++++++++++ 9 files changed, 272 insertions(+), 22 deletions(-) create mode 100644 SampleAgent.js diff --git a/.env.example b/.env.example index 38d6249..4b4bb36 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,13 @@ +# for v1 api support TWITTER_USERNAME=myaccount TWITTER_PASSWORD=MyPassword!!! TWITTER_EMAIL=myemail@gmail.com -PROXY_URL= # HTTP(s) proxy for requests (optional) \ No newline at end of file + +# for v2 api support +TWITTER_API_KEY=key +TWITTER_API_SECRET_KEY=secret +TWITTER_ACCESS_TOKEN=token +TWITTER_ACCESS_TOKEN_SECRET=tokensecret + +# optional +PROXY_URL= # HTTP(s) proxy for requests diff --git a/README.md b/README.md index 94dc3e5..1341d79 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,13 @@ This is a modified version of [@the-convocation/twitter-scraper](https://github.com/the-convocation/twitter-scraper) with added functionality for sending tweets and retweets. This package does not require Twitter API to use, and will run in both the browser and server. ## Installation + ```sh npm install agent-twitter-client ``` ## Setup + Configure environment variables for authentication. ``` @@ -15,38 +17,74 @@ TWITTER_USERNAME= # Account username TWITTER_PASSWORD= # Account password TWITTER_EMAIL= # Account email PROXY_URL= # HTTP(s) proxy for requests (necessary for browsers) + +# Twitter API v2 credentials for tweet and poll functionality +TWITTER_API_KEY= # Twitter API Key +TWITTER_API_SECRET_KEY= # Twitter API Secret Key +TWITTER_ACCESS_TOKEN= # Access Token for Twitter API v2 +TWITTER_ACCESS_TOKEN_SECRET= # Access Token Secret for Twitter API v2 ``` ### Getting Twitter Cookies + It is important that you use Twitter cookies so that you don't send a new login request to twitter every time you want to do something. In your application, you will probably want to have a check for cookies. If you don't have cookies, log in with user auth credentials. Then, cache the cookies for future use. + ```ts - const scraper = await getScraper({ authMethod: 'password' }); +const scraper = await getScraper({ authMethod: 'password' }); - scraper.getCookies().then((cookies) => { - console.log(cookies); - // Remove 'Cookies' and save the cookies as a JSON array - }); +scraper.getCookies().then((cookies) => { + console.log(cookies); + // Remove 'Cookies' and save the cookies as a JSON array +}); ``` ## Getting Started + ```ts const scraper = new Scraper(); await scraper.login('username', 'password'); +// if using v2 functionality (currently needed to support polls) +await scraper.login( + 'username', + 'password', + 'email', + 'appKey', + 'appSecret', + 'accessToken', + 'accessSecret', +); const tweets = await scraper.getTweets('elonmusk', 10); const tweetsAndReplies = scraper.getTweetsAndReplies('elonmusk'); const latestTweet = await scraper.getLatestTweet('elonmusk'); const tweet = await scraper.getTweet('1234567890123456789'); await scraper.sendTweet('Hello world!'); +// create a poll +await scraper.sendTweetV2( + `What's got you most hyped? Let us know! 🤖💸`, + undefined, + { + poll: { + options: [ + { label: 'AI Innovations 🤖' }, + { label: 'Crypto Craze 💸' }, + { label: 'Both! 🌌' }, + { label: 'Neither for Me 😅' }, + ], + durationMinutes: 120, // Duration of the poll in minutes + }, + }, +); ``` ## API ### Authentication + ```ts // Log in -await scraper.login('username', 'password'); +await scraper.login('username', 'password'); // Log out await scraper.logout(); @@ -65,15 +103,17 @@ await scraper.clearCookies(); ``` ### Profile + ```ts // Get a user's profile const profile = await scraper.getProfile('TwitterDev'); -// Get a user ID from their screen name +// Get a user ID from their screen name const userId = await scraper.getUserIdByScreenName('TwitterDev'); ``` ### Search + ```ts import { SearchMode } from 'agent-twitter-client'; @@ -81,7 +121,7 @@ import { SearchMode } from 'agent-twitter-client'; const tweets = scraper.searchTweets('#nodejs', 20, SearchMode.Latest); // Search for profiles -const profiles = scraper.searchProfiles('John', 10); +const profiles = scraper.searchProfiles('John', 10); // Fetch a page of tweet results const results = await scraper.fetchSearchTweets('#nodejs', 20, SearchMode.Top); @@ -91,6 +131,7 @@ const profileResults = await scraper.fetchSearchProfiles('John', 10); ``` ### Relationships + ```ts // Get a user's followers const followers = scraper.getFollowers('12345', 100); @@ -101,11 +142,12 @@ const following = scraper.getFollowing('12345', 100); // Fetch a page of a user's followers const followerResults = await scraper.fetchProfileFollowers('12345', 100); -// Fetch a page of who a user is following +// Fetch a page of who a user is following const followingResults = await scraper.fetchProfileFollowing('12345', 100); ``` ### Trends + ```ts // Get current trends const trends = await scraper.getTrends(); @@ -115,6 +157,7 @@ const listTweets = await scraper.fetchListTweets('1234567890', 50); ``` ### Tweets + ```ts // Get a user's tweets const tweets = scraper.getTweets('TwitterDev'); @@ -129,7 +172,7 @@ const tweetsAndReplies = scraper.getTweetsAndReplies('TwitterDev'); const timeline = scraper.getTweets('TwitterDev', 100); const retweets = await scraper.getTweetsWhere( timeline, - (tweet) => tweet.isRetweet + (tweet) => tweet.isRetweet, ); // Get a user's latest tweet @@ -137,4 +180,4 @@ const latestTweet = await scraper.getLatestTweet('TwitterDev'); // Get a specific tweet by ID const tweet = await scraper.getTweet('1234567890123456789'); -``` \ No newline at end of file +``` diff --git a/SampleAgent.js b/SampleAgent.js new file mode 100644 index 0000000..d82ea7a --- /dev/null +++ b/SampleAgent.js @@ -0,0 +1,43 @@ +import { Scraper } from 'agent-twitter-client'; +import dotenv from 'dotenv'; +dotenv.config(); + +async function main() { + // const scraper = new Scraper(); + // // v1 login + // await scraper.login( + // process.env.TWITTER_USERNAME, + // process.env.TWITTER_PASSWORD, + // ); + // // v2 login + // await scraper.login( + // process.env.TWITTER_USERNAME, + // process.env.TWITTER_PASSWORD, + // undefined, + // undefined, + // process.env.TWITTER_API_KEY, + // process.env.TWITTER_API_SECRET_KEY, + // process.env.TWITTER_ACCESS_TOKEN, + // process.env.TWITTER_ACCESS_TOKEN_SECRET, + // ); + // console.log('Logged in successfully!'); + // // Example: Posting a new tweet with a poll + // await scraper.sendTweetV2( + // `When do you think we'll achieve AGI (Artificial General Intelligence)? 🤖 Cast your prediction!`, + // undefined, + // { + // poll: { + // options: [ + // { label: '2025 🗓️' }, + // { label: '2026 📅' }, + // { label: '2027 🛠️' }, + // { label: '2030+ 🚀' }, + // ], + // durationMinutes: 1440, + // }, + // }, + // ); + // console.log(await scraper.getTweet('id')); +} + +main(); diff --git a/package-lock.json b/package-lock.json index df9e5a8..4820d69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,12 +10,14 @@ "license": "MIT", "dependencies": { "@sinclair/typebox": "^0.32.20", + "agent-twitter-client": "^0.0.13", "headers-polyfill": "^3.1.2", "json-stable-stringify": "^1.0.2", "otpauth": "^9.2.2", "set-cookie-parser": "^2.6.0", "tough-cookie": "^4.1.2", - "tslib": "^2.5.2" + "tslib": "^2.5.2", + "twitter-api-v2": "^1.18.2" }, "devDependencies": { "@commitlint/cli": "^17.6.3", @@ -27,14 +29,14 @@ "@types/tough-cookie": "^4.0.2", "@typescript-eslint/eslint-plugin": "^5.59.7", "@typescript-eslint/parser": "^5.59.7", - "dotenv": "^16.3.1", + "dotenv": "^16.4.5", "esbuild": "^0.21.5", "eslint": "^8.41.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-prettier": "^4.2.1", "gh-pages": "^5.0.0", "https-proxy-agent": "^7.0.2", - "jest": "^29.5.0", + "jest": "^29.7.0", "lint-staged": "^13.2.2", "prettier": "^2.8.8", "rimraf": "^5.0.7", @@ -2863,6 +2865,20 @@ "node": ">= 14" } }, + "node_modules/agent-twitter-client": { + "version": "0.0.13", + "resolved": "https://registry.npmjs.org/agent-twitter-client/-/agent-twitter-client-0.0.13.tgz", + "integrity": "sha512-xIVvrMKWe9VfZDlmGwO9hEd0Kav74FUT4euPZV8XiTqPv6D5gOd5PE0KkkBHPKSupOZuHf8BvQuhEwa/5Ac6hg==", + "dependencies": { + "@sinclair/typebox": "^0.32.20", + "headers-polyfill": "^3.1.2", + "json-stable-stringify": "^1.0.2", + "otpauth": "^9.2.2", + "set-cookie-parser": "^2.6.0", + "tough-cookie": "^4.1.2", + "tslib": "^2.5.2" + } + }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -3834,7 +3850,6 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", "dev": true, - "license": "BSD-2-Clause", "engines": { "node": ">=12" }, @@ -5462,7 +5477,6 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, - "license": "MIT", "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -8895,6 +8909,11 @@ "dev": true, "license": "0BSD" }, + "node_modules/twitter-api-v2": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/twitter-api-v2/-/twitter-api-v2-1.18.2.tgz", + "integrity": "sha512-ggImmoAeVgETYqrWeZy+nWnDpwgTP+IvFEc03Pitt1HcgMX+Yw17rP38Fb5FFTinuyNvS07EPtAfZ184uIyB0A==" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 3449d49..ed6a8de 100644 --- a/package.json +++ b/package.json @@ -32,12 +32,14 @@ }, "dependencies": { "@sinclair/typebox": "^0.32.20", + "agent-twitter-client": "^0.0.13", "headers-polyfill": "^3.1.2", "json-stable-stringify": "^1.0.2", "otpauth": "^9.2.2", "set-cookie-parser": "^2.6.0", "tough-cookie": "^4.1.2", - "tslib": "^2.5.2" + "tslib": "^2.5.2", + "twitter-api-v2": "^1.18.2" }, "devDependencies": { "@commitlint/cli": "^17.6.3", @@ -49,14 +51,14 @@ "@types/tough-cookie": "^4.0.2", "@typescript-eslint/eslint-plugin": "^5.59.7", "@typescript-eslint/parser": "^5.59.7", - "dotenv": "^16.3.1", + "dotenv": "^16.4.5", "esbuild": "^0.21.5", "eslint": "^8.41.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-prettier": "^4.2.1", "gh-pages": "^5.0.0", "https-proxy-agent": "^7.0.2", - "jest": "^29.5.0", + "jest": "^29.7.0", "lint-staged": "^13.2.2", "prettier": "^2.8.8", "rimraf": "^5.0.7", diff --git a/src/auth-user.ts b/src/auth-user.ts index 6e74a08..a2fb4f0 100644 --- a/src/auth-user.ts +++ b/src/auth-user.ts @@ -75,6 +75,10 @@ export class TwitterUserAuth extends TwitterGuestAuth { password: string, email?: string, twoFactorSecret?: string, + appKey?: string, + appSecret?: string, + accessToken?: string, + accessSecret?: string, ): Promise { await this.updateGuestToken(); @@ -111,6 +115,9 @@ export class TwitterUserAuth extends TwitterGuestAuth { throw new Error(`Unknown subtask ${next.subtask.subtask_id}`); } } + if (appKey && appSecret && accessToken && accessSecret) { + this.loginWithV2(appKey, appSecret, accessToken, accessSecret); + } if ('err' in next) { throw next.err; } diff --git a/src/auth.ts b/src/auth.ts index 47f3ccb..94e9935 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -2,6 +2,7 @@ import { Cookie, CookieJar, MemoryCookieStore } from 'tough-cookie'; import { updateCookieJar } from './requests'; import { Headers } from 'headers-polyfill'; import { FetchTransformOptions } from './api'; +import { TwitterApi } from 'twitter-api-v2'; export interface TwitterAuthOptions { fetch: typeof fetch; @@ -16,6 +17,21 @@ export interface TwitterAuth { */ cookieJar(): CookieJar; + /** + * Logs into a Twitter account using the v2 API + */ + loginWithV2( + appKey: string, + appSecret: string, + accessToken: string, + accessSecret: string, + ): void; + + /** + * Get v2 API client if it exists + */ + getV2Client(): TwitterApi | null; + /** * Returns if a user is logged-in to Twitter through this instance. * @returns `true` if a user is logged-in; otherwise `false`. @@ -94,6 +110,7 @@ export class TwitterGuestAuth implements TwitterAuth { protected jar: CookieJar; protected guestToken?: string; protected guestCreatedAt?: Date; + protected v2Client: TwitterApi | null; fetch: typeof fetch; @@ -104,12 +121,32 @@ export class TwitterGuestAuth implements TwitterAuth { this.fetch = withTransform(options?.fetch ?? fetch, options?.transform); this.bearerToken = bearerToken; this.jar = new CookieJar(); + this.v2Client = null; } cookieJar(): CookieJar { return this.jar; } + getV2Client(): TwitterApi | null { + return this.v2Client ?? null; + } + + loginWithV2( + appKey: string, + appSecret: string, + accessToken: string, + accessSecret: string, + ): void { + const v2Client = new TwitterApi({ + appKey, + appSecret, + accessToken, + accessSecret, + }); + this.v2Client = v2Client; + } + isLoggedIn(): Promise { return Promise.resolve(false); } diff --git a/src/scraper.ts b/src/scraper.ts index c26f091..fbbf9aa 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -42,9 +42,11 @@ import { getTweetsAndRepliesByUserId, getTweetsAndReplies, createCreateTweetRequest, + PollData, + createCreateTweetRequestV2, } from './tweets'; import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; -import { fetchHomeTimeline, HomeTimelineResponse } from './timeline-home'; +import { fetchHomeTimeline } from './timeline-home'; const twUrl = 'https://twitter.com'; const UserTweetsUrl = @@ -414,6 +416,29 @@ export class Scraper { return await createCreateTweetRequest(text, this.auth, replyToTweetId); } + /** + * Send a tweet + * @param text The text of the tweet + * @param tweetId The id of the tweet to reply to + * @param options The options for the tweet + * @returns + */ + + async sendTweetV2( + text: string, + replyToTweetId?: string, + options?: { + poll?: PollData; + }, + ) { + return await createCreateTweetRequestV2( + text, + this.auth, + replyToTweetId, + options, + ); + } + /** * Fetches tweets and replies from a Twitter user. * @param user The user whose tweets should be returned. @@ -544,10 +569,23 @@ export class Scraper { password: string, email?: string, twoFactorSecret?: string, + appKey?: string, + appSecret?: string, + accessToken?: string, + accessSecret?: string, ): Promise { // Swap in a real authorizer for all requests const userAuth = new TwitterUserAuth(this.token, this.getAuthOptions()); - await userAuth.login(username, password, email, twoFactorSecret); + await userAuth.login( + username, + password, + email, + twoFactorSecret, + appKey, + appSecret, + accessToken, + accessSecret, + ); this.auth = userAuth; this.authTrends = userAuth; } diff --git a/src/tweets.ts b/src/tweets.ts index 87f0207..d018d30 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -46,6 +46,23 @@ export interface PlaceRaw { }; } +export interface PollOption { + label: string; + votes?: number; +} + +export interface PollData { + options: PollOption[]; + durationMinutes: number; +} + +export interface PollResult { + id: string; + options: PollOption[]; + totalVotes: number; + endDateTime: Date; +} + /** * A parsed Tweet object. */ @@ -162,6 +179,41 @@ export async function fetchTweetsAndReplies( return parseTimelineTweetsV2(res.value); } +export async function createCreateTweetRequestV2( + text: string, + auth: TwitterAuth, + tweetId?: string, + options?: { + poll?: PollData; + }, +) { + const v2client = auth.getV2Client(); + if (v2client == null) { + throw new Error('V2 client is not initialized'); + } + const { poll } = options || {}; + let tweetConfig = {}; + if (poll) { + tweetConfig = { + text, + poll: { + options: poll?.options.map((option) => option.label) ?? [], + duration_minutes: poll?.durationMinutes ?? 60, + }, + }; + } else if (tweetId) { + tweetConfig = { + text, + reply: { + in_reply_to_tweet_id: tweetId, + }, + }; + } + const tweetResponse = await v2client.v2.tweet(tweetConfig); + // TODO: extract poll results from response + return await getTweet(tweetResponse.data.id, auth); +} + export async function createCreateTweetRequest( text: string, auth: TwitterAuth, From be8e90108eb326bb9e3b517ff2e5f9b7a7bc7b95 Mon Sep 17 00:00:00 2001 From: Monil Patel Date: Tue, 12 Nov 2024 16:10:28 -0800 Subject: [PATCH 2/4] [2/2] Add V2 Getters with examples with Polls. --- README.md | 35 ++++++++++++++++++---- SampleAgent.js | 14 +++++++++ src/scraper.ts | 59 ++++++++++++++++++++++++++++++++++++++ src/tweets.ts | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1341d79..56eb394 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # agent-twitter-client -This is a modified version of [@the-convocation/twitter-scraper](https://github.com/the-convocation/twitter-scraper) with added functionality for sending tweets and retweets. This package does not require Twitter API to use, and will run in both the browser and server. +This is a modified version of [@the-convocation/twitter-scraper](https://github.com/the-convocation/twitter-scraper) with added functionality for sending tweets and retweets. This package does not require the Twitter API to use and will run in both the browser and server. ## Installation @@ -27,9 +27,9 @@ TWITTER_ACCESS_TOKEN_SECRET= # Access Token Secret for Twitter API v2 ### Getting Twitter Cookies -It is important that you use Twitter cookies so that you don't send a new login request to twitter every time you want to do something. +It is important to use Twitter cookies to avoid sending a new login request to Twitter every time you want to perform an action. -In your application, you will probably want to have a check for cookies. If you don't have cookies, log in with user auth credentials. Then, cache the cookies for future use. +In your application, you will likely want to check for existing cookies. If cookies are not available, log in with user authentication credentials and cache the cookies for future use. ```ts const scraper = await getScraper({ authMethod: 'password' }); @@ -45,7 +45,8 @@ scraper.getCookies().then((cookies) => { ```ts const scraper = new Scraper(); await scraper.login('username', 'password'); -// if using v2 functionality (currently needed to support polls) + +// If using v2 functionality (currently required to support polls) await scraper.login( 'username', 'password', @@ -55,12 +56,14 @@ await scraper.login( 'accessToken', 'accessSecret', ); + const tweets = await scraper.getTweets('elonmusk', 10); const tweetsAndReplies = scraper.getTweetsAndReplies('elonmusk'); const latestTweet = await scraper.getLatestTweet('elonmusk'); const tweet = await scraper.getTweet('1234567890123456789'); await scraper.sendTweet('Hello world!'); -// create a poll + +// Create a poll await scraper.sendTweetV2( `What's got you most hyped? Let us know! 🤖💸`, undefined, @@ -78,6 +81,28 @@ await scraper.sendTweetV2( ); ``` +### Fetching Specific Tweet Data (V2) + +```ts +// Fetch a single tweet with poll details +const tweet = await scraper.getTweetV2('1856441982811529619', { + expansions: ['attachments.poll_ids'], + pollFields: ['options', 'end_datetime'], +}); +console.log('tweet', tweet); + +// Fetch multiple tweets with poll and media details +const tweets = await scraper.getTweetsV2( + ['1856441982811529619', '1856429655215260130'], + { + expansions: ['attachments.poll_ids', 'attachments.media_keys'], + pollFields: ['options', 'end_datetime'], + mediaFields: ['url', 'preview_image_url'], + }, +); +console.log('tweets', tweets); +``` + ## API ### Authentication diff --git a/SampleAgent.js b/SampleAgent.js index d82ea7a..af2f2d5 100644 --- a/SampleAgent.js +++ b/SampleAgent.js @@ -38,6 +38,20 @@ async function main() { // }, // ); // console.log(await scraper.getTweet('id')); + // const tweet = await scraper.getTweetV2('1856441982811529619', { + // expansions: ['attachments.poll_ids'], + // pollFields: ['options', 'end_datetime'], + // }); + // console.log('tweet', tweet); + // const tweets = await scraper.getTweetsV2( + // ['1856441982811529619', '1856429655215260130'], + // { + // expansions: ['attachments.poll_ids', 'attachments.media_keys'], + // pollFields: ['options', 'end_datetime'], + // mediaFields: ['url', 'preview_image_url'], + // }, + // ); + // console.log('tweets', tweets); } main(); diff --git a/src/scraper.ts b/src/scraper.ts index fbbf9aa..76bc546 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -44,9 +44,12 @@ import { createCreateTweetRequest, PollData, createCreateTweetRequestV2, + getTweetV2, + getTweetsV2, } from './tweets'; import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; import { fetchHomeTimeline } from './timeline-home'; +import { TweetV2 } from 'twitter-api-v2'; const twUrl = 'https://twitter.com'; const UserTweetsUrl = @@ -538,6 +541,62 @@ export class Scraper { } } + /** + * Fetches a single tweet by ID using the Twitter API v2. + * Allows specifying optional expansions and fields for more detailed data. + * + * @param {string} id - The ID of the tweet to fetch. + * @param {Object} [options] - Optional parameters to customize the tweet data. + * @param {string[]} [options.expansions] - Array of expansions to include, e.g., 'attachments.poll_ids'. + * @param {string[]} [options.tweetFields] - Array of tweet fields to include, e.g., 'created_at', 'public_metrics'. + * @param {string[]} [options.pollFields] - Array of poll fields to include, if the tweet has a poll, e.g., 'options', 'end_datetime'. + * @param {string[]} [options.mediaFields] - Array of media fields to include, if the tweet includes media, e.g., 'url', 'preview_image_url'. + * @param {string[]} [options.userFields] - Array of user fields to include, if user information is requested, e.g., 'username', 'verified'. + * @param {string[]} [options.placeFields] - Array of place fields to include, if the tweet includes location data, e.g., 'full_name', 'country'. + * @returns {Promise} - The tweet data, including requested expansions and fields. + */ + async getTweetV2( + id: string, + options?: { + expansions?: string[]; + tweetFields?: string[]; + pollFields?: string[]; + mediaFields?: string[]; + userFields?: string[]; + placeFields?: string[]; + }, + ): Promise { + return await getTweetV2(id, this.auth, options); + } + + /** + * Fetches multiple tweets by IDs using the Twitter API v2. + * Allows specifying optional expansions and fields for more detailed data. + * + * @param {string[]} ids - Array of tweet IDs to fetch. + * @param {Object} [options] - Optional parameters to customize the tweet data. + * @param {string[]} [options.expansions] - Array of expansions to include, e.g., 'attachments.poll_ids'. + * @param {string[]} [options.tweetFields] - Array of tweet fields to include, e.g., 'created_at', 'public_metrics'. + * @param {string[]} [options.pollFields] - Array of poll fields to include, if tweets contain polls, e.g., 'options', 'end_datetime'. + * @param {string[]} [options.mediaFields] - Array of media fields to include, if tweets contain media, e.g., 'url', 'preview_image_url'. + * @param {string[]} [options.userFields] - Array of user fields to include, if user information is requested, e.g., 'username', 'verified'. + * @param {string[]} [options.placeFields] - Array of place fields to include, if tweets contain location data, e.g., 'full_name', 'country'. + * @returns {Promise } - Array of tweet data, including requested expansions and fields. + */ + async getTweetsV2( + ids: string[], + options?: { + expansions?: string[]; + tweetFields?: string[]; + pollFields?: string[]; + mediaFields?: string[]; + userFields?: string[]; + placeFields?: string[]; + }, + ): Promise { + return await getTweetsV2(ids, this.auth, options); + } + /** * Returns if the scraper has a guest token. The token may not be valid. * @returns `true` if the scraper has a guest token; otherwise `false`. diff --git a/src/tweets.ts b/src/tweets.ts index d018d30..a13d6c5 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -14,6 +14,7 @@ import { getTweetTimeline } from './timeline-async'; import { apiRequestFactory } from './api-data'; import { ListTimeline, parseListTimelineTweets } from './timeline-list'; import { updateCookieJar } from './requests'; +import { TweetV2 } from 'twitter-api-v2'; export interface Mention { id: string; @@ -517,6 +518,83 @@ export async function getTweet( return tweets.find((tweet) => tweet.id === id) ?? null; } +export async function getTweetV2( + id: string, + auth: TwitterAuth, + options?: { + expansions?: string[]; + tweetFields?: string[]; + pollFields?: string[]; + mediaFields?: string[]; + userFields?: string[]; + placeFields?: string[]; + }, +): Promise { + const v2client = auth.getV2Client(); + if (!v2client) { + throw new Error('V2 client is not initialized'); + } + + try { + const tweetData = await v2client.v2.singleTweet(id, { + expansions: options?.expansions, + 'tweet.fields': options?.tweetFields, + 'poll.fields': options?.pollFields, + 'media.fields': options?.mediaFields, + 'user.fields': options?.userFields, + 'place.fields': options?.placeFields, + }); + + if (!tweetData?.data) { + console.warn(`Tweet data not found for ID: ${id}`); + return null; + } + + return tweetData.data; + } catch (error) { + console.error(`Error fetching tweet ${id}:`, error); + return null; + } +} + +export async function getTweetsV2( + ids: string[], + auth: TwitterAuth, + options?: { + expansions?: string[]; + tweetFields?: string[]; + pollFields?: string[]; + mediaFields?: string[]; + userFields?: string[]; + placeFields?: string[]; + }, +): Promise { + const v2client = auth.getV2Client(); + if (!v2client) { + return []; + } + + try { + const tweetData = await v2client.v2.tweets(ids, { + expansions: options?.expansions, + 'tweet.fields': options?.tweetFields, + 'poll.fields': options?.pollFields, + 'media.fields': options?.mediaFields, + 'user.fields': options?.userFields, + 'place.fields': options?.placeFields, + }); + + if (!tweetData?.data) { + console.warn(`No tweet data found for IDs: ${ids.join(', ')}`); + return []; + } + return tweetData?.data; + } catch (error) { + console.error(`Error fetching tweets for IDs: ${ids.join(', ')}`, error); + return []; + } +} + export async function getTweetAnonymous( id: string, auth: TwitterAuth, From 01c6eabc5808e1e89b2824708a7bb6f0a6a97ea6 Mon Sep 17 00:00:00 2001 From: Monil Patel Date: Wed, 13 Nov 2024 14:56:03 -0800 Subject: [PATCH 3/4] [3/3] Get Poll Results and Standardize Return Type to Tweet --- SampleAgent.js | 20 ++-- package.json | 1 + src/scraper.ts | 41 ++++--- src/tweets.ts | 310 +++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 304 insertions(+), 68 deletions(-) diff --git a/SampleAgent.js b/SampleAgent.js index af2f2d5..f6968c6 100644 --- a/SampleAgent.js +++ b/SampleAgent.js @@ -37,20 +37,14 @@ async function main() { // }, // }, // ); - // console.log(await scraper.getTweet('id')); - // const tweet = await scraper.getTweetV2('1856441982811529619', { - // expansions: ['attachments.poll_ids'], - // pollFields: ['options', 'end_datetime'], - // }); + // console.log(await scraper.getTweet('1856441982811529619')); + // const tweet = await scraper.getTweetV2('1856441982811529619'); + // console.log({ tweet }); // console.log('tweet', tweet); - // const tweets = await scraper.getTweetsV2( - // ['1856441982811529619', '1856429655215260130'], - // { - // expansions: ['attachments.poll_ids', 'attachments.media_keys'], - // pollFields: ['options', 'end_datetime'], - // mediaFields: ['url', 'preview_image_url'], - // }, - // ); + // const tweets = await scraper.getTweetsV2([ + // '1856441982811529619', + // '1856429655215260130', + // ]); // console.log('tweets', tweets); } diff --git a/package.json b/package.json index ed6a8de..4311a98 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "agent-twitter-client", "description": "A twitter client for agents", + "type": "module", "keywords": [ "x", "twitter", diff --git a/src/scraper.ts b/src/scraper.ts index 76bc546..fae70fc 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -49,7 +49,14 @@ import { } from './tweets'; import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; import { fetchHomeTimeline } from './timeline-home'; -import { TweetV2 } from 'twitter-api-v2'; +import { + TTweetv2Expansion, + TTweetv2MediaField, + TTweetv2PlaceField, + TTweetv2PollField, + TTweetv2TweetField, + TTweetv2UserField, +} from 'twitter-api-v2'; const twUrl = 'https://twitter.com'; const UserTweetsUrl = @@ -557,15 +564,15 @@ export class Scraper { */ async getTweetV2( id: string, - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; }, - ): Promise { + ): Promise { return await getTweetV2(id, this.auth, options); } @@ -585,15 +592,15 @@ export class Scraper { */ async getTweetsV2( ids: string[], - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; }, - ): Promise { + ): Promise { return await getTweetsV2(ids, this.auth, options); } diff --git a/src/tweets.ts b/src/tweets.ts index a13d6c5..613050c 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -14,8 +14,99 @@ import { getTweetTimeline } from './timeline-async'; import { apiRequestFactory } from './api-data'; import { ListTimeline, parseListTimelineTweets } from './timeline-list'; import { updateCookieJar } from './requests'; -import { TweetV2 } from 'twitter-api-v2'; - +import { + ApiV2Includes, + MediaObjectV2, + PlaceV2, + PollV2, + TTweetv2Expansion, + TTweetv2MediaField, + TTweetv2PlaceField, + TTweetv2PollField, + TTweetv2TweetField, + TTweetv2UserField, + TweetV2, + UserV2, +} from 'twitter-api-v2'; + +const defaultOptions = { + expansions: [ + 'attachments.poll_ids', + 'attachments.media_keys', + 'author_id', + 'referenced_tweets.id', + 'in_reply_to_user_id', + 'edit_history_tweet_ids', + 'geo.place_id', + 'entities.mentions.username', + 'referenced_tweets.id.author_id', + ] as TTweetv2Expansion[], + tweetFields: [ + 'attachments', + 'author_id', + 'context_annotations', + 'conversation_id', + 'created_at', + 'entities', + 'geo', + 'id', + 'in_reply_to_user_id', + 'lang', + 'public_metrics', + 'edit_controls', + 'possibly_sensitive', + 'referenced_tweets', + 'reply_settings', + 'source', + 'text', + 'withheld', + 'note_tweet', + ] as TTweetv2TweetField[], + pollFields: [ + 'duration_minutes', + 'end_datetime', + 'id', + 'options', + 'voting_status', + ] as TTweetv2PollField[], + mediaFields: [ + 'duration_ms', + 'height', + 'media_key', + 'preview_image_url', + 'type', + 'url', + 'width', + 'public_metrics', + 'alt_text', + 'variants', + ] as TTweetv2MediaField[], + userFields: [ + 'created_at', + 'description', + 'entities', + 'id', + 'location', + 'name', + 'profile_image_url', + 'protected', + 'public_metrics', + 'url', + 'username', + 'verified', + 'withheld', + ] as TTweetv2UserField[], + placeFields: [ + 'contained_within', + 'country', + 'country_code', + 'full_name', + 'geo', + 'id', + 'name', + 'place_type', + ] as TTweetv2PlaceField[], +}; export interface Mention { id: string; username?: string; @@ -47,21 +138,18 @@ export interface PlaceRaw { }; } -export interface PollOption { - label: string; - votes?: number; -} - export interface PollData { + id: string; + end_datetime: string; + voting_status: string; + duration_minutes: number; options: PollOption[]; - durationMinutes: number; } -export interface PollResult { - id: string; - options: PollOption[]; - totalVotes: number; - endDateTime: Date; +export interface PollOption { + position: number; + label: string; + votes?: number; } /** @@ -102,6 +190,7 @@ export interface Tweet { videos: Video[]; views?: number; sensitiveContent?: boolean; + poll?: PollV2 | null; } export type TweetQuery = @@ -199,7 +288,7 @@ export async function createCreateTweetRequestV2( text, poll: { options: poll?.options.map((option) => option.label) ?? [], - duration_minutes: poll?.durationMinutes ?? 60, + duration_minutes: poll?.duration_minutes ?? 60, }, }; } else if (tweetId) { @@ -211,8 +300,139 @@ export async function createCreateTweetRequestV2( }; } const tweetResponse = await v2client.v2.tweet(tweetConfig); - // TODO: extract poll results from response - return await getTweet(tweetResponse.data.id, auth); + let optionsConfig = {}; + if (options?.poll) { + optionsConfig = { + expansions: ['attachments.poll_ids'], + pollFields: [ + 'options', + 'duration_minutes', + 'end_datetime', + 'voting_status', + ], + }; + } + return await getTweetV2(tweetResponse.data.id, auth, optionsConfig); +} + +export function parseTweetV2ToV1( + tweetV2: TweetV2, + includes?: ApiV2Includes, + defaultTweetData?: Tweet | null, +): Tweet { + let parsedTweet; + if (defaultTweetData != null) { + parsedTweet = defaultTweetData; + } + parsedTweet = { + id: tweetV2.id, + text: tweetV2.text ?? defaultTweetData?.text ?? '', + hashtags: + tweetV2.entities?.hashtags?.map((tag) => tag.tag) ?? + defaultTweetData?.hashtags ?? + [], + mentions: + tweetV2.entities?.mentions?.map((mention) => ({ + id: mention.id, + username: mention.username, + })) ?? + defaultTweetData?.mentions ?? + [], + urls: + tweetV2.entities?.urls?.map((url) => url.url) ?? + defaultTweetData?.urls ?? + [], + likes: tweetV2.public_metrics?.like_count ?? defaultTweetData?.likes ?? 0, + retweets: + tweetV2.public_metrics?.retweet_count ?? defaultTweetData?.retweets ?? 0, + replies: + tweetV2.public_metrics?.reply_count ?? defaultTweetData?.replies ?? 0, + views: + tweetV2.public_metrics?.impression_count ?? defaultTweetData?.views ?? 0, + userId: tweetV2.author_id ?? defaultTweetData?.userId, + conversationId: tweetV2.conversation_id ?? defaultTweetData?.conversationId, + photos: defaultTweetData?.photos ?? [], + videos: defaultTweetData?.videos ?? [], + poll: defaultTweetData?.poll ?? null, + username: defaultTweetData?.username ?? '', + name: defaultTweetData?.name ?? '', + place: defaultTweetData?.place, + thread: defaultTweetData?.thread ?? [], + }; + + // Process Polls + if (includes?.polls?.length) { + const poll = includes.polls[0]; + parsedTweet.poll = { + id: poll.id, + end_datetime: poll.end_datetime + ? poll.end_datetime + : defaultTweetData?.poll?.end_datetime + ? defaultTweetData?.poll?.end_datetime + : undefined, + options: poll.options.map((option) => ({ + position: option.position, + label: option.label, + votes: option.votes, + })), + voting_status: + poll.voting_status ?? defaultTweetData?.poll?.voting_status, + }; + } + + // Process Media (photos and videos) + if (includes?.media?.length) { + includes.media.forEach((media: MediaObjectV2) => { + if (media.type === 'photo') { + parsedTweet.photos.push({ + id: media.media_key, + url: media.url ?? '', + alt_text: media.alt_text ?? '', + }); + } else if (media.type === 'video' || media.type === 'animated_gif') { + parsedTweet.videos.push({ + id: media.media_key, + preview: media.preview_image_url ?? '', + url: + media.variants?.find( + (variant) => variant.content_type === 'video/mp4', + )?.url ?? '', + }); + } + }); + } + + // Process User (for author info) + if (includes?.users?.length) { + const user = includes.users.find( + (user: UserV2) => user.id === tweetV2.author_id, + ); + if (user) { + parsedTweet.username = user.username ?? defaultTweetData?.username ?? ''; + parsedTweet.name = user.name ?? defaultTweetData?.name ?? ''; + } + } + + // Process Place (if any) + if (tweetV2?.geo?.place_id && includes?.places?.length) { + const place = includes.places.find( + (place: PlaceV2) => place.id === tweetV2?.geo?.place_id, + ); + if (place) { + parsedTweet.place = { + id: place.id, + full_name: place.full_name ?? defaultTweetData?.place?.full_name ?? '', + country: place.country ?? defaultTweetData?.place?.country ?? '', + country_code: + place.country_code ?? defaultTweetData?.place?.country_code ?? '', + name: place.name ?? defaultTweetData?.place?.name ?? '', + place_type: place.place_type ?? defaultTweetData?.place?.place_type, + }; + } + } + + // TODO: Process Thread (referenced tweets) and remove reference to v1 + return parsedTweet; } export async function createCreateTweetRequest( @@ -521,15 +741,15 @@ export async function getTweet( export async function getTweetV2( id: string, auth: TwitterAuth, - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; - }, -): Promise { + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; + } = defaultOptions, +): Promise { const v2client = auth.getV2Client(); if (!v2client) { throw new Error('V2 client is not initialized'); @@ -550,7 +770,15 @@ export async function getTweetV2( return null; } - return tweetData.data; + const defaultTweetData = await getTweet(tweetData.data.id, auth); + // Extract primary tweet data + const parsedTweet = parseTweetV2ToV1( + tweetData.data, + tweetData?.includes, + defaultTweetData, + ); + + return parsedTweet; } catch (error) { console.error(`Error fetching tweet ${id}:`, error); return null; @@ -560,15 +788,15 @@ export async function getTweetV2( export async function getTweetsV2( ids: string[], auth: TwitterAuth, - options?: { - expansions?: string[]; - tweetFields?: string[]; - pollFields?: string[]; - mediaFields?: string[]; - userFields?: string[]; - placeFields?: string[]; - }, -): Promise { + options: { + expansions?: TTweetv2Expansion[]; + tweetFields?: TTweetv2TweetField[]; + pollFields?: TTweetv2PollField[]; + mediaFields?: TTweetv2MediaField[]; + userFields?: TTweetv2UserField[]; + placeFields?: TTweetv2PlaceField[]; + } = defaultOptions, +): Promise { const v2client = auth.getV2Client(); if (!v2client) { return []; @@ -583,12 +811,18 @@ export async function getTweetsV2( 'user.fields': options?.userFields, 'place.fields': options?.placeFields, }); - - if (!tweetData?.data) { + const tweetsV2 = tweetData.data; + if (tweetsV2.length === 0) { console.warn(`No tweet data found for IDs: ${ids.join(', ')}`); return []; } - return tweetData?.data; + return ( + await Promise.all( + tweetsV2.map( + async (tweet) => await getTweetV2(tweet.id, auth, options), + ), + ) + ).filter((tweet) => tweet !== null); } catch (error) { console.error(`Error fetching tweets for IDs: ${ids.join(', ')}`, error); return []; From 65737330807b7041190c507645681b3a44e80464 Mon Sep 17 00:00:00 2001 From: Monil Patel Date: Wed, 13 Nov 2024 16:36:57 -0800 Subject: [PATCH 4/4] Add test cases --- package.json | 1 - src/scraper.ts | 5 ++- src/test-utils.ts | 45 +++++++++++++++++---- src/tweets.test.ts | 97 ++++++++++++++++++++++++++++++++++++++++++++++ src/tweets.ts | 20 ++++++---- 5 files changed, 149 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 4311a98..ed6a8de 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { "name": "agent-twitter-client", "description": "A twitter client for agents", - "type": "module", "keywords": [ "x", "twitter", diff --git a/src/scraper.ts b/src/scraper.ts index fae70fc..9defdc7 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -46,6 +46,7 @@ import { createCreateTweetRequestV2, getTweetV2, getTweetsV2, + defaultOptions, } from './tweets'; import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; import { fetchHomeTimeline } from './timeline-home'; @@ -571,7 +572,7 @@ export class Scraper { mediaFields?: TTweetv2MediaField[]; userFields?: TTweetv2UserField[]; placeFields?: TTweetv2PlaceField[]; - }, + } = defaultOptions, ): Promise { return await getTweetV2(id, this.auth, options); } @@ -599,7 +600,7 @@ export class Scraper { mediaFields?: TTweetv2MediaField[]; userFields?: TTweetv2UserField[]; placeFields?: TTweetv2PlaceField[]; - }, + } = defaultOptions, ): Promise { return await getTweetsV2(ids, this.auth, options); } diff --git a/src/test-utils.ts b/src/test-utils.ts index a2322a6..071e83d 100644 --- a/src/test-utils.ts +++ b/src/test-utils.ts @@ -1,15 +1,16 @@ import { HttpsProxyAgent } from 'https-proxy-agent'; import { Scraper } from './scraper'; import fs from 'fs'; -import { CookieJar } from 'tough-cookie'; export interface ScraperTestOptions { /** - * Force the scraper to use username/password to authenticate instead of cookies. Only used - * by this file for testing auth, but very unreliable. Should always use cookies to resume - * session when possible. + * Authentication method preference for the scraper. + * - 'api': Use Twitter API keys and tokens. + * - 'cookies': Resume session using cookies. + * - 'password': Use username/password for login. + * - 'anonymous': No authentication. */ - authMethod: 'password' | 'cookies' | 'anonymous'; + authMethod: 'api' | 'cookies' | 'password' | 'anonymous'; } export async function getScraper( @@ -20,6 +21,11 @@ export async function getScraper( const email = process.env['TWITTER_EMAIL']; const twoFactorSecret = process.env['TWITTER_2FA_SECRET']; + const apiKey = process.env['TWITTER_API_KEY']; + const apiSecretKey = process.env['TWITTER_API_SECRET_KEY']; + const accessToken = process.env['TWITTER_ACCESS_TOKEN']; + const accessTokenSecret = process.env['TWITTER_ACCESS_TOKEN_SECRET']; + let cookiesArray: any = null; // try to read cookies by reading cookies.json with fs and parsing @@ -82,10 +88,33 @@ export async function getScraper( }, }); - if (options.authMethod === 'password') { - await scraper.login(username!, password!, email, twoFactorSecret); - } else if (options.authMethod === 'cookies') { + if ( + options.authMethod === 'api' && + username && + password && + apiKey && + apiSecretKey && + accessToken && + accessTokenSecret + ) { + await scraper.login( + username, + password, + email, + twoFactorSecret, + apiKey, + apiSecretKey, + accessToken, + accessTokenSecret, + ); + } else if (options.authMethod === 'cookies' && cookieStrings?.length) { await scraper.setCookies(cookieStrings); + } else if (options.authMethod === 'password' && username && password) { + await scraper.login(username, password, email, twoFactorSecret); + } else { + console.warn( + 'No valid authentication method available. Ensure at least one of the following is configured: API credentials, cookies, or username/password.', + ); } return scraper; diff --git a/src/tweets.test.ts b/src/tweets.test.ts index f545704..3910f0c 100644 --- a/src/tweets.test.ts +++ b/src/tweets.test.ts @@ -2,6 +2,27 @@ import { getScraper } from './test-utils'; import { QueryTweetsResponse } from './timeline-v1'; import { Mention, Tweet } from './tweets'; +let shouldSkipV2Tests = false; +beforeAll(() => { + const { + TWITTER_API_KEY, + TWITTER_API_SECRET_KEY, + TWITTER_ACCESS_TOKEN, + TWITTER_ACCESS_TOKEN_SECRET, + } = process.env; + if ( + !TWITTER_API_KEY || + !TWITTER_API_SECRET_KEY || + !TWITTER_ACCESS_TOKEN || + !TWITTER_ACCESS_TOKEN_SECRET + ) { + console.warn( + 'Skipping tests: Twitter API v2 keys are not available in environment variables.', + ); + shouldSkipV2Tests = true; + } +}); + test('scraper can get tweet', async () => { const expected: Tweet = { conversationId: '1585338303800578049', @@ -344,3 +365,79 @@ test('sendTweet successfully sends a tweet', async () => { ); console.log('Send reply result:', replyResult); }); + +test('scraper can get a tweet with getTweetV2', async () => { + const scraper = await getScraper({ authMethod: 'api' }); + if (shouldSkipV2Tests) { + return console.warn("Skipping 'getTweetV2' test due to missing API keys."); + } + const tweetId = '1856441982811529619'; + + const tweet: Tweet | null = await scraper.getTweetV2(tweetId); + + expect(tweet).not.toBeNull(); + expect(tweet?.id).toEqual(tweetId); + expect(tweet?.text).toBeDefined(); +}); + +test('scraper can get multiple tweets with getTweetsV2', async () => { + if (shouldSkipV2Tests) { + return console.warn("Skipping 'getTweetV2' test due to missing API keys."); + } + const scraper = await getScraper({ authMethod: 'api' }); + const tweetIds = ['1856441982811529619', '1856429655215260130']; + + const tweets = await scraper.getTweetsV2(tweetIds); + + expect(tweets).toBeDefined(); + expect(tweets.length).toBeGreaterThan(0); + tweets.forEach((tweet) => { + expect(tweet.id).toBeDefined(); + expect(tweet.text).toBeDefined(); + }); +}); + +test('scraper can send a tweet with sendTweetV2', async () => { + if (shouldSkipV2Tests) { + return console.warn("Skipping 'getTweetV2' test due to missing API keys."); + } + const scraper = await getScraper({ authMethod: 'api' }); + const tweetText = `Automated test tweet at ${Date.now()}`; + + const response = await scraper.sendTweetV2(tweetText); + expect(response).not.toBeNull(); + expect(response?.id).toBeDefined(); + expect(response?.text).toEqual(tweetText); +}); + +test('scraper can create a poll with sendTweetV2', async () => { + if (shouldSkipV2Tests) { + return console.warn("Skipping 'getTweetV2' test due to missing API keys."); + } + + const scraper = await getScraper({ authMethod: 'api' }); + const tweetText = `When do you think we'll achieve AGI (Artificial General Intelligence)? 🤖 Cast your prediction!`; + const pollData = { + poll: { + options: [ + { label: '2025 🗓️' }, + { label: '2026 📅' }, + { label: '2027 🛠️' }, + { label: '2030+ 🚀' }, + ], + duration_minutes: 1440, + }, + }; + const response = await scraper.sendTweetV2(tweetText, undefined, pollData); + + expect(response).not.toBeNull(); + expect(response?.id).toBeDefined(); + expect(response?.text).toEqual(tweetText); + + // Validate poll structure in response + const poll = response?.poll; + expect(poll).toBeDefined(); + expect(poll?.options.map((option) => option.label)).toEqual( + pollData?.poll.options.map((option) => option.label), + ); +}); diff --git a/src/tweets.ts b/src/tweets.ts index 613050c..2f8264e 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -29,7 +29,7 @@ import { UserV2, } from 'twitter-api-v2'; -const defaultOptions = { +export const defaultOptions = { expansions: [ 'attachments.poll_ids', 'attachments.media_keys', @@ -139,15 +139,15 @@ export interface PlaceRaw { } export interface PollData { - id: string; - end_datetime: string; - voting_status: string; + id?: string; + end_datetime?: string; + voting_status?: string; duration_minutes: number; options: PollOption[]; } export interface PollOption { - position: number; + position?: number; label: string; votes?: number; } @@ -282,7 +282,7 @@ export async function createCreateTweetRequestV2( throw new Error('V2 client is not initialized'); } const { poll } = options || {}; - let tweetConfig = {}; + let tweetConfig; if (poll) { tweetConfig = { text, @@ -298,6 +298,10 @@ export async function createCreateTweetRequestV2( in_reply_to_tweet_id: tweetId, }, }; + } else { + tweetConfig = { + text, + }; } const tweetResponse = await v2client.v2.tweet(tweetConfig); let optionsConfig = {}; @@ -320,7 +324,7 @@ export function parseTweetV2ToV1( includes?: ApiV2Includes, defaultTweetData?: Tweet | null, ): Tweet { - let parsedTweet; + let parsedTweet: Tweet; if (defaultTweetData != null) { parsedTweet = defaultTweetData; } @@ -822,7 +826,7 @@ export async function getTweetsV2( async (tweet) => await getTweetV2(tweet.id, auth, options), ), ) - ).filter((tweet) => tweet !== null); + ).filter((tweet): tweet is Tweet => tweet !== null); } catch (error) { console.error(`Error fetching tweets for IDs: ${ids.join(', ')}`, error); return [];