From 87052eaff2c6e3a8c97e57b2358ede725c33c1dc Mon Sep 17 00:00:00 2001 From: Pakorn Nathong Date: Thu, 2 Jan 2025 00:53:34 +0700 Subject: [PATCH 1/6] feat: add basic article type --- src/timeline-v1.ts | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/timeline-v1.ts b/src/timeline-v1.ts index d9b4661..8ea409f 100644 --- a/src/timeline-v1.ts +++ b/src/timeline-v1.ts @@ -76,6 +76,49 @@ export interface SearchResultRaw { legacy?: LegacyTweetRaw; } +export interface TimelineArticleResultRaw { + id?: string; + title?: string; + preview_text?: string; + cover_media?: { + media_id?: string; + media_info?: { + original_img_url?: string; + original_img_height?: number; + original_img_width?: number; + }; + }; + content_state?: { + blocks?: { + key?: string; + data?: string; + text?: string; + entityRanges?: { + key?: number; + length?: number; + offset?: number; + }[]; + }[]; + }; + entityMap?: { + key?: string; + value?: { + type?: string; // LINK, MEDIA, TWEET + mutability?: string; + data?: { + entityKey?: string; + url?: string; + tweetId?: string; + mediaItems?: { + localMediaId?: string; + mediaCategory?: string; + mediaId?: string; + }[]; + }; + }; + }[]; +} + export interface TimelineResultRaw { rest_id?: string; __typename?: string; @@ -97,6 +140,11 @@ export interface TimelineResultRaw { }; }; }; + article?: { + article_results?: { + result?: TimelineArticleResultRaw; + }; + }; quoted_status_result?: { result?: TimelineResultRaw; }; From 2e592291ca105dab2a6c8477c35044212eebed2f Mon Sep 17 00:00:00 2001 From: Pakorn Nathong Date: Thu, 2 Jan 2025 01:05:50 +0700 Subject: [PATCH 2/6] feat: add parse article fn --- src/timeline-v2.ts | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/timeline-v2.ts b/src/timeline-v2.ts index 61bf4b8..592b4ff 100644 --- a/src/timeline-v2.ts +++ b/src/timeline-v2.ts @@ -423,3 +423,41 @@ export function parseThreadedConversation( return tweets; } + +export interface TimelineArticle { + id: string; + articleId: string; + title: string; + previewText: string; + coverMediaUrl?: string; + text: string; +} + +export function parseArticle( + conversation: ThreadedConversation, +): TimelineArticle[] { + const articles: TimelineArticle[] = []; + for (const instruction of conversation.data + ?.threaded_conversation_with_injections_v2?.instructions ?? []) { + for (const entry of instruction.entries ?? []) { + const id = entry.content?.itemContent?.tweet_results?.result?.rest_id; + const article = + entry.content?.itemContent?.tweet_results?.result?.article + ?.article_results?.result; + if (!id || !article) continue; + const text = + article.content_state?.blocks + ?.map((block) => block.text) + .join('\n\n') ?? ''; + articles.push({ + id, + articleId: article.id || '', + coverMediaUrl: article.cover_media?.media_info?.original_img_url, + previewText: article.preview_text || '', + text, + title: article.title || '', + }); + } + } + return articles; +} From 93e509bc069da35d0a16ca9966f657edc387a037 Mon Sep 17 00:00:00 2001 From: Pakorn Nathong Date: Thu, 2 Jan 2025 01:08:00 +0700 Subject: [PATCH 3/6] feat: add TweetDetail endpoint for article --- src/api-data.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api-data.ts b/src/api-data.ts index f784093..c772d89 100644 --- a/src/api-data.ts +++ b/src/api-data.ts @@ -15,6 +15,8 @@ const endpoints = { 'https://twitter.com/i/api/graphql/eSSNbhECHHWWALkkQq-YTA/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D', TweetDetail: 'https://twitter.com/i/api/graphql/xOhkmRac04YFZmOzU9PJHg/TweetDetail?variables=%7B%22focalTweetId%22%3A%221237110546383724547%22%2C%22with_rux_injections%22%3Afalse%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Afalse%7D', + TweetDetailArticle: + 'https://twitter.com/i/api/graphql/GtcBtFhtQymrpxAs5MALVA/TweetDetail?variables=%7B%22focalTweetId%22%3A%221765884209527394325%22%2C%22with_rux_injections%22%3Atrue%2C%22rankingMode%22%3A%22Relevance%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%2C%22withGrokAnalyze%22%3Afalse%2C%22withDisallowedReplyControls%22%3Afalse%7D', TweetResultByRestId: 'https://twitter.com/i/api/graphql/DJS3BdhUhcaEpZ7B7irJDg/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221237110546383724547%22%2C%22withCommunity%22%3Afalse%2C%22includePromotedContent%22%3Afalse%2C%22withVoice%22%3Afalse%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D', ListTweets: From dd6c7945c59eb860adf72fb6436e683c6f9c97ed Mon Sep 17 00:00:00 2001 From: Pakorn Nathong Date: Thu, 2 Jan 2025 01:08:30 +0700 Subject: [PATCH 4/6] feat: add getArticle fn --- src/scraper.ts | 34 ++++++++++++++++++++++++++++------ src/tweets.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/scraper.ts b/src/scraper.ts index c3f8075..ce63e83 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -53,8 +53,13 @@ import { retweet, createCreateNoteTweetRequest, createCreateLongTweetRequest, + getArticle, } from './tweets'; -import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; +import { + parseTimelineTweetsV2, + TimelineArticle, + TimelineV2, +} from './timeline-v2'; import { fetchHomeTimeline } from './timeline-home'; import { fetchFollowingTimeline } from './timeline-following'; import { @@ -75,9 +80,17 @@ import { fetchAudioSpaceById, fetchAuthenticatePeriscope, fetchBrowseSpaceTopics, - fetchCommunitySelectQuery, fetchLiveVideoStreamStatus, fetchLoginTwitterToken + fetchCommunitySelectQuery, + fetchLiveVideoStreamStatus, + fetchLoginTwitterToken, } from './spaces'; -import {AudioSpace, Community, LiveVideoStreamStatus, LoginTwitterTokenResponse, Subtopic} from './types/spaces'; +import { + AudioSpace, + Community, + LiveVideoStreamStatus, + LoginTwitterTokenResponse, + Subtopic, +} from './types/spaces'; const twUrl = 'https://twitter.com'; const UserTweetsUrl = @@ -945,7 +958,7 @@ export class Scraper { * @returns The status of the Audio Space stream. */ public async getAudioSpaceStreamStatus( - mediaKey: string, + mediaKey: string, ): Promise { return await fetchLiveVideoStreamStatus(mediaKey, this.auth); } @@ -958,7 +971,7 @@ export class Scraper { * @returns The status of the Audio Space stream. */ public async getAudioSpaceStatus( - audioSpaceId: string, + audioSpaceId: string, ): Promise { const audioSpace = await this.getAudioSpaceById(audioSpaceId); @@ -984,7 +997,7 @@ export class Scraper { * @returns The response containing the cookie and user information. */ public async loginTwitterToken( - jwt: string, + jwt: string, ): Promise { return await fetchLoginTwitterToken(jwt, this.auth); } @@ -999,4 +1012,13 @@ export class Scraper { return loginResponse.cookie; } + + /** + * Fetches a article (long form tweet) by its ID. + * @param id The ID of the article to fetch. In the format of (http://x.com/i/article/id) + * @returns The {@link TimelineArticle} object, or `null` if it couldn't be fetched. + */ + public getArticle(id: string): Promise { + return getArticle(id, this.auth); + } } diff --git a/src/tweets.ts b/src/tweets.ts index 13b3fda..7e71ff1 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -9,6 +9,8 @@ import { parseTimelineEntryItemContentRaw, ThreadedConversation, parseThreadedConversation, + parseArticle, + TimelineArticle, } from './timeline-v2'; import { getTweetTimeline } from './timeline-async'; import { apiRequestFactory } from './api-data'; @@ -1479,3 +1481,28 @@ export async function createCreateLongTweetRequest( return response; } + +export async function getArticle( + id: string, + auth: TwitterAuth, +): Promise { + const tweetDetailRequest = + apiRequestFactory.createTweetDetailArticleRequest(); + tweetDetailRequest.variables.focalTweetId = id; + + const res = await requestApi( + tweetDetailRequest.toRequestUrl(), + auth, + ); + + if (!res.success) { + throw res.err; + } + + if (!res.value) { + return null; + } + + const articles = parseArticle(res.value); + return articles.find((article) => article.id === id) ?? null; +} From 4df69d77f9145d8591c9e74c13bca3baef668be4 Mon Sep 17 00:00:00 2001 From: Pakorn Nathong Date: Thu, 2 Jan 2025 01:09:59 +0700 Subject: [PATCH 5/6] test: add getTweet and getArticle test for retrieving article --- src/tweets.test.ts | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/tweets.test.ts b/src/tweets.test.ts index 5d1d0a3..3cf3e96 100644 --- a/src/tweets.test.ts +++ b/src/tweets.test.ts @@ -1,6 +1,6 @@ import { getScraper } from './test-utils'; import { QueryTweetsResponse } from './timeline-v1'; -import { Mention, Tweet } from './tweets'; +import { Mention, Tweet, getTweetAnonymous } from './tweets'; import fs from 'fs'; import path from 'path'; @@ -462,10 +462,10 @@ test('scraper can send a tweet with image and video', async () => { // Read test image and video files from the test-assets directory const imageBuffer = fs.readFileSync( - path.join(__dirname, '../test-assets/test-image.jpeg') + path.join(__dirname, '../test-assets/test-image.jpeg'), ); const videoBuffer = fs.readFileSync( - path.join(__dirname, '../test-assets/test-video.mp4') + path.join(__dirname, '../test-assets/test-video.mp4'), ); // Prepare media data array with both image and video @@ -502,10 +502,10 @@ test('scraper can quote tweet with image and video', async () => { // Read test image and video files from the test-assets directory const imageBuffer = fs.readFileSync( - path.join(__dirname, '../test-assets/test-image.jpeg') + path.join(__dirname, '../test-assets/test-image.jpeg'), ); const videoBuffer = fs.readFileSync( - path.join(__dirname, '../test-assets/test-video.mp4') + path.join(__dirname, '../test-assets/test-video.mp4'), ); // Prepare media data array with both image and video @@ -531,13 +531,11 @@ test('scraper can quote tweet with media', async () => { // Read test image file const imageBuffer = fs.readFileSync( - path.join(__dirname, '../test-assets/test-image.jpeg') + path.join(__dirname, '../test-assets/test-image.jpeg'), ); // Prepare media data with the image - const mediaData = [ - { data: imageBuffer, mediaType: 'image/jpeg' }, - ]; + const mediaData = [{ data: imageBuffer, mediaType: 'image/jpeg' }]; // Send a quote tweet with the image attachment const response = await scraper.sendQuoteTweet(quoteText, quotedTweetId, { @@ -555,13 +553,11 @@ test('sendTweetWithMedia successfully sends a tweet with media', async () => { // Read a test image file const imageBuffer = fs.readFileSync( - path.join(__dirname, '../test-assets/test-image.jpeg') + path.join(__dirname, '../test-assets/test-image.jpeg'), ); // Prepare media data with the image - const mediaData = [ - { data: imageBuffer, mediaType: 'image/jpeg' }, - ]; + const mediaData = [{ data: imageBuffer, mediaType: 'image/jpeg' }]; // Send a tweet with the image attachment const result = await scraper.sendTweet(draftText, undefined, mediaData); @@ -593,4 +589,23 @@ test('scraper can follow user', async () => { // Test should not throw an error await expect(scraper.followUser(username)).resolves.not.toThrow(); -}, 30000); \ No newline at end of file +}, 30000); + +test('scraper cannot get article using getTweet', async () => { + const scraper = await getScraper(); + // X introducing article: http://x.com/i/article/1765821414056120320 + const tweet = await scraper.getTweet('1765884209527394325'); + + expect(tweet).not.toBeNull(); + expect(tweet?.text).toMatch(/https?:\/\/t.co\//); + expect(tweet?.urls[0]).toMatch(/https?:\/\/x.com\/i\/article\//); +}, 30000); + +test('scraper can get article using getArticle', async () => { + const scraper = await getScraper(); + // X introducing article: http://x.com/i/article/1765821414056120320 + const article = await scraper.getArticle('1765884209527394325'); + + expect(article).not.toBeNull(); + expect(article?.title).toMatch(/Introducing Articles on X/); +}, 30000); From d71542ad7528d62d3af90d4fd459cd1f0bf72fa2 Mon Sep 17 00:00:00 2001 From: Pakorn Nathong Date: Thu, 2 Jan 2025 01:14:07 +0700 Subject: [PATCH 6/6] fix: use rest_id to infer article id --- src/timeline-v1.ts | 1 + src/timeline-v2.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/timeline-v1.ts b/src/timeline-v1.ts index 8ea409f..196b4c4 100644 --- a/src/timeline-v1.ts +++ b/src/timeline-v1.ts @@ -78,6 +78,7 @@ export interface SearchResultRaw { export interface TimelineArticleResultRaw { id?: string; + rest_id?: string; title?: string; preview_text?: string; cover_media?: { diff --git a/src/timeline-v2.ts b/src/timeline-v2.ts index 592b4ff..125a61d 100644 --- a/src/timeline-v2.ts +++ b/src/timeline-v2.ts @@ -451,7 +451,7 @@ export function parseArticle( .join('\n\n') ?? ''; articles.push({ id, - articleId: article.id || '', + articleId: article.rest_id || '', coverMediaUrl: article.cover_media?.media_info?.original_img_url, previewText: article.preview_text || '', text,