diff --git a/src/scraper.test.ts b/src/scraper.test.ts index a7f1bc2..81357bf 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -14,6 +14,19 @@ test('scraper can fetch home timeline', async () => { expect(homeTimeline[0]?.rest_id).toBeDefined(); }, 30000); +test('scraper can fetch following timeline', async () => { + const scraper = await getScraper(); + + const count = 20; + const seenTweetIds: string[] = []; + + const homeTimeline = await scraper.fetchFollowingTimeline(count, seenTweetIds); + console.log(homeTimeline); + expect(homeTimeline).toBeDefined(); + expect(homeTimeline?.length).toBeGreaterThan(0); + expect(homeTimeline[0]?.rest_id).toBeDefined(); +}, 30000); + test('scraper uses response transform when provided', async () => { const scraper = new Scraper({ transform: { diff --git a/src/scraper.ts b/src/scraper.ts index e08a6ef..3f2dceb 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -54,6 +54,7 @@ import { } from './tweets'; import { parseTimelineTweetsV2, TimelineV2 } from './timeline-v2'; import { fetchHomeTimeline } from './timeline-home'; +import { fetchFollowingTimeline } from './timeline-following'; import { TTweetv2Expansion, TTweetv2MediaField, @@ -274,7 +275,7 @@ export class Scraper { } /** - * Fetches the home timeline for the current user. + * Fetches the home timeline for the current user. (for you feed) * @param count The number of tweets to fetch. * @param seenTweetIds An array of tweet IDs that have already been seen. * @returns A promise that resolves to the home timeline response. @@ -286,6 +287,19 @@ export class Scraper { return await fetchHomeTimeline(count, seenTweetIds, this.auth); } + /** + * Fetches the home timeline for the current user. (following feed) + * @param count The number of tweets to fetch. + * @param seenTweetIds An array of tweet IDs that have already been seen. + * @returns A promise that resolves to the home timeline response. + */ + public async fetchFollowingTimeline( + count: number, + seenTweetIds: string[], + ): Promise { + return await fetchFollowingTimeline(count, seenTweetIds, this.auth); + } + async getUserTweets( userId: string, maxTweets = 200, diff --git a/src/timeline-following.ts b/src/timeline-following.ts new file mode 100644 index 0000000..8c30833 --- /dev/null +++ b/src/timeline-following.ts @@ -0,0 +1,93 @@ +import { requestApi } from './api'; +import { TwitterAuth } from './auth'; +import { ApiError } from './errors'; +import { TimelineInstruction } from './timeline-v2'; + +export interface HomeLatestTimelineResponse { + data?: { + home: { + home_timeline_urt: { + instructions: TimelineInstruction[]; + }; + }; + }; +} + +export async function fetchFollowingTimeline( + count: number, + seenTweetIds: string[], + auth: TwitterAuth, +): Promise { + const variables = { + count, + includePromotedContent: true, + latestControlAvailable: true, + requestContext: 'launch', + seenTweetIds, + }; + + const features = { + profile_label_improvements_pcf_label_in_post_enabled: true, + rweb_tipjar_consumption_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + tweet_awards_web_tipping_enabled: false, + creator_subscriptions_quote_tweet_preview_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: + true, + rweb_video_timestamps_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_enhance_cards_enabled: false, + }; + + const res = await requestApi( + `https://x.com/i/api/graphql/K0X1xbCZUjttdK8RazKAlw/HomeLatestTimeline?variables=${encodeURIComponent( + JSON.stringify(variables), + )}&features=${encodeURIComponent(JSON.stringify(features))}`, + auth, + 'GET', + ); + + if (!res.success) { + if (res.err instanceof ApiError) { + console.error('Error details:', res.err.data); + } + throw res.err; + } + + const home = res.value?.data?.home.home_timeline_urt?.instructions; + + if (!home) { + return []; + } + + const entries: any[] = []; + + for (const instruction of home) { + if (instruction.type === 'TimelineAddEntries') { + for (const entry of instruction.entries ?? []) { + entries.push(entry); + } + } + } + // get the itemContnent from each entry + const tweets = entries + .map((entry) => entry.content.itemContent?.tweet_results?.result) + .filter((tweet) => tweet !== undefined); + + return tweets; +}