From db46d11069dc4e1b8778d2c36d1eb9d6a0510bb0 Mon Sep 17 00:00:00 2001 From: samarth30 Date: Tue, 17 Dec 2024 00:45:28 +0530 Subject: [PATCH 1/3] feat: added scraping for twitter messages and sending twitter messages --- src/messages.ts | 215 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 src/messages.ts diff --git a/src/messages.ts b/src/messages.ts new file mode 100644 index 0000000..975a64f --- /dev/null +++ b/src/messages.ts @@ -0,0 +1,215 @@ +import { TwitterAuth } from './auth'; +import { updateCookieJar } from './requests'; + +export interface DirectMessage { + id: string; + text: string; + senderId: string; + recipientId: string; + createdAt: string; + mediaUrls?: string[]; + senderScreenName?: string; + recipientScreenName?: string; +} + +export interface DirectMessageConversation { + conversationId: string; + messages: DirectMessage[]; + participants: { + id: string; + screenName: string; + }[]; +} + +export interface DirectMessageEvent { + id: string; + type: string; + message_create: { + sender_id: string; + target: { + recipient_id: string; + }; + message_data: { + text: string; + created_at: string; + entities?: { + urls?: Array<{ + url: string; + expanded_url: string; + display_url: string; + }>; + media?: Array<{ + url: string; + type: string; + }>; + }; + }; + }; +} + +export interface DirectMessagesResponse { + events: DirectMessageEvent[]; + apps?: Record; + next_cursor?: string; +} + +function parseDirectMessageConversations( + data: any, +): DirectMessageConversation[] { + try { + const conversations = data.data.inbox.conversations; + return conversations.map((conv: any) => ({ + conversationId: conv.conversation_id, + messages: parseDirectMessages(conv.messages), + participants: conv.participants.map((p: any) => ({ + id: p.user_id, + screenName: p.screen_name, + })), + })); + } catch (error) { + console.error('Error parsing DM conversations:', error); + return []; + } +} + +function parseDirectMessages(data: any): DirectMessage[] { + try { + return data.map((msg: any) => ({ + id: msg.message_id, + text: msg.message_data.text, + senderId: msg.message_data.sender_id, + recipientId: msg.message_data.recipient_id, + createdAt: msg.message_data.created_at, + mediaUrls: msg.message_data.attachment?.media_urls, + senderScreenName: msg.message_data.sender_screen_name, + recipientScreenName: msg.message_data.recipient_screen_name, + })); + } catch (error) { + console.error('Error parsing DMs:', error); + return []; + } +} + +function parseDirectMessageResponse(data: any): DirectMessage { + try { + const msg = data.data.message_create; + return { + id: msg.message_id, + text: msg.message_data.text, + senderId: msg.message_data.sender_id, + recipientId: msg.message_data.recipient_id, + createdAt: msg.message_data.created_at, + mediaUrls: msg.message_data.attachment?.media_urls, + senderScreenName: msg.message_data.sender_screen_name, + recipientScreenName: msg.message_data.recipient_screen_name, + }; + } catch (error) { + console.error('Error parsing DM response:', error); + throw error; + } +} + +export async function getDirectMessageConversations( + auth: TwitterAuth, + cursor: string, +) { + if (!auth.isLoggedIn()) { + throw new Error('Authentication required to fetch direct messages'); + } + + const url = + 'https://twitter.com/i/api/graphql/7s3kOODhC5vgXlO0OlqYdA/DMInboxTimeline'; + const messageListUrl = 'https://x.com/i/api/1.1/dm/inbox_initial_state.json'; + + const params = new URLSearchParams(); + + if (cursor) { + params.append('cursor', cursor); + } + + const finalUrl = `${url}${params.toString() ? '?' + params.toString() : ''}`; + const cookies = await auth.cookieJar().getCookies(url); + const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); + const userTwitterId = cookies.find((cookie) => cookie.key === 'twid'); + + const headers = new Headers({ + authorization: `Bearer ${(auth as any).bearerToken}`, + cookie: await auth.cookieJar().getCookieString(messageListUrl), + 'content-type': 'application/json', + 'User-Agent': + 'Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36', + 'x-guest-token': (auth as any).guestToken, + 'x-twitter-auth-type': 'OAuth2Client', + 'x-twitter-active-user': 'yes', + 'x-csrf-token': xCsrfToken?.value as string, + }); + + const response = await fetch(finalUrl, { + method: 'GET', + headers, + }); + + await updateCookieJar(auth.cookieJar(), response.headers); + + if (!response.ok) { + throw new Error(await response.text()); + } + + // parse the response + const data = await response.json(); + return parseDirectMessageConversations(data); +} + +export async function sendDirectMessage( + auth: TwitterAuth, + senderId: string, + recipientId: string, + text: string, +): Promise { + if (!auth.isLoggedIn()) { + throw new Error('Authentication required to send direct messages'); + } + + const url = + 'https://twitter.com/i/api/graphql/7s3kOODhC5vgXlO0OlqYdA/DMInboxTimeline'; + const messageDmUrl = 'https://x.com/i/api/1.1/dm/new2.json'; + + const cookies = await auth.cookieJar().getCookies(url); + const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); + + const headers = new Headers({ + authorization: `Bearer ${(auth as any).bearerToken}`, + cookie: await auth.cookieJar().getCookieString(url), + 'content-type': 'application/json', + 'User-Agent': + 'Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36', + 'x-guest-token': (auth as any).guestToken, + 'x-twitter-auth-type': 'OAuth2Client', + 'x-twitter-active-user': 'yes', + 'x-csrf-token': xCsrfToken?.value as string, + }); + + const payload = { + conversation_id: `${senderId}-${recipientId}`, + recipient_ids: false, + text: text, + cards_platform: 'Web-12', + include_cards: 1, + include_quote_count: true, + dm_users: false, + }; + + const response = await fetch(messageDmUrl, { + method: 'POST', + headers, + body: JSON.stringify(payload), + }); + + await updateCookieJar(auth.cookieJar(), response.headers); + + if (!response.ok) { + throw new Error(await response.text()); + } + + return (await response.json()).event; +} From bab20b88f41898f3fa7c8872ebee84137cae5e7b Mon Sep 17 00:00:00 2001 From: samarth30 Date: Tue, 17 Dec 2024 14:19:39 +0530 Subject: [PATCH 2/3] feat: parsed users dm conversations and send messages related to users --- src/messages.test.ts | 125 ++++++++++++++++++++++++++ src/messages.ts | 209 +++++++++++++++++++++++++++++++++---------- 2 files changed, 286 insertions(+), 48 deletions(-) create mode 100644 src/messages.test.ts diff --git a/src/messages.test.ts b/src/messages.test.ts new file mode 100644 index 0000000..4d01800 --- /dev/null +++ b/src/messages.test.ts @@ -0,0 +1,125 @@ +import { getScraper } from './test-utils'; +import { jest } from '@jest/globals'; + +let shouldSkipV2Tests = false; +let testUserId: string; +let testConversationId: string; + +beforeAll(async () => { + const { + TWITTER_API_KEY, + TWITTER_API_SECRET_KEY, + TWITTER_ACCESS_TOKEN, + TWITTER_ACCESS_TOKEN_SECRET, + TWITTER_USERNAME, + } = process.env; + + if ( + !TWITTER_API_KEY || + !TWITTER_API_SECRET_KEY || + !TWITTER_ACCESS_TOKEN || + !TWITTER_ACCESS_TOKEN_SECRET || + !TWITTER_USERNAME + ) { + console.warn( + 'Skipping tests: Twitter API v2 keys are not available in environment variables.', + ); + shouldSkipV2Tests = true; + return; + } + + try { + // Get the user ID from username + const scraper = await getScraper(); + const profile = await scraper.getProfile(TWITTER_USERNAME); + + if (!profile.userId) { + throw new Error('User ID not found'); + } + + testUserId = profile.userId; + + // Get first conversation ID for testing + const conversations = await scraper.getDirectMessageConversations( + testUserId, + ); + + if ( + !conversations.conversations.length && + !conversations.conversations[0].conversationId + ) { + throw new Error('No conversations found'); + } + + // testConversationId = conversations.conversations[0].conversationId; + testConversationId = '1025530896651362304-1247854858931040258'; + } catch (error) { + console.error('Failed to initialize test data:', error); + shouldSkipV2Tests = true; + } +}); + +describe('Direct Message Tests', () => { + beforeEach(() => { + if (shouldSkipV2Tests || !testUserId || !testConversationId) { + console.warn('Skipping test: Required test data not available'); + return; + } + }); + + test('should get DM conversations', async () => { + if (shouldSkipV2Tests) return; + + const scraper = await getScraper(); + const conversations = await scraper.getDirectMessageConversations( + testUserId, + ); + + expect(conversations).toBeDefined(); + expect(conversations.conversations).toBeInstanceOf(Array); + expect(conversations.users).toBeInstanceOf(Array); + }, 30000); + + test('should handle DM send failure gracefully', async () => { + if (shouldSkipV2Tests) return; + + const scraper = await getScraper(); + const invalidConversationId = 'invalid-id'; + + await expect( + scraper.sendDirectMessage(invalidConversationId, 'test message'), + ).rejects.toThrow(); + }, 30000); + + test('should verify DM conversation structure', async () => { + if (shouldSkipV2Tests) return; + + const scraper = await getScraper(); + const conversations = await scraper.getDirectMessageConversations( + testUserId, + ); + + if (conversations.conversations.length > 0) { + const conversation = conversations.conversations[0]; + + // Test conversation structure + expect(conversation).toHaveProperty('conversationId'); + expect(conversation).toHaveProperty('messages'); + expect(conversation).toHaveProperty('participants'); + + // Test participants structure + expect(conversation.participants[0]).toHaveProperty('id'); + expect(conversation.participants[0]).toHaveProperty('screenName'); + + // Test message structure if messages exist + if (conversation.messages.length > 0) { + const message = conversation.messages[0]; + expect(message).toHaveProperty('id'); + expect(message).toHaveProperty('text'); + expect(message).toHaveProperty('senderId'); + expect(message).toHaveProperty('recipientId'); + expect(message).toHaveProperty('createdAt'); + } + } + }, 30000); +}); diff --git a/src/messages.ts b/src/messages.ts index 975a64f..6975320 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -48,41 +48,152 @@ export interface DirectMessageEvent { } export interface DirectMessagesResponse { - events: DirectMessageEvent[]; - apps?: Record; - next_cursor?: string; + conversations: DirectMessageConversation[]; + users: TwitterUser[]; + cursor?: string; + lastSeenEventId?: string; + trustedLastSeenEventId?: string; + untrustedLastSeenEventId?: string; + inboxTimelines?: { + trusted?: { + status: string; + minEntryId?: string; + }; + untrusted?: { + status: string; + minEntryId?: string; + }; + }; + userId: string; +} + +export interface TwitterUser { + id: string; + screenName: string; + name: string; + profileImageUrl: string; + description?: string; + verified?: boolean; + protected?: boolean; + followersCount?: number; + friendsCount?: number; +} + +export interface SendDirectMessageResponse { + entries: { + message: { + id: string; + time: string; + affects_sort: boolean; + conversation_id: string; + message_data: { + id: string; + time: string; + recipient_id: string; + sender_id: string; + text: string; + }; + }; + }[]; + users: Record; } function parseDirectMessageConversations( data: any, -): DirectMessageConversation[] { + userId: string, +): DirectMessagesResponse { try { - const conversations = data.data.inbox.conversations; - return conversations.map((conv: any) => ({ - conversationId: conv.conversation_id, - messages: parseDirectMessages(conv.messages), - participants: conv.participants.map((p: any) => ({ - id: p.user_id, - screenName: p.screen_name, - })), - })); + const inboxState = data?.inbox_initial_state; + const conversations = inboxState?.conversations || {}; + const entries = inboxState?.entries || []; + const users = inboxState?.users || {}; + + // Parse users first + const parsedUsers: TwitterUser[] = Object.values(users).map( + (user: any) => ({ + id: user.id_str, + screenName: user.screen_name, + name: user.name, + profileImageUrl: user.profile_image_url_https, + description: user.description, + verified: user.verified, + protected: user.protected, + followersCount: user.followers_count, + friendsCount: user.friends_count, + }), + ); + + // Group messages by conversation_id + const messagesByConversation: Record = {}; + entries.forEach((entry: any) => { + if (entry.message) { + const convId = entry.message.conversation_id; + if (!messagesByConversation[convId]) { + messagesByConversation[convId] = []; + } + messagesByConversation[convId].push(entry.message); + } + }); + + // Convert to DirectMessageConversation array + const parsedConversations = Object.entries(conversations).map( + ([convId, conv]: [string, any]) => { + const messages = messagesByConversation[convId] || []; + + // Sort messages by time in ascending order + messages.sort((a, b) => Number(a.time) - Number(b.time)); + + return { + conversationId: convId, + messages: parseDirectMessages(messages, users), + participants: conv.participants.map((p: any) => ({ + id: p.user_id, + screenName: users[p.user_id]?.screen_name || p.user_id, + })), + }; + }, + ); + + return { + conversations: parsedConversations, + users: parsedUsers, + cursor: inboxState?.cursor, + lastSeenEventId: inboxState?.last_seen_event_id, + trustedLastSeenEventId: inboxState?.trusted_last_seen_event_id, + untrustedLastSeenEventId: inboxState?.untrusted_last_seen_event_id, + inboxTimelines: { + trusted: inboxState?.inbox_timelines?.trusted && { + status: inboxState.inbox_timelines.trusted.status, + minEntryId: inboxState.inbox_timelines.trusted.min_entry_id, + }, + untrusted: inboxState?.inbox_timelines?.untrusted && { + status: inboxState.inbox_timelines.untrusted.status, + minEntryId: inboxState.inbox_timelines.untrusted.min_entry_id, + }, + }, + userId, + }; } catch (error) { console.error('Error parsing DM conversations:', error); - return []; + return { + conversations: [], + users: [], + userId, + }; } } -function parseDirectMessages(data: any): DirectMessage[] { +function parseDirectMessages(messages: any[], users: any): DirectMessage[] { try { - return data.map((msg: any) => ({ - id: msg.message_id, + return messages.map((msg: any) => ({ + id: msg.message_data.id, text: msg.message_data.text, senderId: msg.message_data.sender_id, recipientId: msg.message_data.recipient_id, - createdAt: msg.message_data.created_at, - mediaUrls: msg.message_data.attachment?.media_urls, - senderScreenName: msg.message_data.sender_screen_name, - recipientScreenName: msg.message_data.recipient_screen_name, + createdAt: msg.message_data.time, + mediaUrls: extractMediaUrls(msg.message_data), + senderScreenName: users[msg.message_data.sender_id]?.screen_name, + recipientScreenName: users[msg.message_data.recipient_id]?.screen_name, })); } catch (error) { console.error('Error parsing DMs:', error); @@ -90,29 +201,31 @@ function parseDirectMessages(data: any): DirectMessage[] { } } -function parseDirectMessageResponse(data: any): DirectMessage { - try { - const msg = data.data.message_create; - return { - id: msg.message_id, - text: msg.message_data.text, - senderId: msg.message_data.sender_id, - recipientId: msg.message_data.recipient_id, - createdAt: msg.message_data.created_at, - mediaUrls: msg.message_data.attachment?.media_urls, - senderScreenName: msg.message_data.sender_screen_name, - recipientScreenName: msg.message_data.recipient_screen_name, - }; - } catch (error) { - console.error('Error parsing DM response:', error); - throw error; +function extractMediaUrls(messageData: any): string[] | undefined { + const urls: string[] = []; + + // Extract URLs from entities if they exist + if (messageData.entities?.urls) { + messageData.entities.urls.forEach((url: any) => { + urls.push(url.expanded_url); + }); } + + // Extract media URLs if they exist + if (messageData.entities?.media) { + messageData.entities.media.forEach((media: any) => { + urls.push(media.media_url_https || media.media_url); + }); + } + + return urls.length > 0 ? urls : undefined; } export async function getDirectMessageConversations( + userId: string, auth: TwitterAuth, - cursor: string, -) { + cursor?: string, +): Promise { if (!auth.isLoggedIn()) { throw new Error('Authentication required to fetch direct messages'); } @@ -127,14 +240,15 @@ export async function getDirectMessageConversations( params.append('cursor', cursor); } - const finalUrl = `${url}${params.toString() ? '?' + params.toString() : ''}`; + const finalUrl = `${messageListUrl}${ + params.toString() ? '?' + params.toString() : '' + }`; const cookies = await auth.cookieJar().getCookies(url); const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); - const userTwitterId = cookies.find((cookie) => cookie.key === 'twid'); const headers = new Headers({ authorization: `Bearer ${(auth as any).bearerToken}`, - cookie: await auth.cookieJar().getCookieString(messageListUrl), + cookie: await auth.cookieJar().getCookieString(url), 'content-type': 'application/json', 'User-Agent': 'Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36', @@ -157,15 +271,14 @@ export async function getDirectMessageConversations( // parse the response const data = await response.json(); - return parseDirectMessageConversations(data); + return parseDirectMessageConversations(data, userId); } export async function sendDirectMessage( auth: TwitterAuth, - senderId: string, - recipientId: string, + conversation_id: string, text: string, -): Promise { +): Promise { if (!auth.isLoggedIn()) { throw new Error('Authentication required to send direct messages'); } @@ -190,7 +303,7 @@ export async function sendDirectMessage( }); const payload = { - conversation_id: `${senderId}-${recipientId}`, + conversation_id: `${conversation_id}`, recipient_ids: false, text: text, cards_platform: 'Web-12', @@ -211,5 +324,5 @@ export async function sendDirectMessage( throw new Error(await response.text()); } - return (await response.json()).event; + return await response.json(); } From 5a5688de7c807879e7c701d51607996700fd36e9 Mon Sep 17 00:00:00 2001 From: samarth30 Date: Tue, 17 Dec 2024 14:33:35 +0530 Subject: [PATCH 3/3] fix: added messages missing functions in scraper file --- src/scraper.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/src/scraper.ts b/src/scraper.ts index bc3f2cd..e10af30 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -25,7 +25,7 @@ import { fetchProfileFollowers, getFollowing, getFollowers, - followUser + followUser, } from './relationships'; import { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1'; import { getTrends } from './trends'; @@ -64,6 +64,12 @@ import { TTweetv2TweetField, TTweetv2UserField, } from 'twitter-api-v2'; +import { + DirectMessagesResponse, + getDirectMessageConversations, + sendDirectMessage, + SendDirectMessageResponse, +} from './messages'; const twUrl = 'https://twitter.com'; const UserTweetsUrl = @@ -448,7 +454,12 @@ export class Scraper { replyToTweetId?: string, mediaData?: { data: Buffer; mediaType: string }[], ) { - return await createCreateTweetRequest(text, this.auth, replyToTweetId, mediaData); + return await createCreateTweetRequest( + text, + this.auth, + replyToTweetId, + mediaData, + ); } /** @@ -463,7 +474,12 @@ export class Scraper { replyToTweetId?: string, mediaData?: { data: Buffer; mediaType: string }[], ) { - return await createCreateLongTweetRequest(text, this.auth, replyToTweetId, mediaData); + return await createCreateLongTweetRequest( + text, + this.auth, + replyToTweetId, + mediaData, + ); } /** @@ -780,7 +796,7 @@ export class Scraper { text: string, quotedTweetId: string, options?: { - mediaData: { data: Buffer; mediaType: string }[], + mediaData: { data: Buffer; mediaType: string }[]; }, ) { return await createQuoteTweetRequest( @@ -821,6 +837,32 @@ export class Scraper { await followUser(userName, this.auth); } + /** + * Fetches direct message conversations + * @param count Number of conversations to fetch (default: 50) + * @param cursor Pagination cursor for fetching more conversations + * @returns Array of DM conversations and other details + */ + public async getDirectMessageConversations( + userId: string, + cursor?: string, + ): Promise { + return await getDirectMessageConversations(userId, this.auth, cursor); + } + + /** + * Sends a direct message to a user. + * @param conversationId The ID of the conversation to send the message to. + * @param text The text of the message to send. + * @returns The response from the Twitter API. + */ + public async sendDirectMessage( + conversationId: string, + text: string, + ): Promise { + return await sendDirectMessage(this.auth, conversationId, text); + } + private getAuthOptions(): Partial { return { fetch: this.options?.fetch,