From 089e608e4b8d5348488e2b3c58b604018d6d1f66 Mon Sep 17 00:00:00 2001 From: Soham Jaiswal Date: Sun, 5 May 2024 10:39:33 +0530 Subject: [PATCH] fix: add scrape error handling --- .../controllers/server-banner.controller.ts | 3 + app/src/controllers/server-icon.controller.ts | 3 + app/src/controllers/user-avatar.controller.ts | 3 + app/src/libs/guilded-scrape.ts | 86 ++++++++++--------- 4 files changed, 54 insertions(+), 41 deletions(-) diff --git a/app/src/controllers/server-banner.controller.ts b/app/src/controllers/server-banner.controller.ts index 0c295e6..c092fc9 100644 --- a/app/src/controllers/server-banner.controller.ts +++ b/app/src/controllers/server-banner.controller.ts @@ -12,6 +12,9 @@ export const serverBannerController = new Elysia() return new Response(await streamToBuffer(banner), { headers: { 'Content-Type': 'image/webp' } }) } const imageBlob = await guildedServerProfileScrape(params.id, 'banner') + if (imageBlob instanceof Error) { + return new Response('User not found', { status: 404 }) + } return new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } }) }, { params: t.Object({ diff --git a/app/src/controllers/server-icon.controller.ts b/app/src/controllers/server-icon.controller.ts index cd95d9b..2169252 100644 --- a/app/src/controllers/server-icon.controller.ts +++ b/app/src/controllers/server-icon.controller.ts @@ -13,6 +13,9 @@ export const serverIconController = new Elysia() return res } const imageBlob = await guildedServerProfileScrape(params.id, 'icon') + if (imageBlob instanceof Error) { + return new Response('User not found', { status: 404 }) + } const res = new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } }) return res }, { diff --git a/app/src/controllers/user-avatar.controller.ts b/app/src/controllers/user-avatar.controller.ts index 3a749bd..b5b105a 100644 --- a/app/src/controllers/user-avatar.controller.ts +++ b/app/src/controllers/user-avatar.controller.ts @@ -13,6 +13,9 @@ export const userAvatarController = new Elysia() return res } const imageBlob = await guildedUserProfileScrape(params.id, 'avatar') + if (imageBlob instanceof Error) { + return new Response('User not found', { status: 404 }) + } const res = new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } }) return res }, { diff --git a/app/src/libs/guilded-scrape.ts b/app/src/libs/guilded-scrape.ts index b9bfcc8..4e6caf6 100644 --- a/app/src/libs/guilded-scrape.ts +++ b/app/src/libs/guilded-scrape.ts @@ -3,54 +3,58 @@ import puppeteer, { Browser } from "puppeteer"; export let browserInstances: Browser[] = [] -export const guildedUserProfileScrape = async (id: string, getElement: 'avatar' | 'banner') => { - const getClass = getElement === 'avatar' ? '.UserProfilePictureControl-picture' : '.UserProfileBackground-image' - const browser = await puppeteer.launch({ - headless: true, - args: ['--no-sandbox', '--disable-setuid-sandbox'], - executablePath: "/usr/bin/chromium", - }) - browserInstances.push(browser) - const page = await browser.newPage() +export const guildedUserProfileScrape: (id: string, getElement: 'avatar' | 'banner') => Promise = async (id: string, getElement: 'avatar' | 'banner') => { try { - await page.goto(`https://www.guilded.gg/profile/${id}`, {waitUntil: 'networkidle0', timeout: 13000}) + const getClass = getElement === 'avatar' ? '.UserProfilePictureControl-picture' : '.UserProfileBackground-image' + const browser = await puppeteer.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'], + executablePath: "/usr/bin/chromium", + }) + browserInstances.push(browser) + const page = await browser.newPage() + try { + await page.goto(`https://www.guilded.gg/profile/${id}`, {waitUntil: 'networkidle0', timeout: 13000}) + } catch (e) { + throw new Error(`User not found,\nError:\n${e}`) + } + const src = await page.$eval(getClass, (el: any) => el.src) + if (getElement === 'avatar') { + await userAvatarBucket.uploadImage(id, src) + } else if (getElement === 'banner') { + await userBannerBucket.uploadImage(id, src) + } + await page.close() + await browser.close() + browserInstances.pop() + return await (await fetch(src)).blob() } catch (e) { - throw new Error(`User not found,\nError:\n${e}`) + return new Error('User not found') } - const src = await page.$eval(getClass, (el: any) => el.src) - if (getElement === 'avatar') { - await userAvatarBucket.uploadImage(id, src) - } else if (getElement === 'banner') { - await userBannerBucket.uploadImage(id, src) - } - await page.close() - await browser.close() - browserInstances.pop() - return await (await fetch(src)).blob() } -export const guildedServerProfileScrape = async (id: string, getElement: 'icon' | 'banner') => { - const getClass = getElement === 'icon' ? '.TeamPlaqueV2-profile-pic' : '.TeamOverviewBanner-banner.TeamPageBanner-overview-banner' - const browser = await puppeteer.launch({ - headless: true, - args: ['--no-sandbox', '--disable-setuid-sandbox'], - executablePath: "/usr/bin/chromium", - }) - browserInstances.push(browser) - const page = await browser.newPage() +export const guildedServerProfileScrape: (id: string, getElement: 'icon' | 'banner') => Promise = async (id, getElement) => { try { + const getClass = getElement === 'icon' ? '.TeamPlaqueV2-profile-pic' : '.TeamOverviewBanner-banner.TeamPageBanner-overview-banner' + const browser = await puppeteer.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'], + executablePath: "/usr/bin/chromium", + }) + browserInstances.push(browser) + const page = await browser.newPage() await page.goto(`https://www.guilded.gg/teams/${id}/overview`, {waitUntil: 'networkidle0', timeout: 13000}) + const src = await page.$eval(getClass, (el: any) => el.src) + if (getElement === 'icon') { + await serverIconBucket.uploadImage(id, src) + } else if (getElement === 'banner') { + await serverBannerBucket.uploadImage(id, src) + } + await page.close() + await browser.close() + browserInstances.pop() + return await (await fetch(src)).blob() } catch (e) { - throw new Error(`Server not found,\nError:\n${e}`) - } - const src = await page.$eval(getClass, (el: any) => el.src) - if (getElement === 'icon') { - await serverIconBucket.uploadImage(id, src) - } else if (getElement === 'banner') { - await serverBannerBucket.uploadImage(id, src) + return new Error('Server not found') } - await page.close() - await browser.close() - browserInstances.pop() - return await (await fetch(src)).blob() } \ No newline at end of file