From f99d9be67776ae3adb6fa1c274f166984a3e7339 Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 19:12:55 +0200 Subject: [PATCH 1/8] :sparkles: Implement better webtoon thumbnail --- .../webtoon/models/enums/webtoon-genres.ts | 18 ++++++++- .../webtoon/webtoon-downloader.service.ts | 15 ++++++-- .../webtoon/webtoon/webtoon-parser.service.ts | 37 ++++++++++++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts b/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts index 17b46ee..dc49167 100644 --- a/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts +++ b/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts @@ -11,7 +11,23 @@ enum WebtoonGenres { SF = "sf", HORROR = "horror", TIPTOON = "tiptoon", - LOCAL = "local" + LOCAL = "local", + SCHOOL = "school", + MARTIAL_ARTS = "martial_arts", + BL_GL = "bl_gl", + ROMANCE_M = "romance_m", + TIME_SLIP = "time_slip", + CITY_OFFICE = "city_office", + MYSTERY = "mystery", + HEARTWARMING = "heartwarming", + SHONEN = "shonen", + EASTERN_PALACE = "eastern_palace", + WEB_NOVEL = "web_novel", + WESTERN_PALACE = "western_palace", + ADAPTATION = "adaptation", + SUPERNATURAL = "supernatural", + HISTORICAL = "historical", + ROMANTIC_FANTASY = "romantic_fantasy", } export default WebtoonGenres; diff --git a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts index f40769d..3ff3a6c 100644 --- a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts @@ -4,6 +4,7 @@ import WebtoonDataModel from "./models/models/webtoon-data.model"; import WebtoonModel from "./models/models/webtoon.model"; import {Injectable, Logger} from "@nestjs/common"; import {MiscService} from "../../misc/misc.service"; +import * as sharp from "sharp"; @Injectable() export class WebtoonDownloaderService{ @@ -14,7 +15,7 @@ export class WebtoonDownloaderService{ private readonly miscService: MiscService, ){} - async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise { + async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise{ this.logger.debug(`Downloading episode ${episode.number}...`); const startTime = Date.now(); const thumbnail: Buffer = await this.miscService.downloadImage(episode.thumbnail); @@ -27,7 +28,7 @@ export class WebtoonDownloaderService{ this.logger.debug(`Downloading ${downloadedCount} of ${imageUrls.length} images (${(imagesPerSecond).toFixed(2)} images/s)...`); }, 1000); - for (let i = 0; i < imageUrls.length; i++) { + for (let i = 0; i < imageUrls.length; i++){ const url = imageUrls[i]; const image = await this.miscService.downloadImage(url, episode.link); conversionPromises.push(this.miscService.convertImageToWebp(image)); @@ -49,7 +50,7 @@ export class WebtoonDownloaderService{ async downloadWebtoon(webtoon: WebtoonModel): Promise{ const downloadPromises: Promise[] = []; - downloadPromises.push(this.miscService.downloadImage(webtoon.thumbnail)); + downloadPromises.push(this.convertThumbnail(webtoon.thumbnail)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.background)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.top)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.mobile)); @@ -65,4 +66,12 @@ export class WebtoonDownloaderService{ mobileBanner: mobile, } as WebtoonDataModel; } + + async convertThumbnail(url: string){ + const webpImage = await this.miscService.downloadImage(url); + return await sharp(webpImage).resize(240, 240, { + fit: "cover", + position: "center" + }).toBuffer(); + } } diff --git a/src/modules/webtoon/webtoon/webtoon-parser.service.ts b/src/modules/webtoon/webtoon/webtoon-parser.service.ts index d8e7385..588f84c 100644 --- a/src/modules/webtoon/webtoon/webtoon-parser.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-parser.service.ts @@ -45,16 +45,24 @@ export class WebtoonParserService{ private async getWebtoonsFromLanguage(language: string): Promise{ const languageWebtoons: CachedWebtoonModel[] = []; + const promises: Promise[] = []; for (const genre of Object.values(WebtoonGenres)) promises.push(this.getWebtoonsFromGenre(language, genre)); const genreResults = await Promise.all(promises); for (const webtoons of genreResults) languageWebtoons.push(...webtoons); + + // for (const genre of Object.values(WebtoonGenres)) + // languageWebtoons.push(...await this.getWebtoonsFromGenre(language, genre)); + return this.removeDuplicateWebtoons(languageWebtoons); } private async getWebtoonsFromGenre(language: string, genre: string): Promise{ + // console.log(language, genre); + const mobileThumbnails = await this.getWebtoonThumbnailFromGenre(language, genre); + // console.log(mobileThumbnails); const url = `https://www.webtoons.com/${language}/genres/${genre}`; const response = await this.miscService.getAxiosInstance().get(url); const document = new JSDOM(response.data).window.document; @@ -69,7 +77,11 @@ export class WebtoonParserService{ const stars = a.querySelector("p.grade_area")?.querySelector("em")?.textContent; const link = a.href; const id = link.split("?title_no=")[1]; - const thumbnail = a.querySelector("img")?.src; + // const thumbnail = a.querySelector("img")?.src; + // console.log("title", title); + let thumbnail = mobileThumbnails.find(t => t.name === title)?.thumbnail; + if(!thumbnail) + thumbnail = a.querySelector("img")?.src; if(!title || !author || !stars || !link || !thumbnail || !id) throw new NotFoundException(`Missing data for webtoon: ${url}`); const webtoon: CachedWebtoonModel = { @@ -87,6 +99,29 @@ export class WebtoonParserService{ return webtoons; } + private async getWebtoonThumbnailFromGenre(language: string, genre: string): Promise[]>{ + const mobileThumbnails: Record[] = []; + const mobileUrl = `https://www.webtoons.com/${language}/genres/${genre}`.replace("www.webtoons", "m.webtoons") + "?webtoon-platform-redirect=true"; + // console.log(mobileUrl); + const mobileResponse = await this.miscService.getAxiosInstance().get(mobileUrl, { + headers: { + "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148" + } + }); + const mobileDocument = new JSDOM((mobileResponse).data).window.document; + const className = `genre_${genre.toUpperCase()}_list`; + const wList = mobileDocument.querySelector(`ul.${className}`)?.querySelectorAll("li"); + // console.log("wList", wList); + if(!wList) return []; + for(const li of wList){ + const webtoonName = li.querySelector("a")?.querySelector("div.info")?.querySelector("p.subj span")?.textContent; + const imgLink = li.querySelector("a")?.querySelector("div.pic")?.querySelector("img")?.src; + if(!webtoonName || !imgLink) continue; + mobileThumbnails.push({name: webtoonName, thumbnail: imgLink}); + } + return mobileThumbnails; + } + private removeDuplicateWebtoons(webtoons: CachedWebtoonModel[]){ const webtoonsWithoutDuplicates: CachedWebtoonModel[] = []; for(const webtoon of webtoons){ From 9b7587456e4efb12144172eb20b2970594bd1cb6 Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 21:27:38 +0200 Subject: [PATCH 2/8] :sparkles: Add route to refresh cache --- src/modules/webtoon/admin/admin.controller.ts | 7 +++++++ .../webtoon/download-manager.service.ts | 18 +++++++++++++++--- .../webtoon/webtoon/webtoon-parser.service.ts | 6 ++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/modules/webtoon/admin/admin.controller.ts b/src/modules/webtoon/admin/admin.controller.ts index 46476d8..c942d9c 100644 --- a/src/modules/webtoon/admin/admin.controller.ts +++ b/src/modules/webtoon/admin/admin.controller.ts @@ -63,4 +63,11 @@ export class AdminController{ async clearQueue(): Promise{ return this.downloadManagerService.clearDownloadQueue(); } + + @Post("refresh-cache") + @ApiBearerAuth() + @ApiResponse({status: HttpStatusCode.Created, description: "Refreshes the cache"}) + async refreshCache(): Promise{ + return this.downloadManagerService.refreshCache(); + } } diff --git a/src/modules/webtoon/webtoon/download-manager.service.ts b/src/modules/webtoon/webtoon/download-manager.service.ts index 6ab73ed..0aea6b5 100644 --- a/src/modules/webtoon/webtoon/download-manager.service.ts +++ b/src/modules/webtoon/webtoon/download-manager.service.ts @@ -17,7 +17,7 @@ export class DownloadManagerService{ private readonly logger = new Logger(DownloadManagerService.name); private cacheLoaded: boolean = false; - private readonly cachePromise: Promise; + private cachePromise: Promise; private readonly downloadQueue: DownloadQueue; constructor( @@ -34,7 +34,7 @@ export class DownloadManagerService{ } this.cachePromise = this.webtoonParser.loadCache(); this.cachePromise.then(() => { - this.cacheLoaded = true + this.cacheLoaded = true; if(downloadInProgress) this.startDownload().then(() => console.log("Download finished.")); }); @@ -44,9 +44,20 @@ export class DownloadManagerService{ return this.cachePromise; } + async refreshCache(): Promise{ + if(!this.cacheLoaded) + throw new HttpException("Cache already loading.", HttpStatusCode.TooEarly); + this.cacheLoaded = false; + this.webtoonParser.clearCache(); + this.cachePromise = this.webtoonParser.loadCache(); + this.cachePromise.then(() => { + this.cacheLoaded = true; + }); + } + async addWebtoonToQueue(webtoonName: string, language = "en"): Promise{ if(!this.cacheLoaded) - throw new Error("Cache not loaded."); + throw new HttpException("Cache already loading.", HttpStatusCode.TooEarly); const webtoonOverview: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoonName, language); // If queue is empty, start download this.downloadQueue.enqueue(webtoonOverview); @@ -69,6 +80,7 @@ export class DownloadManagerService{ if(!this.cacheLoaded) throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly); while(!this.downloadQueue.isQueueEmpty()){ + await this.cachePromise; // Wait for cache to be loaded if it is cleared const currentDownload: CachedWebtoonModel = this.downloadQueue.dequeue(); if(!currentDownload) return; diff --git a/src/modules/webtoon/webtoon/webtoon-parser.service.ts b/src/modules/webtoon/webtoon/webtoon-parser.service.ts index 588f84c..933db7c 100644 --- a/src/modules/webtoon/webtoon/webtoon-parser.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-parser.service.ts @@ -21,6 +21,12 @@ export class WebtoonParserService{ private readonly miscService: MiscService ){} + clearCache(): void{ + if(fs.existsSync("./.cache/webtoons.json")) + fs.unlinkSync("./.cache/webtoons.json"); + this.webtoons = {}; + } + async loadCache(): Promise{ // Load existing cache if(fs.existsSync("./.cache/webtoons.json")){ From a94af821c77b461a93b1bab192fded6635a874b3 Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 21:28:09 +0200 Subject: [PATCH 3/8] :rotating_light: Apply linter style --- src/modules/misc/misc.service.ts | 2 +- src/modules/webtoon/webtoon/download-manager.service.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/modules/misc/misc.service.ts b/src/modules/misc/misc.service.ts index dca7215..aa10094 100644 --- a/src/modules/misc/misc.service.ts +++ b/src/modules/misc/misc.service.ts @@ -22,7 +22,7 @@ export class MiscService{ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0", - ] + ]; constructor(){ this.axiosInstance = axios.create({}); diff --git a/src/modules/webtoon/webtoon/download-manager.service.ts b/src/modules/webtoon/webtoon/download-manager.service.ts index 0aea6b5..e4a4679 100644 --- a/src/modules/webtoon/webtoon/download-manager.service.ts +++ b/src/modules/webtoon/webtoon/download-manager.service.ts @@ -7,7 +7,6 @@ import EpisodeDataModel from "./models/models/episode-data.model"; import {HttpException, Injectable, Logger, NotFoundException} from "@nestjs/common"; import WebtoonModel from "./models/models/webtoon.model"; import WebtoonDataModel from "./models/models/webtoon-data.model"; -import WebtoonQueue from "../../../common/utils/models/webtoon-queue"; import {HttpStatusCode} from "axios"; import DownloadQueue from "../../../common/utils/models/download-queue"; From 181f39a48972c3bb6d5f28f54a58a3fd164d7109 Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 22:22:47 +0200 Subject: [PATCH 4/8] :sparkles: Add webtoon thumbnail update route from old to new images --- src/app.module.ts | 2 + src/modules/misc/misc.service.ts | 8 +++ .../webtoon/update/update.controller.ts | 20 ++++++ src/modules/webtoon/update/update.module.ts | 13 ++++ src/modules/webtoon/update/update.service.ts | 69 +++++++++++++++++++ .../webtoon/webtoon-downloader.service.ts | 10 +-- src/modules/webtoon/webtoon/webtoon.module.ts | 2 +- 7 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 src/modules/webtoon/update/update.controller.ts create mode 100644 src/modules/webtoon/update/update.module.ts create mode 100644 src/modules/webtoon/update/update.service.ts diff --git a/src/app.module.ts b/src/app.module.ts index 36947df..51cf8ba 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -7,6 +7,7 @@ import {ThrottlerGuard, ThrottlerModule} from "@nestjs/throttler"; import {APP_GUARD} from "@nestjs/core"; import {MigrationModule} from "./modules/webtoon/migration/migration.module"; import {ScheduleModule} from "@nestjs/schedule"; +import {UpdateModule} from "./modules/webtoon/update/update.module"; @Module({ imports: [ @@ -20,6 +21,7 @@ import {ScheduleModule} from "@nestjs/schedule"; WebtoonModule, AdminModule, MigrationModule, + UpdateModule, ], controllers: [], providers: [ diff --git a/src/modules/misc/misc.service.ts b/src/modules/misc/misc.service.ts index aa10094..d276134 100644 --- a/src/modules/misc/misc.service.ts +++ b/src/modules/misc/misc.service.ts @@ -97,6 +97,14 @@ export class MiscService{ return response.data as Buffer; } + async convertThumbnail(url: string){ + const webpImage: Buffer = await this.downloadImage(url); + return await sharp(webpImage).resize(240, 240, { + fit: "cover", + position: "center" + }).toBuffer(); + } + parseWebtoonStars(stars: string): WebtoonStarModel{ // TODO: Fix dutch space and comma issue stars = stars.replace(" ", " "); diff --git a/src/modules/webtoon/update/update.controller.ts b/src/modules/webtoon/update/update.controller.ts new file mode 100644 index 0000000..78dfcac --- /dev/null +++ b/src/modules/webtoon/update/update.controller.ts @@ -0,0 +1,20 @@ +import {Controller, Post, UseGuards} from "@nestjs/common"; +import {UpdateService} from "./update.service"; +import {ApiBearerAuth, ApiTags} from "@nestjs/swagger"; +import {AdminGuard} from "../admin/guard/admin.guard"; + + +@Controller("update") +@ApiTags("Update") +@UseGuards(AdminGuard) +export class UpdateController{ + constructor( + private readonly updateService: UpdateService, + ){} + + @Post("webtoons/thumbnails") + @ApiBearerAuth() + async updateThumbnails(): Promise{ + await this.updateService.updateThumbnails(); + } +} diff --git a/src/modules/webtoon/update/update.module.ts b/src/modules/webtoon/update/update.module.ts new file mode 100644 index 0000000..14b8b99 --- /dev/null +++ b/src/modules/webtoon/update/update.module.ts @@ -0,0 +1,13 @@ +import {Module} from "@nestjs/common"; +import {UpdateService} from "./update.service"; +import {UpdateController} from "./update.controller"; +import {MiscModule} from "../../misc/misc.module"; +import {WebtoonModule} from "../webtoon/webtoon.module"; + + +@Module({ + providers: [UpdateService], + controllers: [UpdateController], + imports: [MiscModule, WebtoonModule] +}) +export class UpdateModule{} diff --git a/src/modules/webtoon/update/update.service.ts b/src/modules/webtoon/update/update.service.ts new file mode 100644 index 0000000..0847e53 --- /dev/null +++ b/src/modules/webtoon/update/update.service.ts @@ -0,0 +1,69 @@ +import {Injectable} from "@nestjs/common"; +import {PrismaService} from "../../misc/prisma.service"; +import {WebtoonParserService} from "../webtoon/webtoon-parser.service"; +import CachedWebtoonModel from "../webtoon/models/models/cached-webtoon.model"; +import {MiscService} from "../../misc/misc.service"; +import {WebtoonDatabaseService} from "../webtoon/webtoon-database.service"; +import ImageTypes from "../webtoon/models/enums/image-types"; + + +@Injectable() +export class UpdateService{ + + constructor( + private readonly prismaService: PrismaService, + private readonly webtoonParser: WebtoonParserService, + private readonly miscService: MiscService, + private readonly webtoonDatabaseService: WebtoonDatabaseService, + ){} + + async updateThumbnails(): Promise{ + const dbWebtoons: any[] = await this.prismaService.webtoons.findMany({ + select: { + id: true, + title: true, + language: true, + thumbnail_id: true + } + }); + const thumbnailsToDelete: number[] = dbWebtoons.map(webtoon => webtoon.thumbnail_id); + // Save new thumbnails + const dbThumbnailType = await this.prismaService.imageTypes.findFirst({ + where: { + name: ImageTypes.WEBTOON_THUMBNAIL + }, + select: { + id: true + } + }); + for(const webtoon of dbWebtoons){ + const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language); + const thumbnail: Buffer = await this.miscService.convertThumbnail(cachedWebtoon.thumbnail); + const sum: string = this.webtoonDatabaseService.saveImage(thumbnail); + const dbThumbnail = await this.prismaService.images.create({ + data: { + sum: sum, + type_id: dbThumbnailType.id + } + }); + await this.prismaService.webtoons.update({ + where: { + id: webtoon.id + }, + data: { + thumbnail_id: dbThumbnail.id + } + }); + } + + // Delete old thumbnails + await this.prismaService.images.deleteMany({ + where: { + id: { + in: thumbnailsToDelete + } + } + }); + } + +} diff --git a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts index 3ff3a6c..445b164 100644 --- a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts @@ -50,7 +50,7 @@ export class WebtoonDownloaderService{ async downloadWebtoon(webtoon: WebtoonModel): Promise{ const downloadPromises: Promise[] = []; - downloadPromises.push(this.convertThumbnail(webtoon.thumbnail)); + downloadPromises.push(this.miscService.convertThumbnail(webtoon.thumbnail)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.background)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.top)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.mobile)); @@ -66,12 +66,4 @@ export class WebtoonDownloaderService{ mobileBanner: mobile, } as WebtoonDataModel; } - - async convertThumbnail(url: string){ - const webpImage = await this.miscService.downloadImage(url); - return await sharp(webpImage).resize(240, 240, { - fit: "cover", - position: "center" - }).toBuffer(); - } } diff --git a/src/modules/webtoon/webtoon/webtoon.module.ts b/src/modules/webtoon/webtoon/webtoon.module.ts index 10b19ee..e91206d 100644 --- a/src/modules/webtoon/webtoon/webtoon.module.ts +++ b/src/modules/webtoon/webtoon/webtoon.module.ts @@ -10,6 +10,6 @@ import {MiscModule} from "../../misc/misc.module"; imports: [MiscModule], controllers: [WebtoonController], providers: [WebtoonParserService, WebtoonDownloaderService, WebtoonDatabaseService, DownloadManagerService], - exports: [DownloadManagerService, WebtoonDatabaseService], + exports: [DownloadManagerService, WebtoonDatabaseService, WebtoonParserService], }) export class WebtoonModule{} From acc6400766cd5e61aa5f23c046242dd121f0e423 Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 22:33:36 +0200 Subject: [PATCH 5/8] :sparkles: Add image removing when updating webtoon thumbnails --- src/modules/webtoon/update/update.service.ts | 15 +++++++++------ .../webtoon/webtoon/webtoon-database.service.ts | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/modules/webtoon/update/update.service.ts b/src/modules/webtoon/update/update.service.ts index 0847e53..78dc48e 100644 --- a/src/modules/webtoon/update/update.service.ts +++ b/src/modules/webtoon/update/update.service.ts @@ -18,6 +18,8 @@ export class UpdateService{ ){} async updateThumbnails(): Promise{ + this.webtoonParser.clearCache(); + await this.webtoonParser.loadCache(); const dbWebtoons: any[] = await this.prismaService.webtoons.findMany({ select: { id: true, @@ -57,13 +59,14 @@ export class UpdateService{ } // Delete old thumbnails - await this.prismaService.images.deleteMany({ - where: { - id: { - in: thumbnailsToDelete + for(const thumbnailId of thumbnailsToDelete){ + const deletedImage = await this.prismaService.images.delete({ + where: { + id: thumbnailId } - } - }); + }); + this.webtoonDatabaseService.removeImage(deletedImage.sum); + } } } diff --git a/src/modules/webtoon/webtoon/webtoon-database.service.ts b/src/modules/webtoon/webtoon/webtoon-database.service.ts index c83d580..d2be2b4 100644 --- a/src/modules/webtoon/webtoon/webtoon-database.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-database.service.ts @@ -406,4 +406,9 @@ export class WebtoonDatabaseService{ const folder = imageSum.substring(0, 2); return fs.readFileSync(`./images/${folder}/${imageSum}.webp`); } + + removeImage(imageSum: string): void{ + const folder = imageSum.substring(0, 2); + fs.rmSync(`./images/${folder}/${imageSum}.webp`); + } } From f7dd88171bf125e84c0bc793f5ab331b6f9a3d0e Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 22:52:23 +0200 Subject: [PATCH 6/8] :art: Improve logging and fix potential bugs --- .../webtoon/update/update.controller.ts | 7 +- src/modules/webtoon/update/update.service.ts | 100 ++++++++++-------- 2 files changed, 62 insertions(+), 45 deletions(-) diff --git a/src/modules/webtoon/update/update.controller.ts b/src/modules/webtoon/update/update.controller.ts index 78dfcac..87439dc 100644 --- a/src/modules/webtoon/update/update.controller.ts +++ b/src/modules/webtoon/update/update.controller.ts @@ -1,4 +1,4 @@ -import {Controller, Post, UseGuards} from "@nestjs/common"; +import {Controller, Logger, Post, UseGuards} from "@nestjs/common"; import {UpdateService} from "./update.service"; import {ApiBearerAuth, ApiTags} from "@nestjs/swagger"; import {AdminGuard} from "../admin/guard/admin.guard"; @@ -8,6 +8,9 @@ import {AdminGuard} from "../admin/guard/admin.guard"; @ApiTags("Update") @UseGuards(AdminGuard) export class UpdateController{ + + private readonly logger = new Logger(UpdateController.name); + constructor( private readonly updateService: UpdateService, ){} @@ -15,6 +18,6 @@ export class UpdateController{ @Post("webtoons/thumbnails") @ApiBearerAuth() async updateThumbnails(): Promise{ - await this.updateService.updateThumbnails(); + this.updateService.updateThumbnails().then(() => this.logger.log("Thumbnails updated")); } } diff --git a/src/modules/webtoon/update/update.service.ts b/src/modules/webtoon/update/update.service.ts index 78dc48e..f2c6cce 100644 --- a/src/modules/webtoon/update/update.service.ts +++ b/src/modules/webtoon/update/update.service.ts @@ -1,4 +1,4 @@ -import {Injectable} from "@nestjs/common"; +import {Injectable, Logger} from "@nestjs/common"; import {PrismaService} from "../../misc/prisma.service"; import {WebtoonParserService} from "../webtoon/webtoon-parser.service"; import CachedWebtoonModel from "../webtoon/models/models/cached-webtoon.model"; @@ -10,6 +10,8 @@ import ImageTypes from "../webtoon/models/enums/image-types"; @Injectable() export class UpdateService{ + private readonly logger = new Logger(UpdateService.name); + constructor( private readonly prismaService: PrismaService, private readonly webtoonParser: WebtoonParserService, @@ -20,53 +22,65 @@ export class UpdateService{ async updateThumbnails(): Promise{ this.webtoonParser.clearCache(); await this.webtoonParser.loadCache(); - const dbWebtoons: any[] = await this.prismaService.webtoons.findMany({ - select: { - id: true, - title: true, - language: true, - thumbnail_id: true - } - }); - const thumbnailsToDelete: number[] = dbWebtoons.map(webtoon => webtoon.thumbnail_id); - // Save new thumbnails - const dbThumbnailType = await this.prismaService.imageTypes.findFirst({ - where: { - name: ImageTypes.WEBTOON_THUMBNAIL - }, - select: { - id: true - } - }); - for(const webtoon of dbWebtoons){ - const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language); - const thumbnail: Buffer = await this.miscService.convertThumbnail(cachedWebtoon.thumbnail); - const sum: string = this.webtoonDatabaseService.saveImage(thumbnail); - const dbThumbnail = await this.prismaService.images.create({ - data: { - sum: sum, - type_id: dbThumbnailType.id - } - }); - await this.prismaService.webtoons.update({ - where: { - id: webtoon.id - }, - data: { - thumbnail_id: dbThumbnail.id + await this.prismaService.$transaction(async(tx) => { + const dbWebtoons: any[] = await tx.webtoons.findMany({ + select: { + id: true, + title: true, + language: true, + thumbnail_id: true } }); - } + const thumbnailsToDelete: number[] = dbWebtoons.map(webtoon => webtoon.thumbnail_id); - // Delete old thumbnails - for(const thumbnailId of thumbnailsToDelete){ - const deletedImage = await this.prismaService.images.delete({ + // Save new thumbnails + const dbThumbnailType = await tx.imageTypes.findFirst({ where: { - id: thumbnailId + name: ImageTypes.WEBTOON_THUMBNAIL + }, + select: { + id: true } }); - this.webtoonDatabaseService.removeImage(deletedImage.sum); - } + for(const webtoon of dbWebtoons){ + this.logger.debug(`Updating thumbnail for webtoon ${webtoon.title} (${webtoon.language})`); + const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language); + const thumbnail: Buffer = await this.miscService.convertThumbnail(cachedWebtoon.thumbnail); + const sum: string = this.webtoonDatabaseService.saveImage(thumbnail); + // Check if thumbnail already exists + let dbThumbnail = await tx.images.findFirst({ + where: { + sum: sum + } + }); + if(dbThumbnail) + thumbnailsToDelete.splice(thumbnailsToDelete.indexOf(dbThumbnail.id), 1); + else + dbThumbnail = await tx.images.create({ + data: { + sum: sum, + type_id: dbThumbnailType.id + } + }); + // Update webtoon thumbnail + await tx.webtoons.update({ + where: { + id: webtoon.id + }, + data: { + thumbnail_id: dbThumbnail.id + } + }); + } + // Delete old thumbnails + for(const thumbnailId of thumbnailsToDelete){ + const deletedImage = await tx.images.delete({ + where: { + id: thumbnailId + } + }); + this.webtoonDatabaseService.removeImage(deletedImage.sum); + } + }); } - } From 8d103565b5cc5dd2b9862734421d9c09dccf3af2 Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 22:56:30 +0200 Subject: [PATCH 7/8] :art: Add minor code improvements --- src/modules/misc/misc.service.ts | 2 +- src/modules/webtoon/update/update.controller.ts | 4 +++- src/modules/webtoon/update/update.service.ts | 2 +- .../webtoon/webtoon/webtoon-downloader.service.ts | 2 +- src/modules/webtoon/webtoon/webtoon-parser.service.ts | 9 --------- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/modules/misc/misc.service.ts b/src/modules/misc/misc.service.ts index d276134..c3f439a 100644 --- a/src/modules/misc/misc.service.ts +++ b/src/modules/misc/misc.service.ts @@ -97,7 +97,7 @@ export class MiscService{ return response.data as Buffer; } - async convertThumbnail(url: string){ + async convertWebtoonThumbnail(url: string){ const webpImage: Buffer = await this.downloadImage(url); return await sharp(webpImage).resize(240, 240, { fit: "cover", diff --git a/src/modules/webtoon/update/update.controller.ts b/src/modules/webtoon/update/update.controller.ts index 87439dc..e59fe3d 100644 --- a/src/modules/webtoon/update/update.controller.ts +++ b/src/modules/webtoon/update/update.controller.ts @@ -1,7 +1,8 @@ import {Controller, Logger, Post, UseGuards} from "@nestjs/common"; import {UpdateService} from "./update.service"; -import {ApiBearerAuth, ApiTags} from "@nestjs/swagger"; +import {ApiBearerAuth, ApiResponse, ApiTags} from "@nestjs/swagger"; import {AdminGuard} from "../admin/guard/admin.guard"; +import {HttpStatusCode} from "axios"; @Controller("update") @@ -17,6 +18,7 @@ export class UpdateController{ @Post("webtoons/thumbnails") @ApiBearerAuth() + @ApiResponse({status: HttpStatusCode.Created, description: "Thumbnails updated"}) async updateThumbnails(): Promise{ this.updateService.updateThumbnails().then(() => this.logger.log("Thumbnails updated")); } diff --git a/src/modules/webtoon/update/update.service.ts b/src/modules/webtoon/update/update.service.ts index f2c6cce..49df9da 100644 --- a/src/modules/webtoon/update/update.service.ts +++ b/src/modules/webtoon/update/update.service.ts @@ -45,7 +45,7 @@ export class UpdateService{ for(const webtoon of dbWebtoons){ this.logger.debug(`Updating thumbnail for webtoon ${webtoon.title} (${webtoon.language})`); const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language); - const thumbnail: Buffer = await this.miscService.convertThumbnail(cachedWebtoon.thumbnail); + const thumbnail: Buffer = await this.miscService.convertWebtoonThumbnail(cachedWebtoon.thumbnail); const sum: string = this.webtoonDatabaseService.saveImage(thumbnail); // Check if thumbnail already exists let dbThumbnail = await tx.images.findFirst({ diff --git a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts index 445b164..0d8c0aa 100644 --- a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts @@ -50,7 +50,7 @@ export class WebtoonDownloaderService{ async downloadWebtoon(webtoon: WebtoonModel): Promise{ const downloadPromises: Promise[] = []; - downloadPromises.push(this.miscService.convertThumbnail(webtoon.thumbnail)); + downloadPromises.push(this.miscService.convertWebtoonThumbnail(webtoon.thumbnail)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.background)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.top)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.mobile)); diff --git a/src/modules/webtoon/webtoon/webtoon-parser.service.ts b/src/modules/webtoon/webtoon/webtoon-parser.service.ts index 933db7c..88713cb 100644 --- a/src/modules/webtoon/webtoon/webtoon-parser.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-parser.service.ts @@ -51,24 +51,17 @@ export class WebtoonParserService{ private async getWebtoonsFromLanguage(language: string): Promise{ const languageWebtoons: CachedWebtoonModel[] = []; - const promises: Promise[] = []; for (const genre of Object.values(WebtoonGenres)) promises.push(this.getWebtoonsFromGenre(language, genre)); const genreResults = await Promise.all(promises); for (const webtoons of genreResults) languageWebtoons.push(...webtoons); - - // for (const genre of Object.values(WebtoonGenres)) - // languageWebtoons.push(...await this.getWebtoonsFromGenre(language, genre)); - return this.removeDuplicateWebtoons(languageWebtoons); } private async getWebtoonsFromGenre(language: string, genre: string): Promise{ - // console.log(language, genre); const mobileThumbnails = await this.getWebtoonThumbnailFromGenre(language, genre); - // console.log(mobileThumbnails); const url = `https://www.webtoons.com/${language}/genres/${genre}`; const response = await this.miscService.getAxiosInstance().get(url); const document = new JSDOM(response.data).window.document; @@ -108,7 +101,6 @@ export class WebtoonParserService{ private async getWebtoonThumbnailFromGenre(language: string, genre: string): Promise[]>{ const mobileThumbnails: Record[] = []; const mobileUrl = `https://www.webtoons.com/${language}/genres/${genre}`.replace("www.webtoons", "m.webtoons") + "?webtoon-platform-redirect=true"; - // console.log(mobileUrl); const mobileResponse = await this.miscService.getAxiosInstance().get(mobileUrl, { headers: { "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148" @@ -117,7 +109,6 @@ export class WebtoonParserService{ const mobileDocument = new JSDOM((mobileResponse).data).window.document; const className = `genre_${genre.toUpperCase()}_list`; const wList = mobileDocument.querySelector(`ul.${className}`)?.querySelectorAll("li"); - // console.log("wList", wList); if(!wList) return []; for(const li of wList){ const webtoonName = li.querySelector("a")?.querySelector("div.info")?.querySelector("p.subj span")?.textContent; From 98c9d1393b902da5e76b497adc5c6209b2b6593c Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Fri, 7 Jun 2024 22:58:47 +0200 Subject: [PATCH 8/8] :mute: Remove unused logs --- src/modules/webtoon/webtoon/webtoon-parser.service.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/modules/webtoon/webtoon/webtoon-parser.service.ts b/src/modules/webtoon/webtoon/webtoon-parser.service.ts index 88713cb..11d4082 100644 --- a/src/modules/webtoon/webtoon/webtoon-parser.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-parser.service.ts @@ -76,8 +76,6 @@ export class WebtoonParserService{ const stars = a.querySelector("p.grade_area")?.querySelector("em")?.textContent; const link = a.href; const id = link.split("?title_no=")[1]; - // const thumbnail = a.querySelector("img")?.src; - // console.log("title", title); let thumbnail = mobileThumbnails.find(t => t.name === title)?.thumbnail; if(!thumbnail) thumbnail = a.querySelector("img")?.src;