diff --git a/src/app.module.ts b/src/app.module.ts index 36947df..51cf8ba 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -7,6 +7,7 @@ import {ThrottlerGuard, ThrottlerModule} from "@nestjs/throttler"; import {APP_GUARD} from "@nestjs/core"; import {MigrationModule} from "./modules/webtoon/migration/migration.module"; import {ScheduleModule} from "@nestjs/schedule"; +import {UpdateModule} from "./modules/webtoon/update/update.module"; @Module({ imports: [ @@ -20,6 +21,7 @@ import {ScheduleModule} from "@nestjs/schedule"; WebtoonModule, AdminModule, MigrationModule, + UpdateModule, ], controllers: [], providers: [ diff --git a/src/modules/misc/misc.service.ts b/src/modules/misc/misc.service.ts index dca7215..c3f439a 100644 --- a/src/modules/misc/misc.service.ts +++ b/src/modules/misc/misc.service.ts @@ -22,7 +22,7 @@ export class MiscService{ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0", - ] + ]; constructor(){ this.axiosInstance = axios.create({}); @@ -97,6 +97,14 @@ export class MiscService{ return response.data as Buffer; } + async convertWebtoonThumbnail(url: string){ + const webpImage: Buffer = await this.downloadImage(url); + return await sharp(webpImage).resize(240, 240, { + fit: "cover", + position: "center" + }).toBuffer(); + } + parseWebtoonStars(stars: string): WebtoonStarModel{ // TODO: Fix dutch space and comma issue stars = stars.replace(" ", " "); diff --git a/src/modules/webtoon/admin/admin.controller.ts b/src/modules/webtoon/admin/admin.controller.ts index 46476d8..c942d9c 100644 --- a/src/modules/webtoon/admin/admin.controller.ts +++ b/src/modules/webtoon/admin/admin.controller.ts @@ -63,4 +63,11 @@ export class AdminController{ async clearQueue(): Promise{ return this.downloadManagerService.clearDownloadQueue(); } + + @Post("refresh-cache") + @ApiBearerAuth() + @ApiResponse({status: HttpStatusCode.Created, description: "Refreshes the cache"}) + async refreshCache(): Promise{ + return this.downloadManagerService.refreshCache(); + } } diff --git a/src/modules/webtoon/update/update.controller.ts b/src/modules/webtoon/update/update.controller.ts new file mode 100644 index 0000000..e59fe3d --- /dev/null +++ b/src/modules/webtoon/update/update.controller.ts @@ -0,0 +1,25 @@ +import {Controller, Logger, Post, UseGuards} from "@nestjs/common"; +import {UpdateService} from "./update.service"; +import {ApiBearerAuth, ApiResponse, ApiTags} from "@nestjs/swagger"; +import {AdminGuard} from "../admin/guard/admin.guard"; +import {HttpStatusCode} from "axios"; + + +@Controller("update") +@ApiTags("Update") +@UseGuards(AdminGuard) +export class UpdateController{ + + private readonly logger = new Logger(UpdateController.name); + + constructor( + private readonly updateService: UpdateService, + ){} + + @Post("webtoons/thumbnails") + @ApiBearerAuth() + @ApiResponse({status: HttpStatusCode.Created, description: "Thumbnails updated"}) + async updateThumbnails(): Promise{ + this.updateService.updateThumbnails().then(() => this.logger.log("Thumbnails updated")); + } +} diff --git a/src/modules/webtoon/update/update.module.ts b/src/modules/webtoon/update/update.module.ts new file mode 100644 index 0000000..14b8b99 --- /dev/null +++ b/src/modules/webtoon/update/update.module.ts @@ -0,0 +1,13 @@ +import {Module} from "@nestjs/common"; +import {UpdateService} from "./update.service"; +import {UpdateController} from "./update.controller"; +import {MiscModule} from "../../misc/misc.module"; +import {WebtoonModule} from "../webtoon/webtoon.module"; + + +@Module({ + providers: [UpdateService], + controllers: [UpdateController], + imports: [MiscModule, WebtoonModule] +}) +export class UpdateModule{} diff --git a/src/modules/webtoon/update/update.service.ts b/src/modules/webtoon/update/update.service.ts new file mode 100644 index 0000000..49df9da --- /dev/null +++ b/src/modules/webtoon/update/update.service.ts @@ -0,0 +1,86 @@ +import {Injectable, Logger} from "@nestjs/common"; +import {PrismaService} from "../../misc/prisma.service"; +import {WebtoonParserService} from "../webtoon/webtoon-parser.service"; +import CachedWebtoonModel from "../webtoon/models/models/cached-webtoon.model"; +import {MiscService} from "../../misc/misc.service"; +import {WebtoonDatabaseService} from "../webtoon/webtoon-database.service"; +import ImageTypes from "../webtoon/models/enums/image-types"; + + +@Injectable() +export class UpdateService{ + + private readonly logger = new Logger(UpdateService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly webtoonParser: WebtoonParserService, + private readonly miscService: MiscService, + private readonly webtoonDatabaseService: WebtoonDatabaseService, + ){} + + async updateThumbnails(): Promise{ + this.webtoonParser.clearCache(); + await this.webtoonParser.loadCache(); + await this.prismaService.$transaction(async(tx) => { + const dbWebtoons: any[] = await tx.webtoons.findMany({ + select: { + id: true, + title: true, + language: true, + thumbnail_id: true + } + }); + const thumbnailsToDelete: number[] = dbWebtoons.map(webtoon => webtoon.thumbnail_id); + + // Save new thumbnails + const dbThumbnailType = await tx.imageTypes.findFirst({ + where: { + name: ImageTypes.WEBTOON_THUMBNAIL + }, + select: { + id: true + } + }); + for(const webtoon of dbWebtoons){ + this.logger.debug(`Updating thumbnail for webtoon ${webtoon.title} (${webtoon.language})`); + const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language); + const thumbnail: Buffer = await this.miscService.convertWebtoonThumbnail(cachedWebtoon.thumbnail); + const sum: string = this.webtoonDatabaseService.saveImage(thumbnail); + // Check if thumbnail already exists + let dbThumbnail = await tx.images.findFirst({ + where: { + sum: sum + } + }); + if(dbThumbnail) + thumbnailsToDelete.splice(thumbnailsToDelete.indexOf(dbThumbnail.id), 1); + else + dbThumbnail = await tx.images.create({ + data: { + sum: sum, + type_id: dbThumbnailType.id + } + }); + // Update webtoon thumbnail + await tx.webtoons.update({ + where: { + id: webtoon.id + }, + data: { + thumbnail_id: dbThumbnail.id + } + }); + } + // Delete old thumbnails + for(const thumbnailId of thumbnailsToDelete){ + const deletedImage = await tx.images.delete({ + where: { + id: thumbnailId + } + }); + this.webtoonDatabaseService.removeImage(deletedImage.sum); + } + }); + } +} diff --git a/src/modules/webtoon/webtoon/download-manager.service.ts b/src/modules/webtoon/webtoon/download-manager.service.ts index 6ab73ed..e4a4679 100644 --- a/src/modules/webtoon/webtoon/download-manager.service.ts +++ b/src/modules/webtoon/webtoon/download-manager.service.ts @@ -7,7 +7,6 @@ import EpisodeDataModel from "./models/models/episode-data.model"; import {HttpException, Injectable, Logger, NotFoundException} from "@nestjs/common"; import WebtoonModel from "./models/models/webtoon.model"; import WebtoonDataModel from "./models/models/webtoon-data.model"; -import WebtoonQueue from "../../../common/utils/models/webtoon-queue"; import {HttpStatusCode} from "axios"; import DownloadQueue from "../../../common/utils/models/download-queue"; @@ -17,7 +16,7 @@ export class DownloadManagerService{ private readonly logger = new Logger(DownloadManagerService.name); private cacheLoaded: boolean = false; - private readonly cachePromise: Promise; + private cachePromise: Promise; private readonly downloadQueue: DownloadQueue; constructor( @@ -34,7 +33,7 @@ export class DownloadManagerService{ } this.cachePromise = this.webtoonParser.loadCache(); this.cachePromise.then(() => { - this.cacheLoaded = true + this.cacheLoaded = true; if(downloadInProgress) this.startDownload().then(() => console.log("Download finished.")); }); @@ -44,9 +43,20 @@ export class DownloadManagerService{ return this.cachePromise; } + async refreshCache(): Promise{ + if(!this.cacheLoaded) + throw new HttpException("Cache already loading.", HttpStatusCode.TooEarly); + this.cacheLoaded = false; + this.webtoonParser.clearCache(); + this.cachePromise = this.webtoonParser.loadCache(); + this.cachePromise.then(() => { + this.cacheLoaded = true; + }); + } + async addWebtoonToQueue(webtoonName: string, language = "en"): Promise{ if(!this.cacheLoaded) - throw new Error("Cache not loaded."); + throw new HttpException("Cache already loading.", HttpStatusCode.TooEarly); const webtoonOverview: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoonName, language); // If queue is empty, start download this.downloadQueue.enqueue(webtoonOverview); @@ -69,6 +79,7 @@ export class DownloadManagerService{ if(!this.cacheLoaded) throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly); while(!this.downloadQueue.isQueueEmpty()){ + await this.cachePromise; // Wait for cache to be loaded if it is cleared const currentDownload: CachedWebtoonModel = this.downloadQueue.dequeue(); if(!currentDownload) return; diff --git a/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts b/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts index 17b46ee..dc49167 100644 --- a/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts +++ b/src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts @@ -11,7 +11,23 @@ enum WebtoonGenres { SF = "sf", HORROR = "horror", TIPTOON = "tiptoon", - LOCAL = "local" + LOCAL = "local", + SCHOOL = "school", + MARTIAL_ARTS = "martial_arts", + BL_GL = "bl_gl", + ROMANCE_M = "romance_m", + TIME_SLIP = "time_slip", + CITY_OFFICE = "city_office", + MYSTERY = "mystery", + HEARTWARMING = "heartwarming", + SHONEN = "shonen", + EASTERN_PALACE = "eastern_palace", + WEB_NOVEL = "web_novel", + WESTERN_PALACE = "western_palace", + ADAPTATION = "adaptation", + SUPERNATURAL = "supernatural", + HISTORICAL = "historical", + ROMANTIC_FANTASY = "romantic_fantasy", } export default WebtoonGenres; diff --git a/src/modules/webtoon/webtoon/webtoon-database.service.ts b/src/modules/webtoon/webtoon/webtoon-database.service.ts index c83d580..d2be2b4 100644 --- a/src/modules/webtoon/webtoon/webtoon-database.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-database.service.ts @@ -406,4 +406,9 @@ export class WebtoonDatabaseService{ const folder = imageSum.substring(0, 2); return fs.readFileSync(`./images/${folder}/${imageSum}.webp`); } + + removeImage(imageSum: string): void{ + const folder = imageSum.substring(0, 2); + fs.rmSync(`./images/${folder}/${imageSum}.webp`); + } } diff --git a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts index f40769d..0d8c0aa 100644 --- a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts @@ -4,6 +4,7 @@ import WebtoonDataModel from "./models/models/webtoon-data.model"; import WebtoonModel from "./models/models/webtoon.model"; import {Injectable, Logger} from "@nestjs/common"; import {MiscService} from "../../misc/misc.service"; +import * as sharp from "sharp"; @Injectable() export class WebtoonDownloaderService{ @@ -14,7 +15,7 @@ export class WebtoonDownloaderService{ private readonly miscService: MiscService, ){} - async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise { + async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise{ this.logger.debug(`Downloading episode ${episode.number}...`); const startTime = Date.now(); const thumbnail: Buffer = await this.miscService.downloadImage(episode.thumbnail); @@ -27,7 +28,7 @@ export class WebtoonDownloaderService{ this.logger.debug(`Downloading ${downloadedCount} of ${imageUrls.length} images (${(imagesPerSecond).toFixed(2)} images/s)...`); }, 1000); - for (let i = 0; i < imageUrls.length; i++) { + for (let i = 0; i < imageUrls.length; i++){ const url = imageUrls[i]; const image = await this.miscService.downloadImage(url, episode.link); conversionPromises.push(this.miscService.convertImageToWebp(image)); @@ -49,7 +50,7 @@ export class WebtoonDownloaderService{ async downloadWebtoon(webtoon: WebtoonModel): Promise{ const downloadPromises: Promise[] = []; - downloadPromises.push(this.miscService.downloadImage(webtoon.thumbnail)); + downloadPromises.push(this.miscService.convertWebtoonThumbnail(webtoon.thumbnail)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.background)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.top)); downloadPromises.push(this.miscService.downloadImage(webtoon.banner.mobile)); diff --git a/src/modules/webtoon/webtoon/webtoon-parser.service.ts b/src/modules/webtoon/webtoon/webtoon-parser.service.ts index d8e7385..11d4082 100644 --- a/src/modules/webtoon/webtoon/webtoon-parser.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-parser.service.ts @@ -21,6 +21,12 @@ export class WebtoonParserService{ private readonly miscService: MiscService ){} + clearCache(): void{ + if(fs.existsSync("./.cache/webtoons.json")) + fs.unlinkSync("./.cache/webtoons.json"); + this.webtoons = {}; + } + async loadCache(): Promise{ // Load existing cache if(fs.existsSync("./.cache/webtoons.json")){ @@ -55,6 +61,7 @@ export class WebtoonParserService{ } private async getWebtoonsFromGenre(language: string, genre: string): Promise{ + const mobileThumbnails = await this.getWebtoonThumbnailFromGenre(language, genre); const url = `https://www.webtoons.com/${language}/genres/${genre}`; const response = await this.miscService.getAxiosInstance().get(url); const document = new JSDOM(response.data).window.document; @@ -69,7 +76,9 @@ export class WebtoonParserService{ const stars = a.querySelector("p.grade_area")?.querySelector("em")?.textContent; const link = a.href; const id = link.split("?title_no=")[1]; - const thumbnail = a.querySelector("img")?.src; + let thumbnail = mobileThumbnails.find(t => t.name === title)?.thumbnail; + if(!thumbnail) + thumbnail = a.querySelector("img")?.src; if(!title || !author || !stars || !link || !thumbnail || !id) throw new NotFoundException(`Missing data for webtoon: ${url}`); const webtoon: CachedWebtoonModel = { @@ -87,6 +96,27 @@ export class WebtoonParserService{ return webtoons; } + private async getWebtoonThumbnailFromGenre(language: string, genre: string): Promise[]>{ + const mobileThumbnails: Record[] = []; + const mobileUrl = `https://www.webtoons.com/${language}/genres/${genre}`.replace("www.webtoons", "m.webtoons") + "?webtoon-platform-redirect=true"; + const mobileResponse = await this.miscService.getAxiosInstance().get(mobileUrl, { + headers: { + "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148" + } + }); + const mobileDocument = new JSDOM((mobileResponse).data).window.document; + const className = `genre_${genre.toUpperCase()}_list`; + const wList = mobileDocument.querySelector(`ul.${className}`)?.querySelectorAll("li"); + if(!wList) return []; + for(const li of wList){ + const webtoonName = li.querySelector("a")?.querySelector("div.info")?.querySelector("p.subj span")?.textContent; + const imgLink = li.querySelector("a")?.querySelector("div.pic")?.querySelector("img")?.src; + if(!webtoonName || !imgLink) continue; + mobileThumbnails.push({name: webtoonName, thumbnail: imgLink}); + } + return mobileThumbnails; + } + private removeDuplicateWebtoons(webtoons: CachedWebtoonModel[]){ const webtoonsWithoutDuplicates: CachedWebtoonModel[] = []; for(const webtoon of webtoons){ diff --git a/src/modules/webtoon/webtoon/webtoon.module.ts b/src/modules/webtoon/webtoon/webtoon.module.ts index 10b19ee..e91206d 100644 --- a/src/modules/webtoon/webtoon/webtoon.module.ts +++ b/src/modules/webtoon/webtoon/webtoon.module.ts @@ -10,6 +10,6 @@ import {MiscModule} from "../../misc/misc.module"; imports: [MiscModule], controllers: [WebtoonController], providers: [WebtoonParserService, WebtoonDownloaderService, WebtoonDatabaseService, DownloadManagerService], - exports: [DownloadManagerService, WebtoonDatabaseService], + exports: [DownloadManagerService, WebtoonDatabaseService, WebtoonParserService], }) export class WebtoonModule{}