Merge pull request #37 from Open-Webtoon-Reader/feature/better-thumbnail

Merge better thumbnail to Main

src/app.module.ts

@@ -7,6 +7,7 @@ import {ThrottlerGuard, ThrottlerModule} from "@nestjs/throttler";
 import {APP_GUARD} from "@nestjs/core";
 import {MigrationModule} from "./modules/webtoon/migration/migration.module";
 import {ScheduleModule} from "@nestjs/schedule";
+import {UpdateModule} from "./modules/webtoon/update/update.module";
 
 @Module({
     imports: [
@@ -20,6 +21,7 @@ import {ScheduleModule} from "@nestjs/schedule";
         WebtoonModule,
         AdminModule,
         MigrationModule,
+        UpdateModule,
     ],
     controllers: [],
     providers: [

src/modules/misc/misc.service.ts

@@ -22,7 +22,7 @@ export class MiscService{
         "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91",
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0",
         "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0",
-    ]
+    ];
 
     constructor(){
         this.axiosInstance = axios.create({});
@@ -97,6 +97,14 @@ export class MiscService{
         return response.data as Buffer;
     }
 
+    async convertWebtoonThumbnail(url: string){
+        const webpImage: Buffer = await this.downloadImage(url);
+        return await sharp(webpImage).resize(240, 240, {
+            fit: "cover",
+            position: "center"
+        }).toBuffer();
+    }
+
     parseWebtoonStars(stars: string): WebtoonStarModel{
         // TODO: Fix dutch space and comma issue
         stars = stars.replace(" ", " ");

src/modules/webtoon/admin/admin.controller.ts

@@ -63,4 +63,11 @@ export class AdminController{
     async clearQueue(): Promise<void>{
         return this.downloadManagerService.clearDownloadQueue();
     }
+
+    @Post("refresh-cache")
+    @ApiBearerAuth()
+    @ApiResponse({status: HttpStatusCode.Created, description: "Refreshes the cache"})
+    async refreshCache(): Promise<void>{
+        return this.downloadManagerService.refreshCache();
+    }
 }

src/modules/webtoon/update/update.controller.ts

@@ -0,0 +1,25 @@
+import {Controller, Logger, Post, UseGuards} from "@nestjs/common";
+import {UpdateService} from "./update.service";
+import {ApiBearerAuth, ApiResponse, ApiTags} from "@nestjs/swagger";
+import {AdminGuard} from "../admin/guard/admin.guard";
+import {HttpStatusCode} from "axios";
+
+
+@Controller("update")
+@ApiTags("Update")
+@UseGuards(AdminGuard)
+export class UpdateController{
+
+    private readonly logger = new Logger(UpdateController.name);
+
+    constructor(
+        private readonly updateService: UpdateService,
+    ){}
+
+    @Post("webtoons/thumbnails")
+    @ApiBearerAuth()
+    @ApiResponse({status: HttpStatusCode.Created, description: "Thumbnails updated"})
+    async updateThumbnails(): Promise<void>{
+        this.updateService.updateThumbnails().then(() => this.logger.log("Thumbnails updated"));
+    }
+}

src/modules/webtoon/update/update.module.ts

@@ -0,0 +1,13 @@
+import {Module} from "@nestjs/common";
+import {UpdateService} from "./update.service";
+import {UpdateController} from "./update.controller";
+import {MiscModule} from "../../misc/misc.module";
+import {WebtoonModule} from "../webtoon/webtoon.module";
+
+
+@Module({
+    providers: [UpdateService],
+    controllers: [UpdateController],
+    imports: [MiscModule, WebtoonModule]
+})
+export class UpdateModule{}

src/modules/webtoon/update/update.service.ts

@@ -0,0 +1,86 @@
+import {Injectable, Logger} from "@nestjs/common";
+import {PrismaService} from "../../misc/prisma.service";
+import {WebtoonParserService} from "../webtoon/webtoon-parser.service";
+import CachedWebtoonModel from "../webtoon/models/models/cached-webtoon.model";
+import {MiscService} from "../../misc/misc.service";
+import {WebtoonDatabaseService} from "../webtoon/webtoon-database.service";
+import ImageTypes from "../webtoon/models/enums/image-types";
+
+
+@Injectable()
+export class UpdateService{
+
+    private readonly logger = new Logger(UpdateService.name);
+
+    constructor(
+        private readonly prismaService: PrismaService,
+        private readonly webtoonParser: WebtoonParserService,
+        private readonly miscService: MiscService,
+        private readonly webtoonDatabaseService: WebtoonDatabaseService,
+    ){}
+
+    async updateThumbnails(): Promise<void>{
+        this.webtoonParser.clearCache();
+        await this.webtoonParser.loadCache();
+        await this.prismaService.$transaction(async(tx) => {
+            const dbWebtoons: any[] = await tx.webtoons.findMany({
+                select: {
+                    id: true,
+                    title: true,
+                    language: true,
+                    thumbnail_id: true
+                }
+            });
+            const thumbnailsToDelete: number[] = dbWebtoons.map(webtoon => webtoon.thumbnail_id);
+
+            // Save new thumbnails
+            const dbThumbnailType = await tx.imageTypes.findFirst({
+                where: {
+                    name: ImageTypes.WEBTOON_THUMBNAIL
+                },
+                select: {
+                    id: true
+                }
+            });
+            for(const webtoon of dbWebtoons){
+                this.logger.debug(`Updating thumbnail for webtoon ${webtoon.title} (${webtoon.language})`);
+                const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language);
+                const thumbnail: Buffer = await this.miscService.convertWebtoonThumbnail(cachedWebtoon.thumbnail);
+                const sum: string = this.webtoonDatabaseService.saveImage(thumbnail);
+                // Check if thumbnail already exists
+                let dbThumbnail = await tx.images.findFirst({
+                    where: {
+                        sum: sum
+                    }
+                });
+                if(dbThumbnail)
+                    thumbnailsToDelete.splice(thumbnailsToDelete.indexOf(dbThumbnail.id), 1);
+                else
+                    dbThumbnail = await tx.images.create({
+                        data: {
+                            sum: sum,
+                            type_id: dbThumbnailType.id
+                        }
+                    });
+                // Update webtoon thumbnail
+                await tx.webtoons.update({
+                    where: {
+                        id: webtoon.id
+                    },
+                    data: {
+                        thumbnail_id: dbThumbnail.id
+                    }
+                });
+            }
+            // Delete old thumbnails
+            for(const thumbnailId of thumbnailsToDelete){
+                const deletedImage = await tx.images.delete({
+                    where: {
+                        id: thumbnailId
+                    }
+                });
+                this.webtoonDatabaseService.removeImage(deletedImage.sum);
+            }
+        });
+    }
+}

src/modules/webtoon/webtoon/download-manager.service.ts

@@ -7,7 +7,6 @@ import EpisodeDataModel from "./models/models/episode-data.model";
 import {HttpException, Injectable, Logger, NotFoundException} from "@nestjs/common";
 import WebtoonModel from "./models/models/webtoon.model";
 import WebtoonDataModel from "./models/models/webtoon-data.model";
-import WebtoonQueue from "../../../common/utils/models/webtoon-queue";
 import {HttpStatusCode} from "axios";
 import DownloadQueue from "../../../common/utils/models/download-queue";
 
@@ -17,7 +16,7 @@ export class DownloadManagerService{
     private readonly logger = new Logger(DownloadManagerService.name);
 
     private cacheLoaded: boolean = false;
-    private readonly cachePromise: Promise<void>;
+    private cachePromise: Promise<void>;
     private readonly downloadQueue: DownloadQueue;
 
     constructor(
@@ -34,7 +33,7 @@ export class DownloadManagerService{
         }
         this.cachePromise = this.webtoonParser.loadCache();
         this.cachePromise.then(() => {
-            this.cacheLoaded = true
+            this.cacheLoaded = true;
             if(downloadInProgress)
                 this.startDownload().then(() => console.log("Download finished."));
         });
@@ -44,9 +43,20 @@ export class DownloadManagerService{
         return this.cachePromise;
     }
 
+    async refreshCache(): Promise<void>{
+        if(!this.cacheLoaded)
+            throw new HttpException("Cache already loading.", HttpStatusCode.TooEarly);
+        this.cacheLoaded = false;
+        this.webtoonParser.clearCache();
+        this.cachePromise = this.webtoonParser.loadCache();
+        this.cachePromise.then(() => {
+            this.cacheLoaded = true;
+        });
+    }
+
     async addWebtoonToQueue(webtoonName: string, language = "en"): Promise<void>{
         if(!this.cacheLoaded)
-            throw new Error("Cache not loaded.");
+            throw new HttpException("Cache already loading.", HttpStatusCode.TooEarly);
         const webtoonOverview: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoonName, language);
         // If queue is empty, start download
         this.downloadQueue.enqueue(webtoonOverview);
@@ -69,6 +79,7 @@ export class DownloadManagerService{
         if(!this.cacheLoaded)
             throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly);
         while(!this.downloadQueue.isQueueEmpty()){
+            await this.cachePromise; // Wait for cache to be loaded if it is cleared
             const currentDownload: CachedWebtoonModel = this.downloadQueue.dequeue();
             if(!currentDownload)
                 return;

src/modules/webtoon/webtoon/models/enums/webtoon-genres.ts

@@ -11,7 +11,23 @@ enum WebtoonGenres {
     SF = "sf",
     HORROR = "horror",
     TIPTOON = "tiptoon",
-    LOCAL = "local"
+    LOCAL = "local",
+    SCHOOL = "school",
+    MARTIAL_ARTS = "martial_arts",
+    BL_GL = "bl_gl",
+    ROMANCE_M = "romance_m",
+    TIME_SLIP = "time_slip",
+    CITY_OFFICE = "city_office",
+    MYSTERY = "mystery",
+    HEARTWARMING = "heartwarming",
+    SHONEN = "shonen",
+    EASTERN_PALACE = "eastern_palace",
+    WEB_NOVEL = "web_novel",
+    WESTERN_PALACE = "western_palace",
+    ADAPTATION = "adaptation",
+    SUPERNATURAL = "supernatural",
+    HISTORICAL = "historical",
+    ROMANTIC_FANTASY = "romantic_fantasy",
 }
 
 export default WebtoonGenres;

src/modules/webtoon/webtoon/webtoon-database.service.ts

@@ -406,4 +406,9 @@ export class WebtoonDatabaseService{
         const folder = imageSum.substring(0, 2);
         return fs.readFileSync(`./images/${folder}/${imageSum}.webp`);
     }
+
+    removeImage(imageSum: string): void{
+        const folder = imageSum.substring(0, 2);
+        fs.rmSync(`./images/${folder}/${imageSum}.webp`);
+    }
 }

src/modules/webtoon/webtoon/webtoon-downloader.service.ts

@@ -4,6 +4,7 @@ import WebtoonDataModel from "./models/models/webtoon-data.model";
 import WebtoonModel from "./models/models/webtoon.model";
 import {Injectable, Logger} from "@nestjs/common";
 import {MiscService} from "../../misc/misc.service";
+import * as sharp from "sharp";
 
 @Injectable()
 export class WebtoonDownloaderService{
@@ -14,7 +15,7 @@ export class WebtoonDownloaderService{
         private readonly miscService: MiscService,
     ){}
 
-    async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise<EpisodeDataModel> {
+    async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise<EpisodeDataModel>{
         this.logger.debug(`Downloading episode ${episode.number}...`);
         const startTime = Date.now();
         const thumbnail: Buffer = await this.miscService.downloadImage(episode.thumbnail);
@@ -27,7 +28,7 @@ export class WebtoonDownloaderService{
             this.logger.debug(`Downloading ${downloadedCount} of ${imageUrls.length} images (${(imagesPerSecond).toFixed(2)} images/s)...`);
         }, 1000);
 
-        for (let i = 0; i < imageUrls.length; i++) {
+        for (let i = 0; i < imageUrls.length; i++){
             const url = imageUrls[i];
             const image = await this.miscService.downloadImage(url, episode.link);
             conversionPromises.push(this.miscService.convertImageToWebp(image));
@@ -49,7 +50,7 @@ export class WebtoonDownloaderService{
 
     async downloadWebtoon(webtoon: WebtoonModel): Promise<WebtoonDataModel>{
         const downloadPromises: Promise<Buffer>[] = [];
-        downloadPromises.push(this.miscService.downloadImage(webtoon.thumbnail));
+        downloadPromises.push(this.miscService.convertWebtoonThumbnail(webtoon.thumbnail));
         downloadPromises.push(this.miscService.downloadImage(webtoon.banner.background));
         downloadPromises.push(this.miscService.downloadImage(webtoon.banner.top));
         downloadPromises.push(this.miscService.downloadImage(webtoon.banner.mobile));

src/modules/webtoon/webtoon/webtoon-parser.service.ts

@@ -21,6 +21,12 @@ export class WebtoonParserService{
         private readonly miscService: MiscService
     ){}
 
+    clearCache(): void{
+        if(fs.existsSync("./.cache/webtoons.json"))
+            fs.unlinkSync("./.cache/webtoons.json");
+        this.webtoons = {};
+    }
+
     async loadCache(): Promise<void>{
         // Load existing cache
         if(fs.existsSync("./.cache/webtoons.json")){
@@ -55,6 +61,7 @@ export class WebtoonParserService{
     }
 
     private async getWebtoonsFromGenre(language: string, genre: string): Promise<CachedWebtoonModel[]>{
+        const mobileThumbnails = await this.getWebtoonThumbnailFromGenre(language, genre);
         const url = `https://www.webtoons.com/${language}/genres/${genre}`;
         const response = await this.miscService.getAxiosInstance().get(url);
         const document = new JSDOM(response.data).window.document;
@@ -69,7 +76,9 @@ export class WebtoonParserService{
             const stars = a.querySelector("p.grade_area")?.querySelector("em")?.textContent;
             const link = a.href;
             const id = link.split("?title_no=")[1];
-            const thumbnail = a.querySelector("img")?.src;
+            let thumbnail = mobileThumbnails.find(t => t.name === title)?.thumbnail;
+            if(!thumbnail)
+                thumbnail = a.querySelector("img")?.src;
             if(!title || !author || !stars || !link || !thumbnail || !id)
                 throw new NotFoundException(`Missing data for webtoon: ${url}`);
             const webtoon: CachedWebtoonModel = {
@@ -87,6 +96,27 @@ export class WebtoonParserService{
         return webtoons;
     }
 
+    private async getWebtoonThumbnailFromGenre(language: string, genre: string): Promise<Record<string, string>[]>{
+        const mobileThumbnails: Record<string, string>[] = [];
+        const mobileUrl = `https://www.webtoons.com/${language}/genres/${genre}`.replace("www.webtoons", "m.webtoons") + "?webtoon-platform-redirect=true";
+        const mobileResponse = await this.miscService.getAxiosInstance().get(mobileUrl, {
+            headers: {
+                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"
+            }
+        });
+        const mobileDocument = new JSDOM((mobileResponse).data).window.document;
+        const className = `genre_${genre.toUpperCase()}_list`;
+        const wList = mobileDocument.querySelector(`ul.${className}`)?.querySelectorAll("li");
+        if(!wList) return [];
+        for(const li of wList){
+            const webtoonName = li.querySelector("a")?.querySelector("div.info")?.querySelector("p.subj span")?.textContent;
+            const imgLink = li.querySelector("a")?.querySelector("div.pic")?.querySelector("img")?.src;
+            if(!webtoonName || !imgLink) continue;
+            mobileThumbnails.push({name: webtoonName, thumbnail: imgLink});
+        }
+        return mobileThumbnails;
+    }
+
     private removeDuplicateWebtoons(webtoons: CachedWebtoonModel[]){
         const webtoonsWithoutDuplicates: CachedWebtoonModel[] = [];
         for(const webtoon of webtoons){

src/modules/webtoon/webtoon/webtoon.module.ts

@@ -10,6 +10,6 @@ import {MiscModule} from "../../misc/misc.module";
     imports: [MiscModule],
     controllers: [WebtoonController],
     providers: [WebtoonParserService, WebtoonDownloaderService, WebtoonDatabaseService, DownloadManagerService],
-    exports: [DownloadManagerService, WebtoonDatabaseService],
+    exports: [DownloadManagerService, WebtoonDatabaseService, WebtoonParserService],
 })
 export class WebtoonModule{}