From 92ae35d234e9ca5de48917feea04290f82291d6c Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Thu, 23 May 2024 13:01:39 +0200 Subject: [PATCH 1/3] :bug: Fix https port logging when starting the api in https --- src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index bacaf12..f4cb060 100644 --- a/src/main.ts +++ b/src/main.ts @@ -72,7 +72,7 @@ async function startHttpsServer(){ const httpsApp = await NestFactory.create(AppModule, new FastifyAdapter({https: httpsOptions})); await loadServer(httpsApp, getServerAddress(process.env.BIND_ADDRESS, process.env.HTTP_PORT, "https")); await httpsApp.listen(process.env.HTTPS_PORT, process.env.BIND_ADDRESS); - logServerStart(process.env.BIND_ADDRESS, process.env.HTTP_PORT, "https"); + logServerStart(process.env.BIND_ADDRESS, process.env.HTTPS_PORT, "https"); } // eslint-disable-next-line @typescript-eslint/no-unused-vars From b9b84ea2c6643753ddbae5db51399b5a9a23490a Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Thu, 23 May 2024 13:25:29 +0200 Subject: [PATCH 2/3] :zap: Implement user agent changing for faster downloads --- src/modules/misc/misc.service.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/modules/misc/misc.service.ts b/src/modules/misc/misc.service.ts index 53a4431..dca7215 100644 --- a/src/modules/misc/misc.service.ts +++ b/src/modules/misc/misc.service.ts @@ -12,12 +12,31 @@ export class MiscService{ private readonly axiosInstance: AxiosInstance; + private readonly userAgents: string[] = [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.1; rv:109.0) Gecko/20100101 Firefox/121.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0", + ] + constructor(){ this.axiosInstance = axios.create({}); - this.axiosInstance.defaults.headers.common["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1"; + this.randomUserAgentChange(1); + } + + randomUserAgentChange(probability: number = 0){ + if(Math.random() > probability) + return; + this.axiosInstance.defaults.headers.common["User-Agent"] = this.userAgents[this.randomInt(0, this.userAgents.length - 1)]; } getAxiosInstance(){ + this.randomUserAgentChange(0.1); return this.axiosInstance; } @@ -69,7 +88,7 @@ export class MiscService{ } async downloadImage(url: string, referer: string = "https://www.webtoons.com/fr/"): Promise{ - const response = await this.axiosInstance.get(url, { + const response = await this.getAxiosInstance().get(url, { responseType: "arraybuffer", headers: { "Referer": referer From 2e2156b19209abacaca2ea3ebd64cc3ff147c4da Mon Sep 17 00:00:00 2001 From: Xen0Xys Date: Thu, 23 May 2024 15:05:39 +0200 Subject: [PATCH 3/3] :loud_sound: Improve logging --- .../webtoon/download-manager.service.ts | 6 ++-- .../webtoon/webtoon-downloader.service.ts | 28 ++++++++++++++----- .../webtoon/webtoon/webtoon-parser.service.ts | 10 +++---- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/modules/webtoon/webtoon/download-manager.service.ts b/src/modules/webtoon/webtoon/download-manager.service.ts index ef6cb27..9249567 100644 --- a/src/modules/webtoon/webtoon/download-manager.service.ts +++ b/src/modules/webtoon/webtoon/download-manager.service.ts @@ -4,7 +4,7 @@ import {WebtoonParserService} from "./webtoon-parser.service"; import CachedWebtoonModel from "./models/models/cached-webtoon.model"; import EpisodeModel from "./models/models/episode.model"; import EpisodeDataModel from "./models/models/episode-data.model"; -import {HttpException, Injectable, NotFoundException} from "@nestjs/common"; +import {HttpException, Injectable, Logger, NotFoundException} from "@nestjs/common"; import WebtoonModel from "./models/models/webtoon.model"; import WebtoonDataModel from "./models/models/webtoon-data.model"; import WebtoonQueue from "../../../common/utils/models/webtoon-queue"; @@ -13,6 +13,8 @@ import {HttpStatusCode} from "axios"; @Injectable() export class DownloadManagerService{ + private readonly logger = new Logger(DownloadManagerService.name); + private cacheLoaded: boolean = false; private readonly cachePromise: Promise; private readonly queue: WebtoonQueue; @@ -63,7 +65,7 @@ export class DownloadManagerService{ this.currentDownload = this.queue.dequeue(); if(!this.currentDownload) return; - console.log(`Downloading ${this.currentDownload.title} (${this.currentDownload.language}).`); + this.logger.debug(`Downloading ${this.currentDownload.title} (${this.currentDownload.language}).`); if(!await this.webtoonDatabase.isWebtoonSaved(this.currentDownload.title, this.currentDownload.language)){ const webtoon: WebtoonModel = await this.webtoonParser.getWebtoonInfos(this.currentDownload); const webtoonData: WebtoonDataModel = await this.webtoonDownloader.downloadWebtoon(webtoon); diff --git a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts index 779dbf2..f40769d 100644 --- a/src/modules/webtoon/webtoon/webtoon-downloader.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-downloader.service.ts @@ -2,31 +2,45 @@ import EpisodeModel from "./models/models/episode.model"; import EpisodeDataModel from "./models/models/episode-data.model"; import WebtoonDataModel from "./models/models/webtoon-data.model"; import WebtoonModel from "./models/models/webtoon.model"; -import {Injectable} from "@nestjs/common"; +import {Injectable, Logger} from "@nestjs/common"; import {MiscService} from "../../misc/misc.service"; @Injectable() export class WebtoonDownloaderService{ + private readonly logger = new Logger(WebtoonDownloaderService.name); + constructor( private readonly miscService: MiscService, ){} - async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise{ - console.log(`Downloading episode ${episode.number}...`); + async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise { + this.logger.debug(`Downloading episode ${episode.number}...`); + const startTime = Date.now(); const thumbnail: Buffer = await this.miscService.downloadImage(episode.thumbnail); const conversionPromises: Promise[] = []; - for (let i = 0; i < imageUrls.length; i++){ - console.log(`Downloading image ${i + 1}/${imageUrls.length}...`); + let downloadedCount = 0; + + const interval = setInterval(() => { + const elapsedSeconds = (Date.now() - startTime) / 1000; + const imagesPerSecond = downloadedCount / elapsedSeconds; + this.logger.debug(`Downloading ${downloadedCount} of ${imageUrls.length} images (${(imagesPerSecond).toFixed(2)} images/s)...`); + }, 1000); + + for (let i = 0; i < imageUrls.length; i++) { const url = imageUrls[i]; const image = await this.miscService.downloadImage(url, episode.link); conversionPromises.push(this.miscService.convertImageToWebp(image)); + downloadedCount++; await new Promise(resolve => setTimeout(resolve, this.miscService.randomInt(50, 200))); } + + clearInterval(interval); + this.logger.debug(`Downloaded ${downloadedCount}/${imageUrls.length} images in ${((Date.now() - startTime) / 1000).toFixed(2)} seconds.`); + // Convert all images to webp - console.log("Converting images to webp..."); const convertedImages: Buffer[] = await Promise.all(conversionPromises); - console.log(`Download complete for episode ${episode.number}!`); + this.logger.debug(`Download complete for episode ${episode.number}!`); return { thumbnail, images: convertedImages diff --git a/src/modules/webtoon/webtoon/webtoon-parser.service.ts b/src/modules/webtoon/webtoon/webtoon-parser.service.ts index b204b9b..d8e7385 100644 --- a/src/modules/webtoon/webtoon/webtoon-parser.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-parser.service.ts @@ -24,20 +24,20 @@ export class WebtoonParserService{ async loadCache(): Promise{ // Load existing cache if(fs.existsSync("./.cache/webtoons.json")){ - this.logger.log("Loading webtoon list from cache..."); + this.logger.debug("Loading webtoon list from cache..."); this.webtoons = JSON.parse(fs.readFileSync("./.cache/webtoons.json").toString()); const webtoonCount = Object.values(this.webtoons).reduce((acc, val: any) => acc + val.length, 0); - this.logger.log(`Loaded ${webtoonCount} webtoons!`); + this.logger.debug(`Loaded ${webtoonCount} webtoons!`); return; } - this.logger.log("Loading webtoon list..."); + this.logger.debug("Loading webtoon list..."); // Generate and save cache for (const language of Object.values(WebtoonLanguages)){ - this.logger.log(`Loading webtoons for language: ${language}`); + this.logger.debug(`Loading webtoons for language: ${language}`); this.webtoons[language] = await this.getWebtoonsFromLanguage(language); } const webtoonCount = Object.values(this.webtoons).reduce((acc, val: any) => acc + val.length, 0); - this.logger.log(`Loaded ${webtoonCount} webtoons!`); + this.logger.debug(`Loaded ${webtoonCount} webtoons!`); // Save cache fs.mkdirSync("./.cache", {recursive: true}); fs.writeFileSync("./.cache/webtoons.json", JSON.stringify(this.webtoons, null, 2));