Merge pull request #29 from Open-Webtoon-Reader/improvement/global-im…

…provements

Merge global improvements to Main

src/main.ts

@@ -72,7 +72,7 @@ async function startHttpsServer(){
     const httpsApp = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter({https: httpsOptions}));
     await loadServer(httpsApp, getServerAddress(process.env.BIND_ADDRESS, process.env.HTTP_PORT, "https"));
     await httpsApp.listen(process.env.HTTPS_PORT, process.env.BIND_ADDRESS);
-    logServerStart(process.env.BIND_ADDRESS, process.env.HTTP_PORT, "https");
+    logServerStart(process.env.BIND_ADDRESS, process.env.HTTPS_PORT, "https");
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars

src/modules/misc/misc.service.ts

@@ -12,12 +12,31 @@ export class MiscService{
 
     private readonly axiosInstance: AxiosInstance;
 
+    private readonly userAgents: string[] = [
+        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.1; rv:109.0) Gecko/20100101 Firefox/121.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.91",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0",
+    ]
+
     constructor(){
         this.axiosInstance = axios.create({});
-        this.axiosInstance.defaults.headers.common["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1";
+        this.randomUserAgentChange(1);
+    }
+
+    randomUserAgentChange(probability: number = 0){
+        if(Math.random() > probability)
+            return;
+        this.axiosInstance.defaults.headers.common["User-Agent"] = this.userAgents[this.randomInt(0, this.userAgents.length - 1)];
     }
 
     getAxiosInstance(){
+        this.randomUserAgentChange(0.1);
         return this.axiosInstance;
     }
 
@@ -69,7 +88,7 @@ export class MiscService{
     }
 
     async downloadImage(url: string, referer: string = "https://www.webtoons.com/fr/"): Promise<Buffer>{
-        const response = await this.axiosInstance.get(url, {
+        const response = await this.getAxiosInstance().get(url, {
             responseType: "arraybuffer",
             headers: {
                 "Referer": referer

src/modules/webtoon/webtoon/download-manager.service.ts

@@ -4,7 +4,7 @@ import {WebtoonParserService} from "./webtoon-parser.service";
 import CachedWebtoonModel from "./models/models/cached-webtoon.model";
 import EpisodeModel from "./models/models/episode.model";
 import EpisodeDataModel from "./models/models/episode-data.model";
-import {HttpException, Injectable, NotFoundException} from "@nestjs/common";
+import {HttpException, Injectable, Logger, NotFoundException} from "@nestjs/common";
 import WebtoonModel from "./models/models/webtoon.model";
 import WebtoonDataModel from "./models/models/webtoon-data.model";
 import WebtoonQueue from "../../../common/utils/models/webtoon-queue";
@@ -13,6 +13,8 @@ import {HttpStatusCode} from "axios";
 @Injectable()
 export class DownloadManagerService{
 
+    private readonly logger = new Logger(DownloadManagerService.name);
+
     private cacheLoaded: boolean = false;
     private readonly cachePromise: Promise<void>;
     private readonly queue: WebtoonQueue;
@@ -63,7 +65,7 @@ export class DownloadManagerService{
             this.currentDownload = this.queue.dequeue();
             if(!this.currentDownload)
                 return;
-            console.log(`Downloading ${this.currentDownload.title} (${this.currentDownload.language}).`);
+            this.logger.debug(`Downloading ${this.currentDownload.title} (${this.currentDownload.language}).`);
             if(!await this.webtoonDatabase.isWebtoonSaved(this.currentDownload.title, this.currentDownload.language)){
                 const webtoon: WebtoonModel = await this.webtoonParser.getWebtoonInfos(this.currentDownload);
                 const webtoonData: WebtoonDataModel = await this.webtoonDownloader.downloadWebtoon(webtoon);

src/modules/webtoon/webtoon/webtoon-downloader.service.ts

@@ -2,31 +2,45 @@ import EpisodeModel from "./models/models/episode.model";
 import EpisodeDataModel from "./models/models/episode-data.model";
 import WebtoonDataModel from "./models/models/webtoon-data.model";
 import WebtoonModel from "./models/models/webtoon.model";
-import {Injectable} from "@nestjs/common";
+import {Injectable, Logger} from "@nestjs/common";
 import {MiscService} from "../../misc/misc.service";
 
 @Injectable()
 export class WebtoonDownloaderService{
 
+    private readonly logger = new Logger(WebtoonDownloaderService.name);
+
     constructor(
         private readonly miscService: MiscService,
     ){}
 
-    async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise<EpisodeDataModel>{
-        console.log(`Downloading episode ${episode.number}...`);
+    async downloadEpisode(episode: EpisodeModel, imageUrls: string[]): Promise<EpisodeDataModel> {
+        this.logger.debug(`Downloading episode ${episode.number}...`);
+        const startTime = Date.now();
         const thumbnail: Buffer = await this.miscService.downloadImage(episode.thumbnail);
         const conversionPromises: Promise<Buffer>[] = [];
-        for (let i = 0; i < imageUrls.length; i++){
-            console.log(`Downloading image ${i + 1}/${imageUrls.length}...`);
+        let downloadedCount = 0;
+
+        const interval = setInterval(() => {
+            const elapsedSeconds = (Date.now() - startTime) / 1000;
+            const imagesPerSecond = downloadedCount / elapsedSeconds;
+            this.logger.debug(`Downloading ${downloadedCount} of ${imageUrls.length} images (${(imagesPerSecond).toFixed(2)} images/s)...`);
+        }, 1000);
+
+        for (let i = 0; i < imageUrls.length; i++) {
             const url = imageUrls[i];
             const image = await this.miscService.downloadImage(url, episode.link);
             conversionPromises.push(this.miscService.convertImageToWebp(image));
+            downloadedCount++;
             await new Promise(resolve => setTimeout(resolve, this.miscService.randomInt(50, 200)));
         }
+
+        clearInterval(interval);
+        this.logger.debug(`Downloaded ${downloadedCount}/${imageUrls.length} images in ${((Date.now() - startTime) / 1000).toFixed(2)} seconds.`);
+
         // Convert all images to webp
-        console.log("Converting images to webp...");
         const convertedImages: Buffer[] = await Promise.all(conversionPromises);
-        console.log(`Download complete for episode ${episode.number}!`);
+        this.logger.debug(`Download complete for episode ${episode.number}!`);
         return {
             thumbnail,
             images: convertedImages

src/modules/webtoon/webtoon/webtoon-parser.service.ts

@@ -24,20 +24,20 @@ export class WebtoonParserService{
     async loadCache(): Promise<void>{
         // Load existing cache
         if(fs.existsSync("./.cache/webtoons.json")){
-            this.logger.log("Loading webtoon list from cache...");
+            this.logger.debug("Loading webtoon list from cache...");
             this.webtoons = JSON.parse(fs.readFileSync("./.cache/webtoons.json").toString());
             const webtoonCount = Object.values(this.webtoons).reduce((acc, val: any) => acc + val.length, 0);
-            this.logger.log(`Loaded ${webtoonCount} webtoons!`);
+            this.logger.debug(`Loaded ${webtoonCount} webtoons!`);
             return;
         }
-        this.logger.log("Loading webtoon list...");
+        this.logger.debug("Loading webtoon list...");
         // Generate and save cache
         for (const language of Object.values(WebtoonLanguages)){
-            this.logger.log(`Loading webtoons for language: ${language}`);
+            this.logger.debug(`Loading webtoons for language: ${language}`);
             this.webtoons[language] = await this.getWebtoonsFromLanguage(language);
         }
         const webtoonCount = Object.values(this.webtoons).reduce((acc, val: any) => acc + val.length, 0);
-        this.logger.log(`Loaded ${webtoonCount} webtoons!`);
+        this.logger.debug(`Loaded ${webtoonCount} webtoons!`);
         // Save cache
         fs.mkdirSync("./.cache", {recursive: true});
         fs.writeFileSync("./.cache/webtoons.json", JSON.stringify(this.webtoons, null, 2));