Merge pull request #60 from Open-Webtoon-Reader/feature/s3-support

Merge S3 support to Main

.env.example

@@ -13,3 +13,16 @@ PREFIX="api/v1/"
 
 # Security
 ADMIN_KEY="admin"
+
+# S3
+FILESYSTEM=local#local or s3
+S3_ENDPOINT=endpoint
+S3_PORT=9000
+S3_USE_SSL=true
+S3_REGION=region
+S3_ACCESS_KEY=access_key_id
+S3_SECRET_KEY=secret_access_key
+S3_BUCKET_NAME=bucket_name
+
+# Migration
+S3_MIGRATION_BATCH_SIZE=20

package.json

@@ -36,6 +36,7 @@
     "fastify": "4.28.0",
     "jsdom": "^24.1.0",
     "jszip": "^3.10.1",
+    "minio": "^8.0.1",
     "prisma": "^5.16.1",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",

src/modules/file/file.module.ts

@@ -0,0 +1,11 @@
+import {Module} from "@nestjs/common";
+import {FileService} from "./file.service";
+import {MiscModule} from "../misc/misc.module";
+
+@Module({
+    exports: [FileService],
+    imports: [MiscModule],
+    providers: [FileService]
+})
+export class FileModule{}
+

src/modules/file/file.service.ts

@@ -0,0 +1,64 @@
+import {Injectable, NotFoundException} from "@nestjs/common";
+import {ConfigService} from "@nestjs/config";
+import {PrismaService} from "../misc/prisma.service";
+import {MiscService} from "../misc/misc.service";
+import {Saver} from "./saver/saver";
+import {S3Saver} from "./saver/s3.saver";
+import {FileSaver} from "./saver/file.saver";
+
+@Injectable()
+export class FileService{
+
+    private readonly saver: Saver;
+
+    constructor(
+        private readonly configService: ConfigService,
+        private readonly prismaService: PrismaService,
+        private readonly cipherService: MiscService,
+    ){
+        if(this.configService.get("FILESYSTEM") === "s3")
+            this.saver = this.getS3Saver();
+        else
+            this.saver = this.getFileSaver();
+    }
+
+    getS3Saver(){
+        return new S3Saver(
+            this.configService.get("S3_ENDPOINT"),
+            this.configService.get("S3_PORT"),
+            this.configService.get("S3_USE_SSL") === "true",
+            this.configService.get("S3_REGION"),
+            this.configService.get("S3_ACCESS_KEY"),
+            this.configService.get("S3_SECRET_KEY"),
+            this.configService.get("S3_BUCKET_NAME")
+        );
+    }
+
+    getFileSaver(){
+        return new FileSaver("images");
+    }
+
+    async saveImage(data: Buffer): Promise<string>{
+        const sum = this.cipherService.getSum(data);
+        await this.saver.saveFile(data, sum);
+        return sum;
+    }
+
+    async loadImage(sum: string): Promise<Buffer>{
+        try{
+            const stream = await this.saver.getFile(sum);
+            return await new Promise<Buffer>((resolve, reject) => {
+                const chunks: Buffer[] = [];
+                stream.on("data", (chunk: Buffer) => chunks.push(chunk));
+                stream.on("end", () => resolve(Buffer.concat(chunks)));
+                stream.on("error", reject);
+            });
+        }catch(e){
+            throw new NotFoundException("Image not found");
+        }
+    }
+
+    async removeImage(sum: string): Promise<void>{
+        await this.saver.removeFile(sum);
+    }
+}

src/modules/file/saver/file.saver.ts

@@ -0,0 +1,30 @@
+import {Saver} from "./saver";
+import * as fs from "fs";
+import {createReadStream, ReadStream} from "fs";
+
+export class FileSaver implements Saver{
+
+    private readonly uploadFolderName: string;
+
+    constructor(uploadFolderName: string){
+        this.uploadFolderName = uploadFolderName;
+    }
+
+    async saveFile(data: Buffer, sum: string): Promise<void>{
+        const path = `./${this.uploadFolderName}/${sum.substring(0, 2)}`;
+        if(!fs.existsSync(path)){
+            fs.mkdirSync(path, {
+                recursive: true
+            });
+        }
+        fs.writeFileSync(`${path}/${sum}.webp`, data);
+    }
+
+    async getFile(sum: string): Promise<ReadStream>{
+        return createReadStream(`./${this.uploadFolderName}/${sum.substring(0, 2)}/${sum}.webp`);
+    }
+
+    async removeFile(sum: string): Promise<void>{
+        fs.unlinkSync(`./${this.uploadFolderName}/${sum.substring(0, 2)}/${sum}.webp`);
+    }
+}

src/modules/file/saver/s3.saver.ts

@@ -0,0 +1,70 @@
+import {Readable} from "stream";
+import * as Minio from "minio";
+import {Saver} from "./saver";
+import {ReadStream} from "fs";
+
+export class S3Saver implements Saver{
+
+    private readonly s3Client: Minio.Client;
+    private readonly bucketName: string;
+    private bucketExists: boolean = false;
+
+    constructor(endpoint: string, port: number, useSSL: boolean, region: string, accessKey: string, secretKey: string, bucketName: string){
+        this.s3Client = new Minio.Client({
+            endPoint: endpoint,
+            port,
+            useSSL,
+            accessKey,
+            secretKey,
+            region,
+        });
+        this.bucketName = bucketName;
+    }
+
+    public async createBucketIfNotExists(): Promise<void>{
+        if(this.bucketExists)
+            return;
+        try{
+            const bucketExists = await this.s3Client.bucketExists(this.bucketName);
+            if(!bucketExists)
+                await this.s3Client.makeBucket(this.bucketName);
+        }catch (e){
+            console.log(e);
+        }
+        this.bucketExists = true;
+    }
+
+    async saveFile(data: Buffer, sum: string): Promise<void>{
+        await this.createBucketIfNotExists();
+        await this.s3Client.putObject(this.bucketName, `${sum.substring(0, 2)}/${sum}.webp`, data);
+    }
+    async getFile(sum: string): Promise<ReadStream>{
+        await this.createBucketIfNotExists();
+        const readable: Readable = await this.s3Client.getObject(this.bucketName, `${sum.substring(0, 2)}/${sum}.webp`);
+        return readable as ReadStream;
+    }
+
+    async removeFile(sum: string): Promise<void>{
+        await this.createBucketIfNotExists();
+        await this.s3Client.removeObject(this.bucketName, `${sum.substring(0, 2)}/${sum}.webp`);
+    }
+
+    async clearBucket(){
+        const objectsList = [];
+
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-expect-error
+        const objectsStream = this.s3Client.listObjects(this.bucketName, "", true, {IncludeVersion: true});
+        objectsStream.on("data", function(obj){
+            objectsList.push(obj);
+        });
+        objectsStream.on("error", function(e){
+            return console.log(e);
+        });
+        objectsStream.on("end", async() => {
+            console.log(`Clearing ${objectsList.length} objects from the bucket`);
+            await this.s3Client.removeObjects(this.bucketName, objectsList);
+            console.log("Bucket cleared");
+        });
+    }
+}

src/modules/file/saver/saver.ts

@@ -0,0 +1,7 @@
+import {ReadStream} from "fs";
+
+export interface Saver{
+    saveFile(data: Buffer, fileName: string): Promise<void>;
+    getFile(fileName: string): Promise<ReadStream>;
+    removeFile(fileName: string): Promise<void>;
+}

src/modules/webtoon/image/image.controller.ts

@@ -3,11 +3,12 @@ import {ApiResponse, ApiTags} from "@nestjs/swagger";
 import {WebtoonDatabaseService} from "../webtoon/webtoon-database.service";
 import {HttpStatusCode} from "axios";
 import {ImageSumDto} from "./models/dto/image-sum.dto";
-import {SkipThrottle} from "@nestjs/throttler";
+import {Throttle} from "@nestjs/throttler";
 
 
 @Controller("image")
 @ApiTags("Image")
+@Throttle({default: {limit: 400, ttl: 60000}})
 export class ImageController{
 
     constructor(
@@ -16,11 +17,10 @@ export class ImageController{
 
     @Get(":sum")
     @Header("Content-Type", "image/webp")
-    @Header("Cache-Control", "public, max-age=2592000")
+    @Header("Cache-Control", "public, max-age=604800000")
     @ApiResponse({status: HttpStatusCode.Ok, description: "Get image"})
     @ApiResponse({status: HttpStatusCode.NotFound, description: "Not found"})
     @ApiResponse({status: HttpStatusCode.BadRequest, description: "Invalid sha256 sum"})
-    @SkipThrottle()
     getImage(@Param() imageSumDto: ImageSumDto){
         const regex = new RegExp("^[a-f0-9]{64}$");
         if(!regex.test(imageSumDto.sum))

src/modules/webtoon/migration/migration.controller.ts

@@ -42,4 +42,22 @@ export class MigrationController{
     async getDatabase(@Res({passthrough: true}) _: Response): Promise<StreamableFile>{
         return new StreamableFile(await this.migrationService.getDatabase());
     }
+
+    @Post("to/s3")
+    @ApiBearerAuth()
+    async migrateToS3(){
+        this.migrationService.migrateToS3();
+    }
+
+    @Post("to/local")
+    @ApiBearerAuth()
+    async migrateToLocal(){
+        this.migrationService.migrateToLocal();
+    }
+
+    // @Delete("s3")
+    // @ApiBearerAuth()
+    // async removeS3(){
+    //     this.migrationService.clearS3();
+    // }
 }

src/modules/webtoon/migration/migration.module.ts

@@ -3,10 +3,11 @@ import {MigrationController} from "./migration.controller";
 import {MigrationService} from "./migration.service";
 import {WebtoonModule} from "../webtoon/webtoon.module";
 import {MiscModule} from "../../misc/misc.module";
+import {FileModule} from "../../file/file.module";
 
 @Module({
     providers: [MigrationService],
     controllers: [MigrationController],
-    imports: [WebtoonModule, MiscModule]
+    imports: [WebtoonModule, MiscModule, FileModule]
 })
 export class MigrationModule{}

src/modules/webtoon/migration/migration.service.ts

@@ -8,6 +8,8 @@ import axios from "axios";
 import {PrismaService} from "../../misc/prisma.service";
 import * as fs from "node:fs";
 import * as https from "node:https";
+import {FileService} from "../../file/file.service";
+import {ConfigService} from "@nestjs/config";
 
 @Injectable()
 export class MigrationService{
@@ -16,11 +18,13 @@ export class MigrationService{
 
     constructor(
         private readonly webtoonDatabaseService: WebtoonDatabaseService,
-        private readonly prismaService: PrismaService
+        private readonly prismaService: PrismaService,
+        private readonly fileService: FileService,
+        private readonly configService: ConfigService,
     ){}
 
     async migrateFrom(url: string, adminKey: string){
-        this.logger.debug(`Start migration from ${url}`)
+        this.logger.debug(`Start migration from ${url}`);
         // Get migration infos using axios from the url
         const response = await axios.get(url + "/api/v1/migration/infos", {
             headers: {
@@ -98,4 +102,60 @@ export class MigrationService{
             throw error;
         }
     }
+
+    async migrateToS3(){
+        const s3Saver = this.fileService.getS3Saver();
+        const dbImageBatchSize = 10000;
+        const s3BatchSize = parseInt(this.configService.get("S3_BATCH_SIZE"));
+        const imageCount = await this.prismaService.images.count();
+        await s3Saver.createBucketIfNotExists();
+        for(let i = 0; i < imageCount; i += dbImageBatchSize){
+            this.logger.debug(`Migrating images from ${i} to ${i + dbImageBatchSize}`);
+            const images = await this.prismaService.images.findMany({
+                skip: i,
+                take: dbImageBatchSize,
+                select: {
+                    id: true,
+                    sum: true
+                }
+            });
+            const imageSums = images.map(image => image.sum);
+            for(let j = 0; j < imageSums.length; j += s3BatchSize){
+                this.logger.debug(`Uploading images from ${j} to ${j + s3BatchSize}`);
+                const batch = imageSums.slice(j, j + s3BatchSize);
+                await Promise.all(batch.map(async(sum) => s3Saver.saveFile(await this.fileService.loadImage(sum), sum)));
+            }
+        }
+        this.logger.debug("Migration to S3 completed!");
+    }
+
+    async migrateToLocal(){
+        const fileSaver = this.fileService.getFileSaver();
+        const dbImageBatchSize = 10000;
+        const localBatchSize = parseInt(this.configService.get("S3_BATCH_SIZE"));
+        const imageCount = await this.prismaService.images.count();
+        for(let i = 0; i < imageCount; i += dbImageBatchSize){
+            this.logger.debug(`Migrating images from ${i} to ${i + dbImageBatchSize}`);
+            const images = await this.prismaService.images.findMany({
+                skip: i,
+                take: dbImageBatchSize,
+                select: {
+                    id: true,
+                    sum: true
+                }
+            });
+            const imageSums = images.map(image => image.sum);
+            for(let j = 0; j < imageSums.length; j += localBatchSize){
+                this.logger.debug(`Saving images from ${j} to ${j + localBatchSize}`);
+                const batch = imageSums.slice(j, j + localBatchSize);
+                await Promise.all(batch.map(async(sum) => fileSaver.saveFile(await this.fileService.loadImage(sum), sum)));
+            }
+        }
+        this.logger.debug("Migration to local completed!");
+    }
+
+    async clearS3(): Promise<void>{
+        const s3Saver = this.fileService.getS3Saver();
+        await s3Saver.clearBucket();
+    }
 }

src/modules/webtoon/update/update.service.ts

@@ -46,7 +46,7 @@ export class UpdateService{
                 this.logger.debug(`Updating thumbnail for webtoon ${webtoon.title} (${webtoon.language})`);
                 const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language);
                 const thumbnail: Buffer = await this.miscService.convertWebtoonThumbnail(cachedWebtoon.thumbnail);
-                const sum: string = this.webtoonDatabaseService.saveImage(thumbnail);
+                const sum: string = await this.webtoonDatabaseService.saveImage(thumbnail);
                 // Check if thumbnail already exists
                 let dbThumbnail = await tx.images.findFirst({
                     where: {