diff --git a/.env.example b/.env.example index 28dabf9..c154746 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,16 @@ PREFIX="api/v1/" # Security ADMIN_KEY="admin" + +# S3 +FILESYSTEM=local#local or s3 +S3_ENDPOINT=endpoint +S3_PORT=9000 +S3_USE_SSL=true +S3_REGION=region +S3_ACCESS_KEY=access_key_id +S3_SECRET_KEY=secret_access_key +S3_BUCKET_NAME=bucket_name + +# Migration +S3_MIGRATION_BATCH_SIZE=20 diff --git a/package.json b/package.json index 2d2c092..d4e14cb 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "fastify": "4.28.0", "jsdom": "^24.1.0", "jszip": "^3.10.1", + "minio": "^8.0.1", "prisma": "^5.16.1", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d7f933f..b8019f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: jszip: specifier: ^3.10.1 version: 3.10.1 + minio: + specifier: ^8.0.1 + version: 8.0.1 prisma: specifier: ^5.16.1 version: 5.16.1 @@ -1013,6 +1016,9 @@ packages: '@xtuc/long@4.2.2': resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + '@zxing/text-encoding@0.9.0': + resolution: {integrity: sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==} + abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -1128,6 +1134,9 @@ packages: asap@2.0.6: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} + async@3.2.6: + resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} + asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -1135,6 +1144,10 @@ packages: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} + available-typed-arrays@1.0.7: + resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} + engines: {node: '>= 0.4'} + avvio@8.3.2: resolution: {integrity: sha512-st8e519GWHa/azv8S87mcJvZs4WsgTBjOw/Ih1CP6u+8SZvcOeAYNG6JbsIrAUUJJ7JfmrnOkR8ipDS+u9SIRQ==} @@ -1179,6 +1192,9 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + block-stream2@2.1.0: + resolution: {integrity: sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -1189,6 +1205,9 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + browser-or-node@2.1.1: + resolution: {integrity: sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==} + browserslist@4.23.1: resolution: {integrity: sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -1201,6 +1220,10 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -1409,6 +1432,10 @@ packages: decimal.js@10.4.3: resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} + decode-uri-component@0.2.2: + resolution: {integrity: sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==} + engines: {node: '>=0.10'} + dedent@1.5.3: resolution: {integrity: sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==} peerDependencies: @@ -1605,6 +1632,9 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} + eventemitter3@5.0.1: + resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -1663,6 +1693,10 @@ packages: fast-uri@2.4.0: resolution: {integrity: sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA==} + fast-xml-parser@4.5.0: + resolution: {integrity: sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==} + hasBin: true + fastify-plugin@4.5.1: resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} @@ -1691,6 +1725,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + filter-obj@1.1.0: + resolution: {integrity: sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==} + engines: {node: '>=0.10.0'} + find-my-way@8.2.0: resolution: {integrity: sha512-HdWXgFYc6b1BJcOBDBwjqWuHJj1WYiqrxSh25qtU4DabpMFdj/gSunNBQb83t+8Zt67D7CXEzJWTkxaShMTMOA==} engines: {node: '>=14'} @@ -1719,6 +1757,9 @@ packages: debug: optional: true + for-each@0.3.3: + resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} + foreground-child@3.2.1: resolution: {integrity: sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==} engines: {node: '>=14'} @@ -1852,6 +1893,10 @@ packages: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} engines: {node: '>= 0.4'} + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -1941,6 +1986,14 @@ packages: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} + ipaddr.js@2.2.0: + resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + engines: {node: '>= 10'} + + is-arguments@1.1.1: + resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==} + engines: {node: '>= 0.4'} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -1951,6 +2004,10 @@ packages: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} + is-callable@1.2.7: + resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} + engines: {node: '>= 0.4'} + is-core-module@2.14.0: resolution: {integrity: sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==} engines: {node: '>= 0.4'} @@ -1967,6 +2024,10 @@ packages: resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==} engines: {node: '>=6'} + is-generator-function@1.0.10: + resolution: {integrity: sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==} + engines: {node: '>= 0.4'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -1990,6 +2051,10 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} + is-typed-array@1.1.13: + resolution: {integrity: sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==} + engines: {node: '>= 0.4'} + is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -2365,6 +2430,10 @@ packages: minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minio@8.0.1: + resolution: {integrity: sha512-FzDO6yGnqLtm8sp3mXafWtiRUOslJSSg/aI0v9YbN5vjw5KLoODKAROCyi766NIvTSxcfHBrbhCSGk1A+MOzDg==} + engines: {node: ^16 || ^18 || >=20} + minipass@4.2.8: resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} engines: {node: '>=8'} @@ -2553,6 +2622,10 @@ packages: resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} engines: {node: '>=4'} + possible-typed-array-names@1.0.0: + resolution: {integrity: sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==} + engines: {node: '>= 0.4'} + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -2610,6 +2683,10 @@ packages: resolution: {integrity: sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==} engines: {node: '>=0.6'} + query-string@7.1.3: + resolution: {integrity: sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==} + engines: {node: '>=6'} + querystringify@2.2.0: resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} @@ -2747,6 +2824,9 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + sax@1.4.1: + resolution: {integrity: sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==} + saxes@6.0.0: resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} engines: {node: '>=v12.22.7'} @@ -2838,6 +2918,10 @@ packages: resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==} engines: {node: '>= 8'} + split-on-first@1.1.0: + resolution: {integrity: sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==} + engines: {node: '>=6'} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -2853,6 +2937,16 @@ packages: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} + stream-chain@2.2.5: + resolution: {integrity: sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==} + + stream-json@1.8.0: + resolution: {integrity: sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==} + + strict-uri-encode@2.0.0: + resolution: {integrity: sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==} + engines: {node: '>=4'} + string-length@4.0.2: resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==} engines: {node: '>=10'} @@ -2895,6 +2989,9 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strnum@1.0.5: + resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==} + superagent@9.0.2: resolution: {integrity: sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==} engines: {node: '>=14.18.0'} @@ -2971,6 +3068,9 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} + through2@4.0.2: + resolution: {integrity: sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==} + through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} @@ -3130,6 +3230,9 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + util@0.12.5: + resolution: {integrity: sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==} + uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true @@ -3159,6 +3262,9 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + web-encoding@1.1.5: + resolution: {integrity: sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -3199,6 +3305,10 @@ packages: whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which-typed-array@1.1.15: + resolution: {integrity: sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==} + engines: {node: '>= 0.4'} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -3243,6 +3353,14 @@ packages: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} + xml2js@0.5.0: + resolution: {integrity: sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==} + engines: {node: '>=4.0.0'} + + xmlbuilder@11.0.1: + resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} + engines: {node: '>=4.0'} + xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} @@ -4378,6 +4496,9 @@ snapshots: '@xtuc/long@4.2.2': {} + '@zxing/text-encoding@0.9.0': + optional: true + abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -4482,10 +4603,16 @@ snapshots: asap@2.0.6: {} + async@3.2.6: {} + asynckit@0.4.0: {} atomic-sleep@1.0.0: {} + available-typed-arrays@1.0.7: + dependencies: + possible-typed-array-names: 1.0.0 + avvio@8.3.2: dependencies: '@fastify/error': 3.4.1 @@ -4563,6 +4690,10 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + block-stream2@2.1.0: + dependencies: + readable-stream: 3.6.2 + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 @@ -4576,6 +4707,8 @@ snapshots: dependencies: fill-range: 7.1.1 + browser-or-node@2.1.1: {} + browserslist@4.23.1: dependencies: caniuse-lite: 1.0.30001639 @@ -4591,6 +4724,8 @@ snapshots: dependencies: node-int64: 0.4.0 + buffer-crc32@1.0.0: {} + buffer-from@1.1.2: {} buffer@5.7.1: @@ -4798,6 +4933,8 @@ snapshots: decimal.js@10.4.3: {} + decode-uri-component@0.2.2: {} + dedent@1.5.3: {} deep-is@0.1.4: {} @@ -4977,6 +5114,8 @@ snapshots: event-target-shim@5.0.1: {} + eventemitter3@5.0.1: {} + events@3.3.0: {} execa@5.1.1: @@ -5047,6 +5186,10 @@ snapshots: fast-uri@2.4.0: {} + fast-xml-parser@4.5.0: + dependencies: + strnum: 1.0.5 + fastify-plugin@4.5.1: {} fastify@4.28.0: @@ -5093,6 +5236,8 @@ snapshots: dependencies: to-regex-range: 5.0.1 + filter-obj@1.1.0: {} + find-my-way@8.2.0: dependencies: fast-deep-equal: 3.1.3 @@ -5119,6 +5264,10 @@ snapshots: follow-redirects@1.15.6: {} + for-each@0.3.3: + dependencies: + is-callable: 1.2.7 + foreground-child@3.2.1: dependencies: cross-spawn: 7.0.3 @@ -5266,6 +5415,10 @@ snapshots: has-symbols@1.0.3: {} + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.0.3 + hasown@2.0.2: dependencies: function-bind: 1.1.2 @@ -5377,6 +5530,13 @@ snapshots: ipaddr.js@1.9.1: {} + ipaddr.js@2.2.0: {} + + is-arguments@1.1.1: + dependencies: + call-bind: 1.0.7 + has-tostringtag: 1.0.2 + is-arrayish@0.2.1: {} is-arrayish@0.3.2: {} @@ -5385,6 +5545,8 @@ snapshots: dependencies: binary-extensions: 2.3.0 + is-callable@1.2.7: {} + is-core-module@2.14.0: dependencies: hasown: 2.0.2 @@ -5395,6 +5557,10 @@ snapshots: is-generator-fn@2.1.0: {} + is-generator-function@1.0.10: + dependencies: + has-tostringtag: 1.0.2 + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -5409,6 +5575,10 @@ snapshots: is-stream@2.0.1: {} + is-typed-array@1.1.13: + dependencies: + which-typed-array: 1.1.15 + is-unicode-supported@0.1.0: {} is-unicode-supported@1.3.0: {} @@ -5972,6 +6142,23 @@ snapshots: minimist@1.2.8: {} + minio@8.0.1: + dependencies: + async: 3.2.6 + block-stream2: 2.1.0 + browser-or-node: 2.1.1 + buffer-crc32: 1.0.0 + eventemitter3: 5.0.1 + fast-xml-parser: 4.5.0 + ipaddr.js: 2.2.0 + lodash: 4.17.21 + mime-types: 2.1.35 + query-string: 7.1.3 + stream-json: 1.8.0 + through2: 4.0.2 + web-encoding: 1.1.5 + xml2js: 0.5.0 + minipass@4.2.8: {} minipass@7.1.2: {} @@ -6140,6 +6327,8 @@ snapshots: pluralize@8.0.0: {} + possible-typed-array-names@1.0.0: {} + prelude-ls@1.2.1: {} prettier-linter-helpers@1.0.0: @@ -6186,6 +6375,13 @@ snapshots: dependencies: side-channel: 1.0.6 + query-string@7.1.3: + dependencies: + decode-uri-component: 0.2.2 + filter-obj: 1.1.0 + split-on-first: 1.1.0 + strict-uri-encode: 2.0.0 + querystringify@2.2.0: {} queue-microtask@1.2.3: {} @@ -6305,6 +6501,8 @@ snapshots: safer-buffer@2.1.2: {} + sax@1.4.1: {} + saxes@6.0.0: dependencies: xmlchars: 2.2.0 @@ -6415,6 +6613,8 @@ snapshots: source-map@0.7.4: {} + split-on-first@1.1.0: {} + split2@4.2.0: {} sprintf-js@1.0.3: {} @@ -6425,6 +6625,14 @@ snapshots: statuses@2.0.1: {} + stream-chain@2.2.5: {} + + stream-json@1.8.0: + dependencies: + stream-chain: 2.2.5 + + strict-uri-encode@2.0.0: {} + string-length@4.0.2: dependencies: char-regex: 1.0.2 @@ -6466,6 +6674,8 @@ snapshots: strip-json-comments@3.1.1: {} + strnum@1.0.5: {} + superagent@9.0.2: dependencies: component-emitter: 1.3.1 @@ -6544,6 +6754,10 @@ snapshots: dependencies: real-require: 0.2.0 + through2@4.0.2: + dependencies: + readable-stream: 3.6.2 + through@2.3.8: {} tmp@0.0.33: @@ -6682,6 +6896,14 @@ snapshots: util-deprecate@1.0.2: {} + util@0.12.5: + dependencies: + inherits: 2.0.4 + is-arguments: 1.1.1 + is-generator-function: 1.0.10 + is-typed-array: 1.1.13 + which-typed-array: 1.1.15 + uuid@9.0.1: {} v8-compile-cache-lib@3.0.1: {} @@ -6711,6 +6933,12 @@ snapshots: dependencies: defaults: 1.0.4 + web-encoding@1.1.5: + dependencies: + util: 0.12.5 + optionalDependencies: + '@zxing/text-encoding': 0.9.0 + webidl-conversions@3.0.1: {} webidl-conversions@7.0.0: {} @@ -6766,6 +6994,14 @@ snapshots: tr46: 0.0.3 webidl-conversions: 3.0.1 + which-typed-array@1.1.15: + dependencies: + available-typed-arrays: 1.0.7 + call-bind: 1.0.7 + for-each: 0.3.3 + gopd: 1.0.1 + has-tostringtag: 1.0.2 + which@2.0.2: dependencies: isexe: 2.0.0 @@ -6801,6 +7037,13 @@ snapshots: xml-name-validator@5.0.0: {} + xml2js@0.5.0: + dependencies: + sax: 1.4.1 + xmlbuilder: 11.0.1 + + xmlbuilder@11.0.1: {} + xmlchars@2.2.0: {} y18n@5.0.8: {} diff --git a/src/modules/file/file.module.ts b/src/modules/file/file.module.ts new file mode 100644 index 0000000..559b630 --- /dev/null +++ b/src/modules/file/file.module.ts @@ -0,0 +1,11 @@ +import {Module} from "@nestjs/common"; +import {FileService} from "./file.service"; +import {MiscModule} from "../misc/misc.module"; + +@Module({ + exports: [FileService], + imports: [MiscModule], + providers: [FileService] +}) +export class FileModule{} + diff --git a/src/modules/file/file.service.ts b/src/modules/file/file.service.ts new file mode 100644 index 0000000..87b2177 --- /dev/null +++ b/src/modules/file/file.service.ts @@ -0,0 +1,64 @@ +import {Injectable, NotFoundException} from "@nestjs/common"; +import {ConfigService} from "@nestjs/config"; +import {PrismaService} from "../misc/prisma.service"; +import {MiscService} from "../misc/misc.service"; +import {Saver} from "./saver/saver"; +import {S3Saver} from "./saver/s3.saver"; +import {FileSaver} from "./saver/file.saver"; + +@Injectable() +export class FileService{ + + private readonly saver: Saver; + + constructor( + private readonly configService: ConfigService, + private readonly prismaService: PrismaService, + private readonly cipherService: MiscService, + ){ + if(this.configService.get("FILESYSTEM") === "s3") + this.saver = this.getS3Saver(); + else + this.saver = this.getFileSaver(); + } + + getS3Saver(){ + return new S3Saver( + this.configService.get("S3_ENDPOINT"), + this.configService.get("S3_PORT"), + this.configService.get("S3_USE_SSL") === "true", + this.configService.get("S3_REGION"), + this.configService.get("S3_ACCESS_KEY"), + this.configService.get("S3_SECRET_KEY"), + this.configService.get("S3_BUCKET_NAME") + ); + } + + getFileSaver(){ + return new FileSaver("images"); + } + + async saveImage(data: Buffer): Promise{ + const sum = this.cipherService.getSum(data); + await this.saver.saveFile(data, sum); + return sum; + } + + async loadImage(sum: string): Promise{ + try{ + const stream = await this.saver.getFile(sum); + return await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("error", reject); + }); + }catch(e){ + throw new NotFoundException("Image not found"); + } + } + + async removeImage(sum: string): Promise{ + await this.saver.removeFile(sum); + } +} diff --git a/src/modules/file/saver/file.saver.ts b/src/modules/file/saver/file.saver.ts new file mode 100644 index 0000000..2e4e87e --- /dev/null +++ b/src/modules/file/saver/file.saver.ts @@ -0,0 +1,30 @@ +import {Saver} from "./saver"; +import * as fs from "fs"; +import {createReadStream, ReadStream} from "fs"; + +export class FileSaver implements Saver{ + + private readonly uploadFolderName: string; + + constructor(uploadFolderName: string){ + this.uploadFolderName = uploadFolderName; + } + + async saveFile(data: Buffer, sum: string): Promise{ + const path = `./${this.uploadFolderName}/${sum.substring(0, 2)}`; + if(!fs.existsSync(path)){ + fs.mkdirSync(path, { + recursive: true + }); + } + fs.writeFileSync(`${path}/${sum}.webp`, data); + } + + async getFile(sum: string): Promise{ + return createReadStream(`./${this.uploadFolderName}/${sum.substring(0, 2)}/${sum}.webp`); + } + + async removeFile(sum: string): Promise{ + fs.unlinkSync(`./${this.uploadFolderName}/${sum.substring(0, 2)}/${sum}.webp`); + } +} diff --git a/src/modules/file/saver/s3.saver.ts b/src/modules/file/saver/s3.saver.ts new file mode 100644 index 0000000..f4ce6ff --- /dev/null +++ b/src/modules/file/saver/s3.saver.ts @@ -0,0 +1,70 @@ +import {Readable} from "stream"; +import * as Minio from "minio"; +import {Saver} from "./saver"; +import {ReadStream} from "fs"; + +export class S3Saver implements Saver{ + + private readonly s3Client: Minio.Client; + private readonly bucketName: string; + private bucketExists: boolean = false; + + constructor(endpoint: string, port: number, useSSL: boolean, region: string, accessKey: string, secretKey: string, bucketName: string){ + this.s3Client = new Minio.Client({ + endPoint: endpoint, + port, + useSSL, + accessKey, + secretKey, + region, + }); + this.bucketName = bucketName; + } + + public async createBucketIfNotExists(): Promise{ + if(this.bucketExists) + return; + try{ + const bucketExists = await this.s3Client.bucketExists(this.bucketName); + if(!bucketExists) + await this.s3Client.makeBucket(this.bucketName); + }catch (e){ + console.log(e); + } + this.bucketExists = true; + } + + async saveFile(data: Buffer, sum: string): Promise{ + await this.createBucketIfNotExists(); + await this.s3Client.putObject(this.bucketName, `${sum.substring(0, 2)}/${sum}.webp`, data); + } + async getFile(sum: string): Promise{ + await this.createBucketIfNotExists(); + const readable: Readable = await this.s3Client.getObject(this.bucketName, `${sum.substring(0, 2)}/${sum}.webp`); + return readable as ReadStream; + } + + async removeFile(sum: string): Promise{ + await this.createBucketIfNotExists(); + await this.s3Client.removeObject(this.bucketName, `${sum.substring(0, 2)}/${sum}.webp`); + } + + async clearBucket(){ + const objectsList = []; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + const objectsStream = this.s3Client.listObjects(this.bucketName, "", true, {IncludeVersion: true}); + objectsStream.on("data", function(obj){ + objectsList.push(obj); + }); + objectsStream.on("error", function(e){ + return console.log(e); + }); + objectsStream.on("end", async() => { + console.log(`Clearing ${objectsList.length} objects from the bucket`); + await this.s3Client.removeObjects(this.bucketName, objectsList); + console.log("Bucket cleared"); + }); + } +} diff --git a/src/modules/file/saver/saver.ts b/src/modules/file/saver/saver.ts new file mode 100644 index 0000000..bbb401a --- /dev/null +++ b/src/modules/file/saver/saver.ts @@ -0,0 +1,7 @@ +import {ReadStream} from "fs"; + +export interface Saver{ + saveFile(data: Buffer, fileName: string): Promise; + getFile(fileName: string): Promise; + removeFile(fileName: string): Promise; +} diff --git a/src/modules/webtoon/image/image.controller.ts b/src/modules/webtoon/image/image.controller.ts index ac48295..45340f7 100644 --- a/src/modules/webtoon/image/image.controller.ts +++ b/src/modules/webtoon/image/image.controller.ts @@ -3,11 +3,12 @@ import {ApiResponse, ApiTags} from "@nestjs/swagger"; import {WebtoonDatabaseService} from "../webtoon/webtoon-database.service"; import {HttpStatusCode} from "axios"; import {ImageSumDto} from "./models/dto/image-sum.dto"; -import {SkipThrottle} from "@nestjs/throttler"; +import {Throttle} from "@nestjs/throttler"; @Controller("image") @ApiTags("Image") +@Throttle({default: {limit: 400, ttl: 60000}}) export class ImageController{ constructor( @@ -16,11 +17,10 @@ export class ImageController{ @Get(":sum") @Header("Content-Type", "image/webp") - @Header("Cache-Control", "public, max-age=2592000") + @Header("Cache-Control", "public, max-age=604800000") @ApiResponse({status: HttpStatusCode.Ok, description: "Get image"}) @ApiResponse({status: HttpStatusCode.NotFound, description: "Not found"}) @ApiResponse({status: HttpStatusCode.BadRequest, description: "Invalid sha256 sum"}) - @SkipThrottle() getImage(@Param() imageSumDto: ImageSumDto){ const regex = new RegExp("^[a-f0-9]{64}$"); if(!regex.test(imageSumDto.sum)) diff --git a/src/modules/webtoon/migration/migration.controller.ts b/src/modules/webtoon/migration/migration.controller.ts index 9a63b6c..1ce3c73 100644 --- a/src/modules/webtoon/migration/migration.controller.ts +++ b/src/modules/webtoon/migration/migration.controller.ts @@ -42,4 +42,22 @@ export class MigrationController{ async getDatabase(@Res({passthrough: true}) _: Response): Promise{ return new StreamableFile(await this.migrationService.getDatabase()); } + + @Post("to/s3") + @ApiBearerAuth() + async migrateToS3(){ + this.migrationService.migrateToS3(); + } + + @Post("to/local") + @ApiBearerAuth() + async migrateToLocal(){ + this.migrationService.migrateToLocal(); + } + + // @Delete("s3") + // @ApiBearerAuth() + // async removeS3(){ + // this.migrationService.clearS3(); + // } } diff --git a/src/modules/webtoon/migration/migration.module.ts b/src/modules/webtoon/migration/migration.module.ts index 86696f3..a625d88 100644 --- a/src/modules/webtoon/migration/migration.module.ts +++ b/src/modules/webtoon/migration/migration.module.ts @@ -3,10 +3,11 @@ import {MigrationController} from "./migration.controller"; import {MigrationService} from "./migration.service"; import {WebtoonModule} from "../webtoon/webtoon.module"; import {MiscModule} from "../../misc/misc.module"; +import {FileModule} from "../../file/file.module"; @Module({ providers: [MigrationService], controllers: [MigrationController], - imports: [WebtoonModule, MiscModule] + imports: [WebtoonModule, MiscModule, FileModule] }) export class MigrationModule{} diff --git a/src/modules/webtoon/migration/migration.service.ts b/src/modules/webtoon/migration/migration.service.ts index cb3848f..9d91505 100644 --- a/src/modules/webtoon/migration/migration.service.ts +++ b/src/modules/webtoon/migration/migration.service.ts @@ -8,6 +8,8 @@ import axios from "axios"; import {PrismaService} from "../../misc/prisma.service"; import * as fs from "node:fs"; import * as https from "node:https"; +import {FileService} from "../../file/file.service"; +import {ConfigService} from "@nestjs/config"; @Injectable() export class MigrationService{ @@ -16,11 +18,13 @@ export class MigrationService{ constructor( private readonly webtoonDatabaseService: WebtoonDatabaseService, - private readonly prismaService: PrismaService + private readonly prismaService: PrismaService, + private readonly fileService: FileService, + private readonly configService: ConfigService, ){} async migrateFrom(url: string, adminKey: string){ - this.logger.debug(`Start migration from ${url}`) + this.logger.debug(`Start migration from ${url}`); // Get migration infos using axios from the url const response = await axios.get(url + "/api/v1/migration/infos", { headers: { @@ -98,4 +102,60 @@ export class MigrationService{ throw error; } } + + async migrateToS3(){ + const s3Saver = this.fileService.getS3Saver(); + const dbImageBatchSize = 10000; + const s3BatchSize = parseInt(this.configService.get("S3_BATCH_SIZE")); + const imageCount = await this.prismaService.images.count(); + await s3Saver.createBucketIfNotExists(); + for(let i = 0; i < imageCount; i += dbImageBatchSize){ + this.logger.debug(`Migrating images from ${i} to ${i + dbImageBatchSize}`); + const images = await this.prismaService.images.findMany({ + skip: i, + take: dbImageBatchSize, + select: { + id: true, + sum: true + } + }); + const imageSums = images.map(image => image.sum); + for(let j = 0; j < imageSums.length; j += s3BatchSize){ + this.logger.debug(`Uploading images from ${j} to ${j + s3BatchSize}`); + const batch = imageSums.slice(j, j + s3BatchSize); + await Promise.all(batch.map(async(sum) => s3Saver.saveFile(await this.fileService.loadImage(sum), sum))); + } + } + this.logger.debug("Migration to S3 completed!"); + } + + async migrateToLocal(){ + const fileSaver = this.fileService.getFileSaver(); + const dbImageBatchSize = 10000; + const localBatchSize = parseInt(this.configService.get("S3_BATCH_SIZE")); + const imageCount = await this.prismaService.images.count(); + for(let i = 0; i < imageCount; i += dbImageBatchSize){ + this.logger.debug(`Migrating images from ${i} to ${i + dbImageBatchSize}`); + const images = await this.prismaService.images.findMany({ + skip: i, + take: dbImageBatchSize, + select: { + id: true, + sum: true + } + }); + const imageSums = images.map(image => image.sum); + for(let j = 0; j < imageSums.length; j += localBatchSize){ + this.logger.debug(`Saving images from ${j} to ${j + localBatchSize}`); + const batch = imageSums.slice(j, j + localBatchSize); + await Promise.all(batch.map(async(sum) => fileSaver.saveFile(await this.fileService.loadImage(sum), sum))); + } + } + this.logger.debug("Migration to local completed!"); + } + + async clearS3(): Promise{ + const s3Saver = this.fileService.getS3Saver(); + await s3Saver.clearBucket(); + } } diff --git a/src/modules/webtoon/update/update.service.ts b/src/modules/webtoon/update/update.service.ts index 49df9da..8fc1801 100644 --- a/src/modules/webtoon/update/update.service.ts +++ b/src/modules/webtoon/update/update.service.ts @@ -46,7 +46,7 @@ export class UpdateService{ this.logger.debug(`Updating thumbnail for webtoon ${webtoon.title} (${webtoon.language})`); const cachedWebtoon: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoon.title, webtoon.language); const thumbnail: Buffer = await this.miscService.convertWebtoonThumbnail(cachedWebtoon.thumbnail); - const sum: string = this.webtoonDatabaseService.saveImage(thumbnail); + const sum: string = await this.webtoonDatabaseService.saveImage(thumbnail); // Check if thumbnail already exists let dbThumbnail = await tx.images.findFirst({ where: { diff --git a/src/modules/webtoon/webtoon/webtoon-database.service.ts b/src/modules/webtoon/webtoon/webtoon-database.service.ts index 9cee6b5..6f0f061 100644 --- a/src/modules/webtoon/webtoon/webtoon-database.service.ts +++ b/src/modules/webtoon/webtoon/webtoon-database.service.ts @@ -1,4 +1,3 @@ -import * as fs from "fs"; import CachedWebtoonModel from "./models/models/cached-webtoon.model"; import EpisodeModel from "./models/models/episode.model"; import EpisodeDataModel from "./models/models/episode-data.model"; @@ -13,18 +12,20 @@ import {MiscService} from "../../misc/misc.service"; import ImageTypes from "./models/enums/image-types"; import WebtoonResponse from "./models/responses/webtoon-response"; import MigrationInfosResponse from "../migration/models/responses/migration-infos.response"; +import {FileService} from "../../file/file.service"; +import {ConfigService} from "@nestjs/config"; @Injectable() export class WebtoonDatabaseService{ - private readonly CHUNK_SIZE: number = 10; private readonly MIGRATION_CHUNK_SIZE: number = 15000; private readonly logger = new Logger(WebtoonDatabaseService.name); constructor( private readonly prismaService: PrismaService, - private readonly miscService: MiscService + private readonly fileService: FileService, + private readonly configService: ConfigService, ){} async saveEpisode(webtoon: CachedWebtoonModel, episode: EpisodeModel, episodeData: EpisodeDataModel): Promise{ @@ -53,7 +54,7 @@ export class WebtoonDatabaseService{ const thumbnailType = imageTypes.find(type => type.name === ImageTypes.EPISODE_THUMBNAIL); const imageType = imageTypes.find(type => type.name === ImageTypes.EPISODE_IMAGE); - const thumbnailSum: string = this.saveImage(episodeData.thumbnail); + const thumbnailSum: string = await this.saveImage(episodeData.thumbnail); const dbThumbnail = await tx.images.create({ data: { sum: thumbnailSum, @@ -70,7 +71,13 @@ export class WebtoonDatabaseService{ }); // Save images - const imagesSum: string[] = episodeData.images.map((image: Buffer) => this.saveImage(image)); + const imagesSum = []; + const batchSize = parseInt(this.configService.get("S3_BATCH_SIZE")); + for (let i = 0; i < episodeData.images.length; i += batchSize){ + const batch = episodeData.images.slice(i, i + batchSize); + const results = await Promise.all(batch.map(buffer => this.saveImage(buffer))); + imagesSum.push(...results); + } let dbImages: any[] = await tx.images.findMany({ where: { sum: { @@ -125,7 +132,11 @@ export class WebtoonDatabaseService{ updated_at: new Date() } }); - }); + }, + { + timeout: 1000 * 60 * 5 + } + ); } async saveWebtoon(webtoon: WebtoonModel, webtoonData: WebtoonDataModel): Promise{ @@ -158,10 +169,10 @@ export class WebtoonDatabaseService{ const topType = imageTypes.find(type => type.name === ImageTypes.WEBTOON_TOP_BANNER); const mobileType = imageTypes.find(type => type.name === ImageTypes.WEBTOON_MOBILE_BANNER); - const thumbnailSum: string = this.saveImage(webtoonData.thumbnail); - const backgroundSum: string = this.saveImage(webtoonData.backgroundBanner); - const topSum: string = this.saveImage(webtoonData.topBanner); - const mobileSum: string = this.saveImage(webtoonData.mobileBanner); + const thumbnailSum: string = await this.saveImage(webtoonData.thumbnail); + const backgroundSum: string = await this.saveImage(webtoonData.backgroundBanner); + const topSum: string = await this.saveImage(webtoonData.topBanner); + const mobileSum: string = await this.saveImage(webtoonData.mobileBanner); const dbThumbnail = await tx.images.create({ data: { @@ -432,30 +443,20 @@ export class WebtoonDatabaseService{ }); const images: Record = {}; for(const image of dbImages) - images[image.sum] = this.loadImage(image.sum); + images[image.sum] = await this.loadImage(image.sum); return images; } - saveImage(image: Buffer): string{ - if(!fs.existsSync("./images")) - fs.mkdirSync("./images"); - const imageSum: string = this.miscService.getSum(image); - const folder = imageSum.substring(0, 2); - const path = `./images/${folder}`; - if(!fs.existsSync(path)) - fs.mkdirSync(path); - fs.writeFileSync(`${path}/${imageSum}.webp`, image); - return imageSum; + async saveImage(image: Buffer): Promise{ + return await this.fileService.saveImage(image); } - loadImage(imageSum: string): Buffer{ - const folder = imageSum.substring(0, 2); - return fs.readFileSync(`./images/${folder}/${imageSum}.webp`); + async loadImage(imageSum: string): Promise{ + return await this.fileService.loadImage(imageSum); } - removeImage(imageSum: string): void{ - const folder = imageSum.substring(0, 2); - fs.rmSync(`./images/${folder}/${imageSum}.webp`); + async removeImage(imageSum: string): Promise{ + await this.fileService.removeImage(imageSum); } async getRandomThumbnails(){ diff --git a/src/modules/webtoon/webtoon/webtoon.controller.ts b/src/modules/webtoon/webtoon/webtoon.controller.ts index fff3210..646faef 100644 --- a/src/modules/webtoon/webtoon/webtoon.controller.ts +++ b/src/modules/webtoon/webtoon/webtoon.controller.ts @@ -1,4 +1,4 @@ -import {Controller, Get, Param} from "@nestjs/common"; +import {Controller, Get, Header, Param} from "@nestjs/common"; import {ApiResponse, ApiTags} from "@nestjs/swagger"; import {WebtoonDatabaseService} from "./webtoon-database.service"; import {WebtoonIdDto} from "./models/dto/webtoon-id.dto"; @@ -47,6 +47,7 @@ export class WebtoonController{ } @Get("episodes/:episodeId/images") + @Header("Cache-Control", "public, max-age=604800000") @ApiResponse({status: 200, description: "Returns a list of images for an episode", type: String, isArray: true}) async getEpisodeImagesNew(@Param() episodeIdDto: EpisodeIdDto): Promise{ return this.webtoonDatabaseService.getEpisodeImages(episodeIdDto.episodeId); diff --git a/src/modules/webtoon/webtoon/webtoon.module.ts b/src/modules/webtoon/webtoon/webtoon.module.ts index e91206d..e89998f 100644 --- a/src/modules/webtoon/webtoon/webtoon.module.ts +++ b/src/modules/webtoon/webtoon/webtoon.module.ts @@ -5,9 +5,10 @@ import {WebtoonDownloaderService} from "./webtoon-downloader.service"; import {WebtoonDatabaseService} from "./webtoon-database.service"; import {DownloadManagerService} from "./download-manager.service"; import {MiscModule} from "../../misc/misc.module"; +import {FileModule} from "../../file/file.module"; @Module({ - imports: [MiscModule], + imports: [MiscModule, FileModule], controllers: [WebtoonController], providers: [WebtoonParserService, WebtoonDownloaderService, WebtoonDatabaseService, DownloadManagerService], exports: [DownloadManagerService, WebtoonDatabaseService, WebtoonParserService],