diff --git a/README.md b/README.md index 63bb3f53..b84abbdf 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,31 @@ MediaLit is a Node.js based app to store, convert and optimise the media files on any AWS S3 compatible storage. +## Setting up correct access on AWS S3 bucket + +Before you start uploading to your bucket, make sure you have set up the correct access on your S3 bucket. + +### 1. Without Cloudfront + +![BLock public access](./apps/api/assets/without-cloudfront.png) + +### 2. With Cloudfront + +![BLock public access](./apps/api/assets/with-cloudfront.png) + +## Using Cloudfront + +If you need to use a Cloudfront CDN, you can enable it in the app, by setting up the following values in your .env file. + +```sh +USE_CLOUDFRONT=true +CLOUDFRONT_ENDPOINT=CLOUDFRONT_DISTRIBUTION_NAME +CLOUDFRONT_PRIVATE_KEY="PRIVATE_KEY" +CLOUDFRONT_KEY_PAIR_ID=KEY_PAIR_ID +``` + +We assume that since you are using Cloudfront, you have locked down your bucket from public access. Therefore, all the files uploaded to the bucket will have ACL set to `private` i.e. they will require signed URLs in order to access them. + ## Enable trust proxy This app is based on [Express](https://expressjs.com/) which cannot work reliably when it is behind a proxy. For example, it cannot detect if it behind a proxy. diff --git a/apps/api/assets/with-cloudfront.png b/apps/api/assets/with-cloudfront.png new file mode 100644 index 00000000..2c497f65 Binary files /dev/null and b/apps/api/assets/with-cloudfront.png differ diff --git a/apps/api/assets/without-cloudfront.png b/apps/api/assets/without-cloudfront.png new file mode 100644 index 00000000..773a9a8c Binary files /dev/null and b/apps/api/assets/without-cloudfront.png differ diff --git a/apps/api/package.json b/apps/api/package.json index 6f6849c3..49d5ca99 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -32,6 +32,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.55.0", + "@aws-sdk/cloudfront-signer": "^3.572.0", "@aws-sdk/s3-request-presigner": "^3.55.0", "@medialit/images": "workspace:*", "@medialit/models": "workspace:*", diff --git a/apps/api/src/config/constants.ts b/apps/api/src/config/constants.ts index 2c9d6fe1..14191f1d 100644 --- a/apps/api/src/config/constants.ts +++ b/apps/api/src/config/constants.ts @@ -1,30 +1,48 @@ -export const appName = process.env.APP_NAME || "Cloud Upload Service"; -export const dbConnectionString = process.env.DB_CONNECTION_STRING; +// App config +export const appName = process.env.APP_NAME || "MediaLit"; export const jwtSecret = process.env.JWT_SECRET || "r@nd0m1e"; export const jwtExpire = process.env.JWT_EXPIRES_IN || "1d"; -export const mailHost = process.env.EMAIL_HOST; -export const mailUser = process.env.EMAIL_USER; -export const mailPass = process.env.EMAIL_PASS; -export const mailFrom = process.env.EMAIL_FROM; -export const mailPort = parseInt(process.env.EMAIL_PORT || "") || 587; +export const tempFileDirForUploads = process.env.TEMP_FILE_DIR_FOR_UPLOADS; +export const maxFileUploadSize = process.env.MAX_UPLOAD_SIZE || 2147483648; +export const PRESIGNED_URL_VALIDITY_MINUTES = 5; +export const PRESIGNED_URL_LENGTH = 100; +export const MEDIA_ID_LENGTH = 40; export const APIKEY_RESTRICTION_REFERRER = "referrer"; export const APIKEY_RESTRICTION_IP = "ipaddress"; export const APIKEY_RESTRICTION_CUSTOM = "custom"; -export const tempFileDirForUploads = process.env.TEMP_FILE_DIR_FOR_UPLOADS; -export const maxFileUploadSize = process.env.MAX_UPLOAD_SIZE || 2147483648; export const imagePattern = /^image\/(jpe?g|png)$/; export const imagePatternIncludingGif = /^image\/(jpe?g|png|gif|webp)$/; export const videoPattern = /video/; +export const thumbnailWidth = 120; +export const thumbnailHeight = 69; +export const numberOfRecordsPerPage = 10; + +// Database config +export const dbConnectionString = process.env.DB_CONNECTION_STRING; + +// Mail config +export const mailHost = process.env.EMAIL_HOST; +export const mailUser = process.env.EMAIL_USER; +export const mailPass = process.env.EMAIL_PASS; +export const mailFrom = process.env.EMAIL_FROM; +export const mailPort = parseInt(process.env.EMAIL_PORT || "") || 587; + +// AWS S3 config export const cloudEndpoint = process.env.CLOUD_ENDPOINT || ""; export const cloudRegion = process.env.CLOUD_REGION || ""; export const cloudKey = process.env.CLOUD_KEY || ""; export const cloudSecret = process.env.CLOUD_SECRET || ""; export const cloudBucket = process.env.CLOUD_BUCKET_NAME || ""; -export const cdnEndpoint = process.env.CDN_ENDPOINT || ""; -export const thumbnailWidth = 120; -export const thumbnailHeight = 69; -export const numberOfRecordsPerPage = 10; -export const PRESIGNED_URL_VALIDITY_MINUTES = 5; -export const PRESIGNED_URL_LENGTH = 100; -export const MEDIA_ID_LENGTH = 40; export const CLOUD_PREFIX = process.env.CLOUD_PREFIX || ""; +export const S3_ENDPOINT = process.env.S3_ENDPOINT || ""; + +// Cloudfront config +export const USE_CLOUDFRONT = process.env.USE_CLOUDFRONT === "true"; +export const CLOUDFRONT_ENDPOINT = process.env.CLOUDFRONT_ENDPOINT || ""; +export const CLOUDFRONT_KEY_PAIR_ID = process.env.CLOUDFRONT_KEY_PAIR_ID || ""; +export const CLOUDFRONT_PRIVATE_KEY = process.env.CLOUDFRONT_PRIVATE_KEY || ""; +export const CDN_MAX_AGE = process.env.CDN_MAX_AGE + ? +process.env.CDN_MAX_AGE + : 1000 * 60 * 60; // one hour + +export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT; diff --git a/apps/api/src/media/service.ts b/apps/api/src/media/service.ts index 4bdc32f3..9fa81ceb 100644 --- a/apps/api/src/media/service.ts +++ b/apps/api/src/media/service.ts @@ -5,9 +5,8 @@ import { tempFileDirForUploads, imagePattern, videoPattern, - thumbnailWidth, - thumbnailHeight, imagePatternIncludingGif, + USE_CLOUDFRONT, } from "../config/constants"; import imageUtils from "@medialit/images"; import { @@ -18,6 +17,7 @@ import { import type { MediaWithUserId } from "./model"; import { generateSignedUrl, + generateCDNSignedUrl, putObject, deleteObject, UploadParams, @@ -37,7 +37,7 @@ import { } from "./queries"; import * as presignedUrlService from "../presigning/service"; import getTags from "./utils/get-tags"; -import { getMainFileUrl, getThumbnailUrl } from "./utils/get-cdn-urls"; +import { getMainFileUrl, getThumbnailUrl } from "./utils/get-public-urls"; const generateAndUploadThumbnail = async ({ workingDirectory, @@ -69,7 +69,7 @@ const generateAndUploadThumbnail = async ({ Key: key, Body: createReadStream(thumbPath), ContentType: "image/webp", - ACL: "public-read", + ACL: USE_CLOUDFRONT ? "private" : "public-read", Tagging: tags, }); } @@ -122,12 +122,16 @@ async function upload({ const uploadParams: UploadParams = { Key: generateKey({ mediaId: fileName.name, - extension: fileExtension, - type: "main", + access: access === "public" ? "public" : "private", + filename: `main.${fileExtension}`, }), Body: createReadStream(mainFilePath), ContentType: mimeType, - ACL: access === "public" ? "public-read" : "private", + ACL: USE_CLOUDFRONT + ? "private" + : access === "public" + ? "public-read" + : "private", }; const tags = getTags(userId, group); uploadParams.Tagging = tags; @@ -142,7 +146,8 @@ async function upload({ originalFilePath: mainFilePath, key: generateKey({ mediaId: fileName.name, - type: "thumb", + access: "public", + filename: "thumb.webp", }), tags, }); @@ -238,6 +243,12 @@ async function getMediaDetails({ return null; } + const key = generateKey({ + mediaId: media.mediaId, + access: media.accessControl === "private" ? "private" : "public", + filename: `main.${path.extname(media.fileName).replace(".", "")}`, + }); + return { mediaId: media.mediaId, originalFileName: media.originalFileName, @@ -246,15 +257,9 @@ async function getMediaDetails({ access: media.accessControl === "private" ? "private" : "public", file: media.accessControl === "private" - ? await generateSignedUrl({ - name: generateKey({ - mediaId: media.mediaId, - extension: path - .extname(media.fileName) - .replace(".", ""), - type: "main", - }), - }) + ? USE_CLOUDFRONT + ? generateCDNSignedUrl(key) + : await generateSignedUrl(key) : getMainFileUrl(media), thumbnail: media.thumbnailGenerated ? getThumbnailUrl(media.mediaId) @@ -278,16 +283,16 @@ async function deleteMedia({ const key = generateKey({ mediaId, - extension: media.mimeType.split("/")[1], - type: "main", + access: media.accessControl === "private" ? "private" : "public", + filename: `main.${media.fileName.split(".")[1]}`, }); await deleteObject({ Key: key }); if (media.thumbnailGenerated) { const thumbKey = generateKey({ mediaId, - extension: media.mimeType.split("/")[1], - type: "thumb", + access: "public", + filename: "thumb.webp", }); await deleteObject({ Key: thumbKey }); } diff --git a/apps/api/src/media/utils/generate-key.ts b/apps/api/src/media/utils/generate-key.ts index 73ce7f29..1edbc0e1 100644 --- a/apps/api/src/media/utils/generate-key.ts +++ b/apps/api/src/media/utils/generate-key.ts @@ -1,17 +1,15 @@ import { CLOUD_PREFIX } from "../../config/constants"; -interface GenerateKeyProps { - mediaId: string; - type: "main" | "thumb"; - extension?: string; -} - export default function generateKey({ mediaId, - type, - extension, -}: GenerateKeyProps): string { - return `${CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : ""}${mediaId}/${type}.${ - type === "thumb" ? "webp" : extension - }`; + access, + filename, +}: { + mediaId: string; + access: "private" | "public"; + filename: string; +}): string { + return `${ + CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : "" + }${access}/${mediaId}/${filename}`; } diff --git a/apps/api/src/media/utils/get-cdn-urls.ts b/apps/api/src/media/utils/get-cdn-urls.ts deleted file mode 100644 index 3196747d..00000000 --- a/apps/api/src/media/utils/get-cdn-urls.ts +++ /dev/null @@ -1,14 +0,0 @@ -import path from "path"; -import { cdnEndpoint, CLOUD_PREFIX } from "../../config/constants"; -import { Media } from "@medialit/models"; - -export function getMainFileUrl(media: Media) { - return `${cdnEndpoint}/${CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : ""}${ - media.mediaId - }/main${path.extname(media.fileName)}`; -} -export function getThumbnailUrl(mediaId: string) { - return `${cdnEndpoint}/${ - CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : "" - }${mediaId}/thumb.webp`; -} diff --git a/apps/api/src/media/utils/get-public-urls.ts b/apps/api/src/media/utils/get-public-urls.ts new file mode 100644 index 00000000..ea2e8f9a --- /dev/null +++ b/apps/api/src/media/utils/get-public-urls.ts @@ -0,0 +1,15 @@ +import path from "path"; +import { ENDPOINT, CLOUD_PREFIX } from "../../config/constants"; +import { Media } from "@medialit/models"; + +const prefix = CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : ""; + +export function getMainFileUrl(media: Media) { + return `${ENDPOINT}/${prefix}public/${media.mediaId}/main${path.extname( + media.fileName + )}`; +} + +export function getThumbnailUrl(mediaId: string) { + return `${ENDPOINT}/${prefix}public/${mediaId}/thumb.webp`; +} diff --git a/apps/api/src/services/s3.ts b/apps/api/src/services/s3.ts index 42b3de04..939231ac 100644 --- a/apps/api/src/services/s3.ts +++ b/apps/api/src/services/s3.ts @@ -1,5 +1,5 @@ import { ReadStream } from "fs"; -import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { getSignedUrl as getS3SignedUrl } from "@aws-sdk/s3-request-presigner"; import { S3Client, PutObjectCommand, @@ -13,7 +13,12 @@ import { cloudSecret, cloudBucket, cloudRegion, + CLOUDFRONT_KEY_PAIR_ID, + CLOUDFRONT_PRIVATE_KEY, + CDN_MAX_AGE, + CLOUDFRONT_ENDPOINT, } from "../config/constants"; +import { getSignedUrl as getCfSignedUrl } from "@aws-sdk/cloudfront-signer"; export interface UploadParams { Key: string; @@ -65,14 +70,29 @@ export const getObjectTagging = async (params: { Key: string }) => { return response; }; -export const generateSignedUrl = async ({ - name, - mimetype, -}: PresignedURLParams): Promise => { +export const generateSignedUrl = async (key: string): Promise => { const command = new GetObjectCommand({ Bucket: cloudBucket, - Key: name, + Key: key, + }); + const url = await getS3SignedUrl(s3Client, command); + return url; +}; + +export const generateCDNSignedUrl = (key: string): string => { + if ( + !CLOUDFRONT_ENDPOINT || + !CLOUDFRONT_KEY_PAIR_ID || + !CLOUDFRONT_PRIVATE_KEY + ) { + throw new Error("CDN configuration is missing"); + } + + const url = getCfSignedUrl({ + url: `${CLOUDFRONT_ENDPOINT}/${key}`, + keyPairId: CLOUDFRONT_KEY_PAIR_ID, + privateKey: CLOUDFRONT_PRIVATE_KEY, + dateLessThan: new Date(Date.now() + CDN_MAX_AGE).toISOString(), }); - const url = await getSignedUrl(s3Client, command); return url; }; diff --git a/docker-compose.yml b/docker-compose.yml index f4d3d760..11c3876c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,8 @@ services: - CLOUD_KEY=${CLOUD_KEY} - CLOUD_SECRET=${CLOUD_SECRET} - CLOUD_BUCKET_NAME=${CLOUD_BUCKET_NAME} - - CDN_ENDPOINT=${CDN_ENDPOINT} + - CLOUD_PREFIX=${CLOUD_PREFIX} + - S3_ENDPOINT=${S3_ENDPOINT} - TEMP_FILE_DIR_FOR_UPLOADS=${TEMP_FILE_DIR_FOR_UPLOADS} - PORT=8000 - EMAIL_HOST=${EMAIL_HOST} @@ -19,7 +20,11 @@ services: - EMAIL_PASS=${EMAIL_PASS} - EMAIL_FROM=${EMAIL_FROM} - ENABLE_TRUST_PROXY=${ENABLE_TRUST_PROXY} - - CLOUD_PREFIX=${CLOUD_PREFIX} + - USE_CLOUDFRONT=${USE_CLOUDFRONT} + - CLOUDFRONT_ENDPOINT=${CLOUDFRONT_ENDPOINT} + - CLOUDFRONT_KEY_PAIR_ID=${CLOUDFRONT_KEY_PAIR_ID} + - CLOUDFRONT_PRIVATE_KEY=${CLOUDFRONT_PRIVATE_KEY} + - CDN_MAX_AGE=${CDN_MAX_AGE} expose: - 8000 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c650a468..26e9a4c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,6 +38,9 @@ importers: '@aws-sdk/client-s3': specifier: ^3.55.0 version: 3.454.0 + '@aws-sdk/cloudfront-signer': + specifier: ^3.572.0 + version: 3.572.0 '@aws-sdk/s3-request-presigner': specifier: ^3.55.0 version: 3.454.0 @@ -578,6 +581,14 @@ packages: - aws-crt dev: false + /@aws-sdk/cloudfront-signer@3.572.0: + resolution: {integrity: sha512-Y7yejAD+8cR8j9rdPmJ84ZTOi1vXw2wVfTL4GRRnfuFCELCYDE7HoHm7y2715AvKMGHk8lM+me6YK89lijwTFQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/url-parser': 2.2.0 + tslib: 2.6.2 + dev: false + /@aws-sdk/core@3.451.0: resolution: {integrity: sha512-SamWW2zHEf1ZKe3j1w0Piauryl8BQIlej0TBS18A4ACzhjhWXhCs13bO1S88LvPR5mBFXok3XOT6zPOnKDFktw==} engines: {node: '>=14.0.0'} @@ -2996,6 +3007,14 @@ packages: tslib: 2.6.2 dev: false + /@smithy/querystring-parser@2.2.0: + resolution: {integrity: sha512-BvHCDrKfbG5Yhbpj4vsbuPV2GgcpHiAkLeIlcA1LtfpMz3jrqizP1+OguSNSj1MwBHEiN+jwNisXLGdajGDQJA==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/types': 2.12.0 + tslib: 2.6.2 + dev: false + /@smithy/service-error-classification@2.0.6: resolution: {integrity: sha512-fCQ36frtYra2fqY2/DV8+3/z2d0VB/1D1hXbjRcM5wkxTToxq6xHbIY/NGGY6v4carskMyG8FHACxgxturJ9Pg==} engines: {node: '>=14.0.0'} @@ -3035,6 +3054,13 @@ packages: tslib: 2.6.2 dev: false + /@smithy/types@2.12.0: + resolution: {integrity: sha512-QwYgloJ0sVNBeBuBs65cIkTbfzV/Q6ZNPCJ99EICFEdJYG50nGIY/uYXp+TbsdJReIuPr0a0kXmCvren3MbRRw==} + engines: {node: '>=14.0.0'} + dependencies: + tslib: 2.6.2 + dev: false + /@smithy/types@2.5.0: resolution: {integrity: sha512-/a31lYofrMBkJb3BuPlYJTMKDj0hUmKUP6JFZQu6YVuQVoAjubiY0A52U9S0Uysd33n/djexCUSNJ+G9bf3/aA==} engines: {node: '>=14.0.0'} @@ -3050,6 +3076,14 @@ packages: tslib: 2.6.2 dev: false + /@smithy/url-parser@2.2.0: + resolution: {integrity: sha512-hoA4zm61q1mNTpksiSWp2nEl1dt3j726HdRhiNgVJQMj7mLp7dprtF57mOB6JvEk/x9d2bsuL5hlqZbBuHQylQ==} + dependencies: + '@smithy/querystring-parser': 2.2.0 + '@smithy/types': 2.12.0 + tslib: 2.6.2 + dev: false + /@smithy/util-base64@2.0.1: resolution: {integrity: sha512-DlI6XFYDMsIVN+GH9JtcRp3j02JEVuWIn/QOZisVzpIAprdsxGveFed0bjbMRCqmIFe8uetn5rxzNrBtIGrPIQ==} engines: {node: '>=14.0.0'}