Merge pull request #128 from codelitdev/issue-127

Cloudfront compatible hosting

README.md

@@ -2,6 +2,31 @@
 
 MediaLit is a Node.js based app to store, convert and optimise the media files on any AWS S3 compatible storage.
 
+## Setting up correct access on AWS S3 bucket
+
+Before you start uploading to your bucket, make sure you have set up the correct access on your S3 bucket.
+
+### 1. Without Cloudfront
+
+![BLock public access](./apps/api/assets/without-cloudfront.png)
+
+### 2. With Cloudfront
+
+![BLock public access](./apps/api/assets/with-cloudfront.png)
+
+## Using Cloudfront
+
+If you need to use a Cloudfront CDN, you can enable it in the app, by setting up the following values in your .env file.
+
+```sh
+USE_CLOUDFRONT=true
+CLOUDFRONT_ENDPOINT=CLOUDFRONT_DISTRIBUTION_NAME
+CLOUDFRONT_PRIVATE_KEY="PRIVATE_KEY"
+CLOUDFRONT_KEY_PAIR_ID=KEY_PAIR_ID
+```
+
+We assume that since you are using Cloudfront, you have locked down your bucket from public access. Therefore, all the files uploaded to the bucket will have ACL set to `private` i.e. they will require signed URLs in order to access them.
+
 ## Enable trust proxy
 
 This app is based on [Express](https://expressjs.com/) which cannot work reliably when it is behind a proxy. For example, it cannot detect if it behind a proxy.

apps/api/package.json

@@ -32,6 +32,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.55.0",
+    "@aws-sdk/cloudfront-signer": "^3.572.0",
     "@aws-sdk/s3-request-presigner": "^3.55.0",
     "@medialit/images": "workspace:*",
     "@medialit/models": "workspace:*",

apps/api/src/config/constants.ts

@@ -1,30 +1,48 @@
-export const appName = process.env.APP_NAME || "Cloud Upload Service";
-export const dbConnectionString = process.env.DB_CONNECTION_STRING;
+// App config
+export const appName = process.env.APP_NAME || "MediaLit";
 export const jwtSecret = process.env.JWT_SECRET || "r@nd0m1e";
 export const jwtExpire = process.env.JWT_EXPIRES_IN || "1d";
-export const mailHost = process.env.EMAIL_HOST;
-export const mailUser = process.env.EMAIL_USER;
-export const mailPass = process.env.EMAIL_PASS;
-export const mailFrom = process.env.EMAIL_FROM;
-export const mailPort = parseInt(process.env.EMAIL_PORT || "") || 587;
+export const tempFileDirForUploads = process.env.TEMP_FILE_DIR_FOR_UPLOADS;
+export const maxFileUploadSize = process.env.MAX_UPLOAD_SIZE || 2147483648;
+export const PRESIGNED_URL_VALIDITY_MINUTES = 5;
+export const PRESIGNED_URL_LENGTH = 100;
+export const MEDIA_ID_LENGTH = 40;
 export const APIKEY_RESTRICTION_REFERRER = "referrer";
 export const APIKEY_RESTRICTION_IP = "ipaddress";
 export const APIKEY_RESTRICTION_CUSTOM = "custom";
-export const tempFileDirForUploads = process.env.TEMP_FILE_DIR_FOR_UPLOADS;
-export const maxFileUploadSize = process.env.MAX_UPLOAD_SIZE || 2147483648;
 export const imagePattern = /^image\/(jpe?g|png)$/;
 export const imagePatternIncludingGif = /^image\/(jpe?g|png|gif|webp)$/;
 export const videoPattern = /video/;
+export const thumbnailWidth = 120;
+export const thumbnailHeight = 69;
+export const numberOfRecordsPerPage = 10;
+
+// Database config
+export const dbConnectionString = process.env.DB_CONNECTION_STRING;
+
+// Mail config
+export const mailHost = process.env.EMAIL_HOST;
+export const mailUser = process.env.EMAIL_USER;
+export const mailPass = process.env.EMAIL_PASS;
+export const mailFrom = process.env.EMAIL_FROM;
+export const mailPort = parseInt(process.env.EMAIL_PORT || "") || 587;
+
+// AWS S3 config
 export const cloudEndpoint = process.env.CLOUD_ENDPOINT || "";
 export const cloudRegion = process.env.CLOUD_REGION || "";
 export const cloudKey = process.env.CLOUD_KEY || "";
 export const cloudSecret = process.env.CLOUD_SECRET || "";
 export const cloudBucket = process.env.CLOUD_BUCKET_NAME || "";
-export const cdnEndpoint = process.env.CDN_ENDPOINT || "";
-export const thumbnailWidth = 120;
-export const thumbnailHeight = 69;
-export const numberOfRecordsPerPage = 10;
-export const PRESIGNED_URL_VALIDITY_MINUTES = 5;
-export const PRESIGNED_URL_LENGTH = 100;
-export const MEDIA_ID_LENGTH = 40;
 export const CLOUD_PREFIX = process.env.CLOUD_PREFIX || "";
+export const S3_ENDPOINT = process.env.S3_ENDPOINT || "";
+
+// Cloudfront config
+export const USE_CLOUDFRONT = process.env.USE_CLOUDFRONT === "true";
+export const CLOUDFRONT_ENDPOINT = process.env.CLOUDFRONT_ENDPOINT || "";
+export const CLOUDFRONT_KEY_PAIR_ID = process.env.CLOUDFRONT_KEY_PAIR_ID || "";
+export const CLOUDFRONT_PRIVATE_KEY = process.env.CLOUDFRONT_PRIVATE_KEY || "";
+export const CDN_MAX_AGE = process.env.CDN_MAX_AGE
+    ? +process.env.CDN_MAX_AGE
+    : 1000 * 60 * 60; // one hour
+
+export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT;

apps/api/src/media/service.ts

@@ -5,9 +5,8 @@ import {
     tempFileDirForUploads,
     imagePattern,
     videoPattern,
-    thumbnailWidth,
-    thumbnailHeight,
     imagePatternIncludingGif,
+    USE_CLOUDFRONT,
 } from "../config/constants";
 import imageUtils from "@medialit/images";
 import {
@@ -18,6 +17,7 @@ import {
 import type { MediaWithUserId } from "./model";
 import {
     generateSignedUrl,
+    generateCDNSignedUrl,
     putObject,
     deleteObject,
     UploadParams,
@@ -37,7 +37,7 @@ import {
 } from "./queries";
 import * as presignedUrlService from "../presigning/service";
 import getTags from "./utils/get-tags";
-import { getMainFileUrl, getThumbnailUrl } from "./utils/get-cdn-urls";
+import { getMainFileUrl, getThumbnailUrl } from "./utils/get-public-urls";
 
 const generateAndUploadThumbnail = async ({
     workingDirectory,
@@ -69,7 +69,7 @@ const generateAndUploadThumbnail = async ({
             Key: key,
             Body: createReadStream(thumbPath),
             ContentType: "image/webp",
-            ACL: "public-read",
+            ACL: USE_CLOUDFRONT ? "private" : "public-read",
             Tagging: tags,
         });
     }
@@ -122,12 +122,16 @@ async function upload({
     const uploadParams: UploadParams = {
         Key: generateKey({
             mediaId: fileName.name,
-            extension: fileExtension,
-            type: "main",
+            access: access === "public" ? "public" : "private",
+            filename: `main.${fileExtension}`,
         }),
         Body: createReadStream(mainFilePath),
         ContentType: mimeType,
-        ACL: access === "public" ? "public-read" : "private",
+        ACL: USE_CLOUDFRONT
+            ? "private"
+            : access === "public"
+            ? "public-read"
+            : "private",
     };
     const tags = getTags(userId, group);
     uploadParams.Tagging = tags;
@@ -142,7 +146,8 @@ async function upload({
             originalFilePath: mainFilePath,
             key: generateKey({
                 mediaId: fileName.name,
-                type: "thumb",
+                access: "public",
+                filename: "thumb.webp",
             }),
             tags,
         });
@@ -238,6 +243,12 @@ async function getMediaDetails({
         return null;
     }
 
+    const key = generateKey({
+        mediaId: media.mediaId,
+        access: media.accessControl === "private" ? "private" : "public",
+        filename: `main.${path.extname(media.fileName).replace(".", "")}`,
+    });
+
     return {
         mediaId: media.mediaId,
         originalFileName: media.originalFileName,
@@ -246,15 +257,9 @@ async function getMediaDetails({
         access: media.accessControl === "private" ? "private" : "public",
         file:
             media.accessControl === "private"
-                ? await generateSignedUrl({
-                      name: generateKey({
-                          mediaId: media.mediaId,
-                          extension: path
-                              .extname(media.fileName)
-                              .replace(".", ""),
-                          type: "main",
-                      }),
-                  })
+                ? USE_CLOUDFRONT
+                    ? generateCDNSignedUrl(key)
+                    : await generateSignedUrl(key)
                 : getMainFileUrl(media),
         thumbnail: media.thumbnailGenerated
             ? getThumbnailUrl(media.mediaId)
@@ -278,16 +283,16 @@ async function deleteMedia({
 
     const key = generateKey({
         mediaId,
-        extension: media.mimeType.split("/")[1],
-        type: "main",
+        access: media.accessControl === "private" ? "private" : "public",
+        filename: `main.${media.fileName.split(".")[1]}`,
     });
     await deleteObject({ Key: key });
 
     if (media.thumbnailGenerated) {
         const thumbKey = generateKey({
             mediaId,
-            extension: media.mimeType.split("/")[1],
-            type: "thumb",
+            access: "public",
+            filename: "thumb.webp",
         });
         await deleteObject({ Key: thumbKey });
     }

apps/api/src/media/utils/generate-key.ts

@@ -1,17 +1,15 @@
 import { CLOUD_PREFIX } from "../../config/constants";
 
-interface GenerateKeyProps {
-    mediaId: string;
-    type: "main" | "thumb";
-    extension?: string;
-}
-
 export default function generateKey({
     mediaId,
-    type,
-    extension,
-}: GenerateKeyProps): string {
-    return `${CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : ""}${mediaId}/${type}.${
-        type === "thumb" ? "webp" : extension
-    }`;
+    access,
+    filename,
+}: {
+    mediaId: string;
+    access: "private" | "public";
+    filename: string;
+}): string {
+    return `${
+        CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : ""
+    }${access}/${mediaId}/${filename}`;
 }

apps/api/src/media/utils/get-public-urls.ts

@@ -0,0 +1,15 @@
+import path from "path";
+import { ENDPOINT, CLOUD_PREFIX } from "../../config/constants";
+import { Media } from "@medialit/models";
+
+const prefix = CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : "";
+
+export function getMainFileUrl(media: Media) {
+    return `${ENDPOINT}/${prefix}public/${media.mediaId}/main${path.extname(
+        media.fileName
+    )}`;
+}
+
+export function getThumbnailUrl(mediaId: string) {
+    return `${ENDPOINT}/${prefix}public/${mediaId}/thumb.webp`;
+}

apps/api/src/services/s3.ts

@@ -1,5 +1,5 @@
 import { ReadStream } from "fs";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import { getSignedUrl as getS3SignedUrl } from "@aws-sdk/s3-request-presigner";
 import {
     S3Client,
     PutObjectCommand,
@@ -13,7 +13,12 @@ import {
     cloudSecret,
     cloudBucket,
     cloudRegion,
+    CLOUDFRONT_KEY_PAIR_ID,
+    CLOUDFRONT_PRIVATE_KEY,
+    CDN_MAX_AGE,
+    CLOUDFRONT_ENDPOINT,
 } from "../config/constants";
+import { getSignedUrl as getCfSignedUrl } from "@aws-sdk/cloudfront-signer";
 
 export interface UploadParams {
     Key: string;
@@ -65,14 +70,29 @@ export const getObjectTagging = async (params: { Key: string }) => {
     return response;
 };
 
-export const generateSignedUrl = async ({
-    name,
-    mimetype,
-}: PresignedURLParams): Promise<string> => {
+export const generateSignedUrl = async (key: string): Promise<string> => {
     const command = new GetObjectCommand({
         Bucket: cloudBucket,
-        Key: name,
+        Key: key,
+    });
+    const url = await getS3SignedUrl(s3Client, command);
+    return url;
+};
+
+export const generateCDNSignedUrl = (key: string): string => {
+    if (
+        !CLOUDFRONT_ENDPOINT ||
+        !CLOUDFRONT_KEY_PAIR_ID ||
+        !CLOUDFRONT_PRIVATE_KEY
+    ) {
+        throw new Error("CDN configuration is missing");
+    }
+
+    const url = getCfSignedUrl({
+        url: `${CLOUDFRONT_ENDPOINT}/${key}`,
+        keyPairId: CLOUDFRONT_KEY_PAIR_ID,
+        privateKey: CLOUDFRONT_PRIVATE_KEY,
+        dateLessThan: new Date(Date.now() + CDN_MAX_AGE).toISOString(),
     });
-    const url = await getSignedUrl(s3Client, command);
     return url;
 };

docker-compose.yml

@@ -11,15 +11,20 @@ services:
       - CLOUD_KEY=${CLOUD_KEY}
       - CLOUD_SECRET=${CLOUD_SECRET}
       - CLOUD_BUCKET_NAME=${CLOUD_BUCKET_NAME}
-      - CDN_ENDPOINT=${CDN_ENDPOINT}
+      - CLOUD_PREFIX=${CLOUD_PREFIX}
+      - S3_ENDPOINT=${S3_ENDPOINT}
       - TEMP_FILE_DIR_FOR_UPLOADS=${TEMP_FILE_DIR_FOR_UPLOADS}
       - PORT=8000
       - EMAIL_HOST=${EMAIL_HOST}
       - EMAIL_USER=${EMAIL_USER}
       - EMAIL_PASS=${EMAIL_PASS}
       - EMAIL_FROM=${EMAIL_FROM}
       - ENABLE_TRUST_PROXY=${ENABLE_TRUST_PROXY}
-      - CLOUD_PREFIX=${CLOUD_PREFIX}
+      - USE_CLOUDFRONT=${USE_CLOUDFRONT} 
+      - CLOUDFRONT_ENDPOINT=${CLOUDFRONT_ENDPOINT} 
+      - CLOUDFRONT_KEY_PAIR_ID=${CLOUDFRONT_KEY_PAIR_ID} 
+      - CLOUDFRONT_PRIVATE_KEY=${CLOUDFRONT_PRIVATE_KEY}
+      - CDN_MAX_AGE=${CDN_MAX_AGE}
     expose:
       - 8000