Upload files in batch for web-app previews

Only show in developer changelog

newIDE/app/src/EventsFunctionsExtensionsLoader/CodeWriters/BrowserS3EventsFunctionCodeWriter.js

@@ -3,9 +3,52 @@ import {
   type EventsFunctionCodeWriter,
   type EventsFunctionCodeWriterCallbacks,
 } from '..';
-import { uploadObject, getBaseUrl } from '../../Utils/GDevelopServices/Preview';
+import {
+  uploadObjects,
+  getBaseUrl,
+  type UploadedObject,
+} from '../../Utils/GDevelopServices/Preview';
 import { makeTimestampedId } from '../../Utils/TimestampedId';
 import slugs from 'slugs';
+import debounce from 'lodash/debounce';
+
+let batchedUploads: Array<{
+  uploadedObject: UploadedObject,
+  onSuccess: () => void,
+  onError: (error: Error) => void,
+}> = [];
+
+const flushBatchedUploads = debounce(async () => {
+  const uploads = [...batchedUploads];
+  console.info(`Uploading a batch of ${uploads.length} extension generated files...`, uploads);
+
+  batchedUploads = [];
+
+  try {
+    await uploadObjects(
+      uploads.map(upload => upload.uploadedObject)
+    );
+  } catch (error) {
+    uploads.forEach(upload => upload.onError(error));
+
+    return;
+  }
+
+  uploads.forEach(upload => upload.onSuccess());
+}, 10); // Wait for up to 10ms, to avoid adding more latency to extension generation.
+
+/**
+ * Upload a file by batching it with other files that are being uploaded.
+ *
+ * Extension generated files are uploaded in batches to avoid making a *lot* of requests
+ * (games can have from dozens to **hundreds** of extensions and generated files).
+ */
+const uploadObjectInNextBatch = (uploadedObject: UploadedObject) => {
+  return new Promise((resolve, reject) => {
+    batchedUploads.push({ uploadedObject, onSuccess: resolve, onError: reject });
+    flushBatchedUploads();
+  });
+};
 
 /**
  * Create the EventsFunctionCodeWriter that writes generated code for events functions
@@ -29,7 +72,7 @@ export const makeBrowserS3EventsFunctionCodeWriter = ({
       const key = getPathFor(functionCodeNamespace);
       onWriteFile({ includeFile: key, content: code });
       console.log(`Uploading function generated code to ${key}...`);
-      return uploadObject({
+      return uploadObjectInNextBatch({
         Key: getPathFor(functionCodeNamespace),
         Body: code,
         ContentType: 'text/javascript',
@@ -42,7 +85,7 @@ export const makeBrowserS3EventsFunctionCodeWriter = ({
       const key = getPathFor(behaviorCodeNamespace);
       onWriteFile({ includeFile: key, content: code });
       console.log(`Uploading behavior generated code to ${key}...`);
-      return uploadObject({
+      return uploadObjectInNextBatch({
         Key: getPathFor(behaviorCodeNamespace),
         Body: code,
         ContentType: 'text/javascript',
@@ -55,7 +98,7 @@ export const makeBrowserS3EventsFunctionCodeWriter = ({
       const key = getPathFor(objectCodeNamespace);
       onWriteFile({ includeFile: key, content: code });
       console.log(`Uploading object generated code to ${key}...`);
-      return uploadObject({
+      return uploadObjectInNextBatch({
         Key: getPathFor(objectCodeNamespace),
         Body: code,
         ContentType: 'text/javascript',

newIDE/app/src/ExportAndShare/BrowserExporters/BrowserS3FileSystem.js

@@ -1,6 +1,6 @@
 // @flow
 import path from 'path-browserify';
-import { uploadObject } from '../../Utils/GDevelopServices/Preview';
+import { uploadObjects } from '../../Utils/GDevelopServices/Preview';
 const gd: libGDevelop = global.gd;
 
 export type TextFileDescriptor = {|
@@ -60,17 +60,15 @@ export default class BrowserS3FileSystem {
     });
   }
 
-  uploadPendingObjects = () => {
-    return Promise.all(this._pendingUploadObjects.map(uploadObject)).then(
-      result => {
-        console.log('Uploaded all objects:', result);
-        this._pendingUploadObjects = [];
-      },
-      error => {
-        console.error("Can't upload all objects:", error);
-        throw error;
-      }
-    );
+  uploadPendingObjects = async () => {
+    try {
+      console.log(`Uploading ${this._pendingUploadObjects.length} files for preview...`);
+      await uploadObjects(this._pendingUploadObjects);
+      console.log(`Uploaded all ${this._pendingUploadObjects.length} preview files.`);
+    } catch (error) {
+      console.error("Can't upload all objects:", error);
+      throw error;
+    }
   };
 
   mkDir = (path: string) => {

newIDE/app/src/Utils/GDevelopServices/Preview.js

@@ -1,25 +1,39 @@
 // @flow
 import axios from 'axios';
 import { GDevelopGamePreviews } from './ApiConfigs';
-import { getSignedUrl } from './Usage';
+import { getSignedUrls } from './Usage';
 
 export type UploadedObject = {|
   Key: string,
   Body: string,
   ContentType: 'text/javascript' | 'text/html',
 |};
 
-export const uploadObject = (params: UploadedObject): Promise<any> => {
-  return getSignedUrl({
+export const uploadObjects = async (
+  uploadedObjects: Array<UploadedObject>
+): Promise<void> => {
+  const { signedUrls } = await getSignedUrls({
     uploadType: 'preview',
-    key: params.Key,
-    contentType: params.ContentType,
-  }).then(({ signedUrl }) =>
-    axios.put(signedUrl, params.Body, {
-      headers: {
-        'Content-Type': params.ContentType,
-      },
-    })
+    files: uploadedObjects.map(params => ({
+      key: params.Key,
+      contentType: params.ContentType,
+    })),
+  });
+
+  if (signedUrls.length !== uploadedObjects.length) {
+    throw new Error(
+      'Unexpected response from the API (signed urls count is not the same as uploaded objects count).'
+    );
+  }
+
+  await Promise.all(
+    uploadedObjects.map((params, index) =>
+      axios.put(signedUrls[index], params.Body, {
+        headers: {
+          'Content-Type': params.ContentType,
+        },
+      })
+    )
   );
 };
 

newIDE/app/src/Utils/GDevelopServices/Usage.js

@@ -486,6 +486,19 @@ export const getSignedUrl = async (params: {|
   return response.data;
 };
 
+export const getSignedUrls = async (params: {|
+  uploadType: UploadType,
+  files: Array<{|
+    key: string,
+    contentType: string,
+  |}>,
+|}): Promise<{
+  signedUrls: Array<string>,
+}> => {
+  const response = await apiClient.post('/upload-options/signed-url', params);
+  return response.data;
+};
+
 export const getRedirectToSubscriptionPortalUrl = async (
   getAuthorizationHeader: () => Promise<string>,
   userId: string