From fd567aa22b441a6914cdb0d9ed4147355f08688e Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Mon, 18 Nov 2024 17:07:17 +0100 Subject: [PATCH] Upload files in batch for web-app previews Only show in developer changelog --- .../BrowserS3EventsFunctionCodeWriter.js | 56 +++++++++++++++++-- .../BrowserExporters/BrowserS3FileSystem.js | 26 +++++---- .../app/src/Utils/GDevelopServices/Preview.js | 36 ++++++++---- .../app/src/Utils/GDevelopServices/Usage.js | 13 +++++ 4 files changed, 104 insertions(+), 27 deletions(-) diff --git a/newIDE/app/src/EventsFunctionsExtensionsLoader/CodeWriters/BrowserS3EventsFunctionCodeWriter.js b/newIDE/app/src/EventsFunctionsExtensionsLoader/CodeWriters/BrowserS3EventsFunctionCodeWriter.js index 959226d3f77e..bf263639d6d2 100644 --- a/newIDE/app/src/EventsFunctionsExtensionsLoader/CodeWriters/BrowserS3EventsFunctionCodeWriter.js +++ b/newIDE/app/src/EventsFunctionsExtensionsLoader/CodeWriters/BrowserS3EventsFunctionCodeWriter.js @@ -3,9 +3,57 @@ import { type EventsFunctionCodeWriter, type EventsFunctionCodeWriterCallbacks, } from '..'; -import { uploadObject, getBaseUrl } from '../../Utils/GDevelopServices/Preview'; +import { + uploadObjects, + getBaseUrl, + type UploadedObject, +} from '../../Utils/GDevelopServices/Preview'; import { makeTimestampedId } from '../../Utils/TimestampedId'; import slugs from 'slugs'; +import debounce from 'lodash/debounce'; + +let batchedUploads: Array<{ + uploadedObject: UploadedObject, + onSuccess: () => void, + onError: (error: Error) => void, +}> = []; + +const flushBatchedUploads = debounce(async () => { + const uploads = [...batchedUploads]; + console.info( + `Uploading a batch of ${uploads.length} extension generated files...`, + uploads + ); + + batchedUploads = []; + + try { + await uploadObjects(uploads.map(upload => upload.uploadedObject)); + } catch (error) { + uploads.forEach(upload => upload.onError(error)); + + return; + } + + uploads.forEach(upload => upload.onSuccess()); +}, 10); // Wait for up to 10ms, to avoid adding more latency to extension generation. + +/** + * Upload a file by batching it with other files that are being uploaded. + * + * Extension generated files are uploaded in batches to avoid making a *lot* of requests + * (games can have from dozens to **hundreds** of extensions and generated files). + */ +const uploadObjectInNextBatch = (uploadedObject: UploadedObject) => { + return new Promise((resolve, reject) => { + batchedUploads.push({ + uploadedObject, + onSuccess: resolve, + onError: reject, + }); + flushBatchedUploads(); + }); +}; /** * Create the EventsFunctionCodeWriter that writes generated code for events functions @@ -29,7 +77,7 @@ export const makeBrowserS3EventsFunctionCodeWriter = ({ const key = getPathFor(functionCodeNamespace); onWriteFile({ includeFile: key, content: code }); console.log(`Uploading function generated code to ${key}...`); - return uploadObject({ + return uploadObjectInNextBatch({ Key: getPathFor(functionCodeNamespace), Body: code, ContentType: 'text/javascript', @@ -42,7 +90,7 @@ export const makeBrowserS3EventsFunctionCodeWriter = ({ const key = getPathFor(behaviorCodeNamespace); onWriteFile({ includeFile: key, content: code }); console.log(`Uploading behavior generated code to ${key}...`); - return uploadObject({ + return uploadObjectInNextBatch({ Key: getPathFor(behaviorCodeNamespace), Body: code, ContentType: 'text/javascript', @@ -55,7 +103,7 @@ export const makeBrowserS3EventsFunctionCodeWriter = ({ const key = getPathFor(objectCodeNamespace); onWriteFile({ includeFile: key, content: code }); console.log(`Uploading object generated code to ${key}...`); - return uploadObject({ + return uploadObjectInNextBatch({ Key: getPathFor(objectCodeNamespace), Body: code, ContentType: 'text/javascript', diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserS3FileSystem.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserS3FileSystem.js index 2f16ac16f7e1..8e406b82e0a4 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserS3FileSystem.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserS3FileSystem.js @@ -1,6 +1,6 @@ // @flow import path from 'path-browserify'; -import { uploadObject } from '../../Utils/GDevelopServices/Preview'; +import { uploadObjects } from '../../Utils/GDevelopServices/Preview'; const gd: libGDevelop = global.gd; export type TextFileDescriptor = {| @@ -60,17 +60,19 @@ export default class BrowserS3FileSystem { }); } - uploadPendingObjects = () => { - return Promise.all(this._pendingUploadObjects.map(uploadObject)).then( - result => { - console.log('Uploaded all objects:', result); - this._pendingUploadObjects = []; - }, - error => { - console.error("Can't upload all objects:", error); - throw error; - } - ); + uploadPendingObjects = async () => { + try { + console.log( + `Uploading ${this._pendingUploadObjects.length} files for preview...` + ); + await uploadObjects(this._pendingUploadObjects); + console.log( + `Uploaded all ${this._pendingUploadObjects.length} preview files.` + ); + } catch (error) { + console.error("Can't upload all objects:", error); + throw error; + } }; mkDir = (path: string) => { diff --git a/newIDE/app/src/Utils/GDevelopServices/Preview.js b/newIDE/app/src/Utils/GDevelopServices/Preview.js index f65af160da45..c9f30432f18b 100644 --- a/newIDE/app/src/Utils/GDevelopServices/Preview.js +++ b/newIDE/app/src/Utils/GDevelopServices/Preview.js @@ -1,7 +1,7 @@ // @flow import axios from 'axios'; import { GDevelopGamePreviews } from './ApiConfigs'; -import { getSignedUrl } from './Usage'; +import { getSignedUrls } from './Usage'; export type UploadedObject = {| Key: string, @@ -9,17 +9,31 @@ export type UploadedObject = {| ContentType: 'text/javascript' | 'text/html', |}; -export const uploadObject = (params: UploadedObject): Promise => { - return getSignedUrl({ +export const uploadObjects = async ( + uploadedObjects: Array +): Promise => { + const { signedUrls } = await getSignedUrls({ uploadType: 'preview', - key: params.Key, - contentType: params.ContentType, - }).then(({ signedUrl }) => - axios.put(signedUrl, params.Body, { - headers: { - 'Content-Type': params.ContentType, - }, - }) + files: uploadedObjects.map(params => ({ + key: params.Key, + contentType: params.ContentType, + })), + }); + + if (signedUrls.length !== uploadedObjects.length) { + throw new Error( + 'Unexpected response from the API (signed urls count is not the same as uploaded objects count).' + ); + } + + await Promise.all( + uploadedObjects.map((params, index) => + axios.put(signedUrls[index], params.Body, { + headers: { + 'Content-Type': params.ContentType, + }, + }) + ) ); }; diff --git a/newIDE/app/src/Utils/GDevelopServices/Usage.js b/newIDE/app/src/Utils/GDevelopServices/Usage.js index 3498e8508b8a..33b84232af8d 100644 --- a/newIDE/app/src/Utils/GDevelopServices/Usage.js +++ b/newIDE/app/src/Utils/GDevelopServices/Usage.js @@ -486,6 +486,19 @@ export const getSignedUrl = async (params: {| return response.data; }; +export const getSignedUrls = async (params: {| + uploadType: UploadType, + files: Array<{| + key: string, + contentType: string, + |}>, +|}): Promise<{ + signedUrls: Array, +}> => { + const response = await apiClient.post('/upload-options/signed-url', params); + return response.data; +}; + export const getRedirectToSubscriptionPortalUrl = async ( getAuthorizationHeader: () => Promise, userId: string