From a22d9ef0b70c4d1db36a95a91f0dcb475f191a66 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Tue, 20 Feb 2024 13:10:56 -0800 Subject: [PATCH 1/7] feat(multipart uploads): use new async multipart endpoints PE-5386 --- .../lib/src/turbo_streamed_upload.dart | 2 +- .../lib/src/turbo_upload_service.dart | 39 +++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/ardrive_uploader/lib/src/turbo_streamed_upload.dart b/packages/ardrive_uploader/lib/src/turbo_streamed_upload.dart index b69291dc99..481ebc16dc 100644 --- a/packages/ardrive_uploader/lib/src/turbo_streamed_upload.dart +++ b/packages/ardrive_uploader/lib/src/turbo_streamed_upload.dart @@ -2,8 +2,8 @@ import 'package:ardrive_uploader/ardrive_uploader.dart'; import 'package:ardrive_uploader/src/exceptions.dart'; import 'package:ardrive_uploader/src/streamed_upload.dart'; import 'package:ardrive_uploader/src/turbo_upload_service.dart'; -import 'package:arweave/arweave.dart'; import 'package:ardrive_uploader/src/utils/logger.dart'; +import 'package:arweave/arweave.dart'; import 'package:flutter/foundation.dart'; class TurboStreamedUpload implements StreamedUpload { diff --git a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart index 8a90b10daa..fd70638773 100644 --- a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart +++ b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart @@ -138,20 +138,19 @@ class TurboUploadService { final finaliseInfo = await r.retry( () => dio.post( - '$turboUploadUri/chunks/arweave/$uploadId/-1', + '$turboUploadUri/chunks/arweave/$uploadId/finalize', data: null, cancelToken: finalizeCancelToken, - options: Options( - validateStatus: (int? status) { - return status != null && - ((status >= 200 && status < 300) || status == 504); - }, - ), ), ); - if (finaliseInfo.statusCode == 504) { - final confirmInfo = await _confirmUpload(dataItem.id); + if (finaliseInfo.statusCode == 202) { + // TODO: Send this upload to a queue. We'd need to change the + // type of the returned data though. Perhaps the returned object + // could be an event emitter that the calling client cas use to + // listen for async outcomes like finalization success/failure. + final confirmInfo = + await _confirmUpload(dataItemId: dataItem.id, uploadId: uploadId); onSendProgressTimer?.cancel(); return confirmInfo; @@ -176,17 +175,25 @@ class TurboUploadService { } } - Future _confirmUpload(TxID dataItemId) async { + // TODO: This funciton as designed should go away, but some incremental + // improvements that could be helpful: + // - Don't use recursion. Use a while loop instead. + // - Have a max retry count and/or max finalization time + // - Make retries based on an exponential backoff rather than a fixed delay + // - Make starting time for first wait based on some linear function of file size + // - Don't keep retrying infinitely in the case of errors. + Future _confirmUpload( + {required TxID dataItemId, required String uploadId}) async { try { - logger.d('[$dataItemId] Confirming upload to Turbo'); + logger.d( + '[$dataItemId] Confirming upload to Turbo with uploadId $uploadId'); final response = await dio.get( - '$turboUploadUri/v1/tx/$dataItemId/status', + '$turboUploadUri/chunks/arweave/$uploadId/status', ); final responseData = response.data; - if (responseData['status'] == 'CONFIRMED' || - responseData['status'] == 'FINALIZED') { + if (responseData['status'] == 'finalized') { logger.d('[$dataItemId] DataItem confirmed!'); return response; } else { @@ -195,12 +202,12 @@ class TurboUploadService { await Future.delayed(dataItemConfirmationRetryDelay); - return _confirmUpload(dataItemId); + return _confirmUpload(dataItemId: dataItemId, uploadId: uploadId); } } catch (e) { await Future.delayed(dataItemConfirmationRetryDelay); - return _confirmUpload(dataItemId); + return _confirmUpload(dataItemId: dataItemId, uploadId: uploadId); } } From ab89695b31e34bd2043d25bf84a48ac07c9779a5 Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Tue, 20 Feb 2024 22:50:24 -0800 Subject: [PATCH 2/7] feat(multipart uploads): use latest constants PE-5386 --- packages/ardrive_uploader/lib/src/turbo_upload_service.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart index fd70638773..772379283d 100644 --- a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart +++ b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart @@ -193,7 +193,7 @@ class TurboUploadService { final responseData = response.data; - if (responseData['status'] == 'finalized') { + if (responseData['status'] == 'FINALIZED') { logger.d('[$dataItemId] DataItem confirmed!'); return response; } else { From 9a773c5e65bf6f503c010982910ce679cc596c5c Mon Sep 17 00:00:00 2001 From: Ariel Melendez Date: Thu, 22 Feb 2024 16:15:54 -0800 Subject: [PATCH 3/7] feat(multipart uploads): use exponential backoffs for data item finalization to turbo PE-5386 --- .../lib/src/turbo_upload_service.dart | 72 +++++++++++++------ 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart index 772379283d..2aaf798417 100644 --- a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart +++ b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:math'; import 'dart:typed_data'; import 'package:ardrive_uploader/src/exceptions.dart'; @@ -17,7 +18,6 @@ class TurboUploadService { final r = RetryOptions(maxAttempts: 8); final List _cancelTokens = []; final dio = Dio(); - final dataItemConfirmationRetryDelay = Duration(seconds: 15); final maxInFlightData = MiB(100).size; Timer? onSendProgressTimer; @@ -147,10 +147,12 @@ class TurboUploadService { if (finaliseInfo.statusCode == 202) { // TODO: Send this upload to a queue. We'd need to change the // type of the returned data though. Perhaps the returned object - // could be an event emitter that the calling client cas use to + // could be an event emitter that the calling client can use to // listen for async outcomes like finalization success/failure. - final confirmInfo = - await _confirmUpload(dataItemId: dataItem.id, uploadId: uploadId); + final confirmInfo = await _confirmUpload( + dataItemId: dataItem.id, + uploadId: uploadId, + dataItemSize: dataItem.dataItemSize); onSendProgressTimer?.cancel(); return confirmInfo; @@ -183,32 +185,47 @@ class TurboUploadService { // - Make starting time for first wait based on some linear function of file size // - Don't keep retrying infinitely in the case of errors. Future _confirmUpload( - {required TxID dataItemId, required String uploadId}) async { - try { - logger.d( - '[$dataItemId] Confirming upload to Turbo with uploadId $uploadId'); + {required TxID dataItemId, + required String uploadId, + required int dataItemSize}) async { + final fileSizeInGiB = (dataItemSize.toDouble() / GiB(1).size).ceil(); + final maxWaitTime = Duration(minutes: fileSizeInGiB); + logger.d( + '[$dataItemId] Confirming upload to Turbo with uploadId $uploadId for up to ${maxWaitTime.inMinutes} minutes.'); + + final startTime = DateTime.now(); + final cutoffTime = startTime.add(maxWaitTime); + int attemptCount = 0; + + while (DateTime.now().isBefore(cutoffTime)) { final response = await dio.get( '$turboUploadUri/chunks/arweave/$uploadId/status', ); final responseData = response.data; - - if (responseData['status'] == 'FINALIZED') { - logger.d('[$dataItemId] DataItem confirmed!'); - return response; - } else { - logger.d( - '[$dataItemId] DataItem not confirmed. Retrying in ${dataItemConfirmationRetryDelay.toString()}'); - - await Future.delayed(dataItemConfirmationRetryDelay); - - return _confirmUpload(dataItemId: dataItemId, uploadId: uploadId); + final responseStatus = responseData['status']; + switch (responseStatus) { + case 'FINALIZED': + logger.d('[$dataItemId] DataItem confirmed!'); + return response; + case 'UNDERFUNDED': + throw UploadCanceledException('Upload canceled. Underfunded.'); + case 'ASSEMBLING': + case 'VALIDATING': + case 'FINALIZING': + final retryAfterDuration = + dataItemConfirmationRetryDelay(attemptCount++); + logger.d( + '[$dataItemId] DataItem not confirmed. Retrying in ${retryAfterDuration.inMilliseconds}ms'); + + await Future.delayed(retryAfterDuration); + default: + throw UploadCanceledException( + 'Upload canceled. Finalization failed. Status: ${responseData['status']}'); } - } catch (e) { - await Future.delayed(dataItemConfirmationRetryDelay); - - return _confirmUpload(dataItemId: dataItemId, uploadId: uploadId); } + throw UploadCanceledException( + 'Upload canceled. Finalization took too long.'); } Future cancel() { @@ -306,3 +323,12 @@ Stream streamToChunks( yield buffer.toBytes(); } } + +final defaultBaseDuration = Duration(milliseconds: 250); +Duration dataItemConfirmationRetryDelay(int iteration, + {Duration baseDuration = const Duration(milliseconds: 100), + Duration maxDuration = const Duration(seconds: 8)}) { + return Duration( + milliseconds: min(baseDuration.inMilliseconds * pow(2, iteration).toInt(), + maxDuration.inMilliseconds)); +} From 7d7d3f32c2b442b4ae32e6aa1341fc1bc70db0a1 Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Wed, 28 Feb 2024 14:03:10 -0300 Subject: [PATCH 4/7] Update turbo_upload_service.dart - format code --- .../lib/src/turbo_upload_service.dart | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart index 2aaf798417..bb3bb31150 100644 --- a/packages/ardrive_uploader/lib/src/turbo_upload_service.dart +++ b/packages/ardrive_uploader/lib/src/turbo_upload_service.dart @@ -150,9 +150,11 @@ class TurboUploadService { // could be an event emitter that the calling client can use to // listen for async outcomes like finalization success/failure. final confirmInfo = await _confirmUpload( - dataItemId: dataItem.id, - uploadId: uploadId, - dataItemSize: dataItem.dataItemSize); + dataItemId: dataItem.id, + uploadId: uploadId, + dataItemSize: dataItem.dataItemSize, + ); + onSendProgressTimer?.cancel(); return confirmInfo; @@ -184,10 +186,11 @@ class TurboUploadService { // - Make retries based on an exponential backoff rather than a fixed delay // - Make starting time for first wait based on some linear function of file size // - Don't keep retrying infinitely in the case of errors. - Future _confirmUpload( - {required TxID dataItemId, - required String uploadId, - required int dataItemSize}) async { + Future _confirmUpload({ + required TxID dataItemId, + required String uploadId, + required int dataItemSize, + }) async { final fileSizeInGiB = (dataItemSize.toDouble() / GiB(1).size).ceil(); final maxWaitTime = Duration(minutes: fileSizeInGiB); logger.d( @@ -325,10 +328,16 @@ Stream streamToChunks( } final defaultBaseDuration = Duration(milliseconds: 250); -Duration dataItemConfirmationRetryDelay(int iteration, - {Duration baseDuration = const Duration(milliseconds: 100), - Duration maxDuration = const Duration(seconds: 8)}) { + +Duration dataItemConfirmationRetryDelay( + int iteration, { + Duration baseDuration = const Duration(milliseconds: 100), + Duration maxDuration = const Duration(seconds: 8), +}) { return Duration( - milliseconds: min(baseDuration.inMilliseconds * pow(2, iteration).toInt(), - maxDuration.inMilliseconds)); + milliseconds: min( + baseDuration.inMilliseconds * pow(2, iteration).toInt(), + maxDuration.inMilliseconds, + ), + ); } From 381c60e2d546d55e97229f1e8ea0fce440ebbf7b Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Fri, 1 Mar 2024 10:06:53 -0300 Subject: [PATCH 5/7] Update data_bundler_utils.dart - use the length of data item list instead of 2 --- packages/ardrive_uploader/lib/src/utils/data_bundler_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ardrive_uploader/lib/src/utils/data_bundler_utils.dart b/packages/ardrive_uploader/lib/src/utils/data_bundler_utils.dart index 5130c8823d..660cdd435a 100644 --- a/packages/ardrive_uploader/lib/src/utils/data_bundler_utils.dart +++ b/packages/ardrive_uploader/lib/src/utils/data_bundler_utils.dart @@ -5,7 +5,7 @@ Future> createDataItemResultFromDataItemFiles( Wallet wallet, ) async { final List dataItemList = []; - final dataItemCount = 2; + final dataItemCount = dataItemList.length; for (var i = 0; i < dataItemCount; i++) { final dataItem = dataItems[i]; await createDataItemTaskEither( From ba455ca5e1848cfa9365b4358a261027de66623f Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Fri, 1 Mar 2024 10:28:09 -0300 Subject: [PATCH 6/7] Update upload_form.dart --- lib/components/upload_form.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/components/upload_form.dart b/lib/components/upload_form.dart index dd831f5f69..c8d28a3c9e 100644 --- a/lib/components/upload_form.dart +++ b/lib/components/upload_form.dart @@ -822,7 +822,10 @@ class _UploadFormState extends State { } return ArDriveStandardModal( - width: kLargeDialogWidth, + hasCloseButton: true, + width: state.failedTasks != null + ? kLargeDialogWidth + : kMediumDialogWidth, title: 'Problem with Upload', description: appLocalizationsOf(context).yourUploadFailed, content: state.failedTasks != null From 5c5034643eedacd147ece787b753fa2beb507420 Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Fri, 1 Mar 2024 10:54:01 -0300 Subject: [PATCH 7/7] bump version and proposed release notes --- android/fastlane/metadata/android/en-US/changelogs/112.txt | 2 ++ pubspec.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 android/fastlane/metadata/android/en-US/changelogs/112.txt diff --git a/android/fastlane/metadata/android/en-US/changelogs/112.txt b/android/fastlane/metadata/android/en-US/changelogs/112.txt new file mode 100644 index 0000000000..10f2ff6a04 --- /dev/null +++ b/android/fastlane/metadata/android/en-US/changelogs/112.txt @@ -0,0 +1,2 @@ +- Enhances user experience and efficiency of Turbo uploads +- Fixes issue with upload folders using AR \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 84a039995d..0c0e88ca83 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,7 +3,7 @@ description: Secure, permanent storage publish_to: 'none' -version: 2.36.0 +version: 2.37.0 environment: sdk: '>=3.0.2 <4.0.0'