From f97d37aa22a7cfa653f78f150a0978bbb924fb3f Mon Sep 17 00:00:00 2001 From: Thiago Carvalho Date: Tue, 26 Sep 2023 16:55:54 -0300 Subject: [PATCH] fet(uploader) fixes bugs on updating the progress --- lib/blocs/upload/upload_cubit.dart | 123 ++++++++++-------- lib/components/upload_form.dart | 2 +- .../lib/src/ardrive_uploader.dart | 102 ++++++++------- .../lib/src/streamed_upload.dart | 14 +- .../lib/src/upload_controller.dart | 43 +++--- 5 files changed, 156 insertions(+), 128 deletions(-) diff --git a/lib/blocs/upload/upload_cubit.dart b/lib/blocs/upload/upload_cubit.dart index a7bdb1a221..16ee2a0d43 100644 --- a/lib/blocs/upload/upload_cubit.dart +++ b/lib/blocs/upload/upload_cubit.dart @@ -576,7 +576,24 @@ class UploadCubit extends Cubit { driveKey: driveKey, ); + List completedTasks = []; + uploadController.onProgressChange((progress) { + final newCompletedTasks = progress.task.where( + (element) => + element.status == UploadStatus.complete && + !completedTasks.contains(element), + ); + + for (var element in newCompletedTasks) { + completedTasks.add(element); + // TODO: Save as the file is finished the upload + // _saveEntityOnDB(element); + for (var metadata in element.content!) { + logger.d(metadata.metadataTxId ?? 'METADATA IS NULL'); + } + } + emit( UploadInProgressUsingNewUploader( progress: progress, @@ -587,62 +604,64 @@ class UploadCubit extends Cubit { }); uploadController.onDone((tasks) async { - logger.d(tasks.toString()); - unawaited(_profileCubit.refreshBalance()); - // Single file only - // TODO: abstract to the database interface. - // TODO: improve API for finishing a file upload. for (var task in tasks) { - final metadatas = task.content; - - if (metadatas != null) { - for (var metadata in metadatas) { - if (metadata is ARFSFileUploadMetadata) { - final fileMetadata = metadata; - - final revisionAction = - conflictingFiles.containsKey(fileMetadata.name) - ? RevisionAction.uploadNewVersion - : RevisionAction.create; - - final entity = FileEntity( - dataContentType: fileMetadata.dataContentType, - dataTxId: fileMetadata.dataTxId, - driveId: fileMetadata.driveId, - id: fileMetadata.id, - lastModifiedDate: fileMetadata.lastModifiedDate, - name: fileMetadata.name, - parentFolderId: fileMetadata.parentFolderId, - size: fileMetadata.size, - // TODO: pinnedDataOwnerAddress - ); - - if (fileMetadata.metadataTxId == null) { - logger.e('Metadata tx id is null'); - throw Exception('Metadata tx id is null'); - } - - entity.txId = fileMetadata.metadataTxId!; - - await _driveDao.transaction(() async { - // If path is a blob from drag and drop, use file name. Else use the path field from folder upload - // TODO: Changed this logic. PLEASE REVIEW IT. - final filePath = '${_targetFolder.path}/${metadata.name}'; - await _driveDao.writeFileEntity(entity, filePath); - await _driveDao.insertFileRevision( - entity.toRevisionCompanion( - performedAction: revisionAction, - ), - ); - }); - } - - // all files are uploaded - emit(UploadComplete()); + unawaited(_saveEntityOnDB(task)); + } + }); + } + + Future _saveEntityOnDB(UploadTask task) async { + unawaited(_profileCubit.refreshBalance()); + // Single file only + // TODO: abstract to the database interface. + // TODO: improve API for finishing a file upload. + final metadatas = task.content; + + if (metadatas != null) { + for (var metadata in metadatas) { + if (metadata is ARFSFileUploadMetadata) { + final fileMetadata = metadata; + + final revisionAction = conflictingFiles.containsKey(fileMetadata.name) + ? RevisionAction.uploadNewVersion + : RevisionAction.create; + + final entity = FileEntity( + dataContentType: fileMetadata.dataContentType, + dataTxId: fileMetadata.dataTxId, + driveId: fileMetadata.driveId, + id: fileMetadata.id, + lastModifiedDate: fileMetadata.lastModifiedDate, + name: fileMetadata.name, + parentFolderId: fileMetadata.parentFolderId, + size: fileMetadata.size, + // TODO: pinnedDataOwnerAddress + ); + + if (fileMetadata.metadataTxId == null) { + logger.e('Metadata tx id is null'); + throw Exception('Metadata tx id is null'); } + + entity.txId = fileMetadata.metadataTxId!; + + await _driveDao.transaction(() async { + // If path is a blob from drag and drop, use file name. Else use the path field from folder upload + // TODO: Changed this logic. PLEASE REVIEW IT. + final filePath = '${_targetFolder.path}/${metadata.name}'; + await _driveDao.writeFileEntity(entity, filePath); + await _driveDao.insertFileRevision( + entity.toRevisionCompanion( + performedAction: revisionAction, + ), + ); + }); } + + // all files are uploaded + emit(UploadComplete()); } - }); + } } Future skipLargeFilesAndCheckForConflicts() async { diff --git a/lib/components/upload_form.dart b/lib/components/upload_form.dart index 5196b9d3f7..8904074d3b 100644 --- a/lib/components/upload_form.dart +++ b/lib/components/upload_form.dart @@ -912,7 +912,7 @@ class _UploadFormState extends State { break; } - if (progress.totalSize > 0 && task.dataItem != null) { + if (task.isProgressAvailable && task.dataItem != null) { progressText = '${filesize(((task.dataItem!.dataItemSize) * task.progress).ceil())}/${filesize(task.dataItem!.dataItemSize)}'; } else { diff --git a/packages/ardrive_uploader/lib/src/ardrive_uploader.dart b/packages/ardrive_uploader/lib/src/ardrive_uploader.dart index cda0de2d10..72a46bbe87 100644 --- a/packages/ardrive_uploader/lib/src/ardrive_uploader.dart +++ b/packages/ardrive_uploader/lib/src/ardrive_uploader.dart @@ -294,58 +294,75 @@ class _ArDriveUploader implements ArDriveUploader { required UploadController controller, }) async { List> activeUploads = []; - int totalSize = 0; // size accumulator + List contents = []; + List tasks = []; + int totalSize = 0; + + for (var f in files) { + final metadata = await _metadataGenerator.generateMetadata( + f.$2, + f.$1, + ); + + final uploadTask = UploadTask( + status: UploadStatus.notStarted, + content: [metadata], + ); + + tasks.add(uploadTask); + + controller.updateProgress(task: uploadTask); + + contents.add(metadata); + } for (int i = 0; i < files.length; i++) { int fileSize = await files[i].$2.length; - if (fileSize >= 500 * 1024 * 1024) { - // File size is >= 500MiB - - // Wait for ongoing uploads to complete - await Future.wait(activeUploads); - totalSize = 0; // Reset the accumulator - activeUploads.clear(); // Clear the active uploads - } else { - while (activeUploads.length >= 25 || - totalSize + fileSize >= 500 * 1024 * 1024) { - await Future.any(activeUploads); // Wait for at least one to complete - // Recalculate totalSize and remove completed futures - totalSize = 0; - var remainingFutures = >[]; - for (var future in activeUploads) { - remainingFutures.add(future.whenComplete(() { - totalSize -= fileSize; // Subtract the size of the completed file - remainingFutures.remove(future); - })); - // Add the size of the corresponding file to totalSize - // This part will depend on how you keep track of file sizes - } - activeUploads = remainingFutures; + while (activeUploads.length >= 25 || + totalSize + fileSize >= 500 * 1024 * 1024) { + await Future.any(activeUploads); + + // Remove completed uploads and update totalSize + int recalculatedSize = 0; + List> ongoingUploads = []; + + for (var f in activeUploads) { + // You need to figure out how to get the file size for the ongoing upload here + // Add its size to recalculatedSize + int ongoingFileSize = await files[i].$2.length; + recalculatedSize += ongoingFileSize; + + ongoingUploads.add(f); } - totalSize += fileSize; // Add to the accumulator + + activeUploads = ongoingUploads; + totalSize = recalculatedSize; } - // Existing logic to start upload - late Future uploadFuture; + totalSize += fileSize; - uploadFuture = _uploadSingleFile( + Future uploadFuture = _uploadSingleFile( file: files[i].$2, + uploadController: controller, wallet: wallet, driveKey: driveKey, - uploadController: controller, - uploadTask: UploadTask( - status: UploadStatus.notStarted, - ), - args: files[i].$1, - ).then((value) { - // activeUploads.remove(uploadFuture); + metadata: contents[i], + uploadTask: tasks[i], + ); + + uploadFuture.then((_) { + activeUploads.remove(uploadFuture); + totalSize -= fileSize; + }).catchError((error) { + activeUploads.remove(uploadFuture); + totalSize -= fileSize; + // TODO: Handle error }); activeUploads.add(uploadFuture); } - // Wait for any remaining uploads to complete await Future.wait(activeUploads); } @@ -355,19 +372,8 @@ class _ArDriveUploader implements ArDriveUploader { required Wallet wallet, SecretKey? driveKey, required UploadTask uploadTask, - ARFSUploadMetadataArgs? args, + required ARFSUploadMetadata metadata, }) async { - final metadata = await _metadataGenerator.generateMetadata( - file, - args, - ); - - // adds the metadata of the file to the upload task - uploadTask = uploadTask.copyWith( - content: [metadata], - status: UploadStatus.bundling, - ); - final createDataBundle = _dataBundler.createDataBundle( file: file, metadata: metadata, diff --git a/packages/ardrive_uploader/lib/src/streamed_upload.dart b/packages/ardrive_uploader/lib/src/streamed_upload.dart index 7694215d0a..32081367c4 100644 --- a/packages/ardrive_uploader/lib/src/streamed_upload.dart +++ b/packages/ardrive_uploader/lib/src/streamed_upload.dart @@ -3,6 +3,7 @@ import 'package:ardrive_uploader/ardrive_uploader.dart'; import 'package:ardrive_uploader/src/turbo_upload_service_base.dart'; import 'package:ardrive_utils/ardrive_utils.dart'; import 'package:arweave/arweave.dart'; +import 'package:flutter/foundation.dart'; import 'package:uuid/uuid.dart'; abstract class StreamedUpload { @@ -47,12 +48,14 @@ class TurboStreamedUpload implements StreamedUpload { }, ); - print( - 'Sending request to turbo. Is possible get progress: ${controller.isPossibleGetProgress}'); - handle = handle.copyWith(status: UploadStatus.inProgress); controller.updateProgress(task: handle); + if (kIsWeb && handle.dataItem!.dataItemSize > 1024 * 1024 * 500) { + handle.isProgressAvailable = false; + controller.updateProgress(task: handle); + } + // TODO: set if its possible to get the progress. Check the turbo web impl // gets the streamed request @@ -78,6 +81,11 @@ class TurboStreamedUpload implements StreamedUpload { .then((value) async { print('Turbo response: ${value.statusCode}'); + if (!handle.isProgressAvailable) { + print('Progress is not available, setting to 1'); + handle.progress = 1; + } + controller.updateProgress( task: handle, ); diff --git a/packages/ardrive_uploader/lib/src/upload_controller.dart b/packages/ardrive_uploader/lib/src/upload_controller.dart index 3a071bbde6..4ca717da79 100644 --- a/packages/ardrive_uploader/lib/src/upload_controller.dart +++ b/packages/ardrive_uploader/lib/src/upload_controller.dart @@ -82,9 +82,6 @@ abstract class UploadController { }); void onProgressChange(Function(UploadProgress progress) callback); - bool get isPossibleGetProgress; - set isPossibleGetProgress(bool value); - factory UploadController( StreamController progressStream, ) { @@ -210,31 +207,29 @@ class _UploadController implements UploadController { print('Upload Finished'); }; - @override - bool get isPossibleGetProgress => _isPossibleGetProgress; - - @override - set isPossibleGetProgress(bool value) => _isPossibleGetProgress = value; - - bool _isPossibleGetProgress = true; - @override final List tasks = []; + // CALCULATE BASED ON TOTAL SIZE NOT ONLY ON THE NUMBER OF TASKS double calculateTotalProgress(List tasks) { - double totalProgress = 0.0; - - for (var task in tasks) { - if (task.dataItem == null) { - continue; - } - - if (task.isProgressAvailable) { - totalProgress += (task.progress * task.dataItem!.dataItemSize); - } - } - - return (totalSize(tasks) == 0) ? 0.0 : totalProgress / totalSize(tasks); + // double totalProgress = 0.0; + + // for (var task in tasks) { + // if (task.dataItem == null) { + // totalProgress += 0; + // continue; + // } + + // if (task.isProgressAvailable) { + // totalProgress += (task.progress * task.dataItem!.dataItemSize); + // } + // } + + // return (totalSize(tasks) == 0) ? 0.0 : totalProgress / totalSize(tasks); + return tasks + .map((e) => e.progress) + .reduce((value, element) => value + element) / + tasks.length; } int totalSize(List tasks) {