Merge pull request #1415 from ardriveapp/PE-4754-address-code-review-…

…comments-on-uploader-downloader-implementations PE-4754: Uploader/Downloader
ardriveapp · Oct 26, 2023 · 983259a · 983259a
2 parents 4df2848 + 097c448
commit 983259a
Show file tree

Hide file tree

Showing 140 changed files with 2,403 additions and 941 deletions.
diff --git a/assets/config/dev.json b/assets/config/dev.json
@@ -2,7 +2,7 @@
   "defaultArweaveGatewayUrl": "https://arweave.net",
   "useTurboUpload": true,
   "useTurboPayment": true,
-  "defaultTurboUploadUrl": "https://upload.ardrive.io",
+  "defaultTurboUploadUrl": "https://upload.ardrive.dev",
   "defaultTurboPaymentUrl": "https://payment.ardrive.dev",
   "allowedDataItemSizeForTurbo": 500000,
   "enableQuickSyncAuthoring": true,

diff --git a/assets/config/prod.json b/assets/config/prod.json
@@ -11,5 +11,5 @@
   "enableAudioPreview": true,
   "stripePublishableKey": "pk_live_51JUAtwC8apPOWkDLMQqNF9sPpfneNSPnwX8YZ8y1FNDl6v94hZIwzgFSYl27bWE4Oos8CLquunUswKrKcaDhDO6m002Yj9AeKj",
   "enablePins": true,
-  "useNewUploader": false
+  "useNewUploader": true
 }
diff --git a/lib/blocs/create_manifest/create_manifest_cubit.dart b/lib/blocs/create_manifest/create_manifest_cubit.dart
@@ -3,17 +3,18 @@ import 'dart:async';
 import 'package:ardrive/blocs/blocs.dart';
 import 'package:ardrive/entities/entities.dart';
 import 'package:ardrive/entities/manifest_data.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/services/services.dart';
 import 'package:ardrive/turbo/services/upload_service.dart';
 import 'package:ardrive/utils/logger/logger.dart';
+import 'package:ardrive_utils/ardrive_utils.dart';
 import 'package:arweave/arweave.dart';
 import 'package:arweave/utils.dart';
 import 'package:collection/collection.dart';
 import 'package:equatable/equatable.dart';
 import 'package:flutter/widgets.dart';
 import 'package:flutter_bloc/flutter_bloc.dart';
+import 'package:pst/pst.dart';
 import 'package:uuid/uuid.dart';
 
 import '../../core/upload/cost_calculator.dart';

diff --git a/lib/blocs/create_snapshot/create_snapshot_cubit.dart b/lib/blocs/create_snapshot/create_snapshot_cubit.dart
@@ -8,7 +8,6 @@ import 'package:ardrive/blocs/profile/profile_cubit.dart';
 import 'package:ardrive/blocs/upload/upload_cubit.dart';
 import 'package:ardrive/core/upload/cost_calculator.dart';
 import 'package:ardrive/entities/snapshot_entity.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/daos/daos.dart';
 import 'package:ardrive/services/services.dart';
 import 'package:ardrive/turbo/services/payment_service.dart';
@@ -25,6 +24,7 @@ import 'package:arweave/arweave.dart';
 import 'package:equatable/equatable.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter_bloc/flutter_bloc.dart';
+import 'package:pst/pst.dart';
 import 'package:stash_shared_preferences/stash_shared_preferences.dart';
 import 'package:uuid/uuid.dart';
 
@@ -421,7 +421,7 @@ class CreateSnapshotCubit extends Cubit<CreateSnapshotState> {
 
     _turboBalance = turboBalance;
     _hasNoTurboBalance = turboBalance == BigInt.zero;
-    _turboCredits = convertCreditsToLiteralString(turboBalance);
+    _turboCredits = convertWinstonToLiteralString(turboBalance);
     _sufficentCreditsBalance = _costEstimateTurbo.totalCost <= _turboBalance;
     _computeIsTurboEnabled();
     _computeIsButtonEnabled();
@@ -464,8 +464,8 @@ class CreateSnapshotCubit extends Cubit<CreateSnapshotState> {
 
     _turboBalance = turboBalance;
     _hasNoTurboBalance = turboBalance == BigInt.zero;
-    _turboCredits = convertCreditsToLiteralString(turboBalance);
-    _arBalance = convertCreditsToLiteralString(auth.currentUser.walletBalance);
+    _turboCredits = convertWinstonToLiteralString(turboBalance);
+    _arBalance = convertWinstonToLiteralString(auth.currentUser.walletBalance);
   }
 
   void _computeIsTurboEnabled() async {

diff --git a/lib/blocs/drive_attach/drive_attach_cubit.dart b/lib/blocs/drive_attach/drive_attach_cubit.dart
@@ -2,7 +2,6 @@ import 'dart:async';
 import 'dart:convert';
 
 import 'package:ardrive/blocs/blocs.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/services/services.dart';
 import 'package:ardrive/utils/logger/logger.dart';

diff --git a/lib/blocs/drive_detail/drive_detail_cubit.dart b/lib/blocs/drive_detail/drive_detail_cubit.dart
@@ -3,13 +3,13 @@ import 'dart:async';
 import 'package:ardrive/authentication/ardrive_auth.dart';
 import 'package:ardrive/blocs/blocs.dart';
 import 'package:ardrive/entities/constants.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/pages/pages.dart';
 import 'package:ardrive/services/services.dart';
 import 'package:ardrive/utils/logger/logger.dart';
 import 'package:ardrive/utils/open_url.dart';
 import 'package:ardrive/utils/user_utils.dart';
+import 'package:ardrive_utils/ardrive_utils.dart';
 import 'package:drift/drift.dart';
 import 'package:equatable/equatable.dart';
 import 'package:flutter/foundation.dart';

diff --git a/lib/blocs/drives/drives_cubit.dart b/lib/blocs/drives/drives_cubit.dart
@@ -2,9 +2,9 @@ import 'dart:async';
 
 import 'package:ardrive/authentication/ardrive_auth.dart';
 import 'package:ardrive/blocs/blocs.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/utils/user_utils.dart';
+import 'package:ardrive_utils/ardrive_utils.dart';
 import 'package:drift/drift.dart';
 import 'package:equatable/equatable.dart';
 import 'package:flutter_bloc/flutter_bloc.dart';

diff --git a/lib/blocs/file_download/file_download_cubit.dart b/lib/blocs/file_download/file_download_cubit.dart
@@ -5,9 +5,10 @@ import 'package:ardrive/core/arfs/entities/arfs_entities.dart';
 import 'package:ardrive/core/arfs/repository/arfs_repository.dart';
 import 'package:ardrive/core/crypto/crypto.dart';
 import 'package:ardrive/download/ardrive_downloader.dart';
+import 'package:ardrive/download/limits.dart';
+import 'package:ardrive/entities/constants.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/services/services.dart';
-import 'package:ardrive/utils/data_size.dart';
 import 'package:ardrive/utils/logger/logger.dart';
 import 'package:ardrive_http/ardrive_http.dart';
 import 'package:ardrive_io/ardrive_io.dart' as io;

diff --git a/lib/blocs/file_download/file_download_state.dart b/lib/blocs/file_download/file_download_state.dart
@@ -81,6 +81,7 @@ class FileDownloadAborted extends FileDownloadState {}
 enum FileDownloadFailureReason {
   unknownError,
   fileAboveLimit,
+  browserDoesNotSupportLargeDownloads,
   networkConnectionError,
   fileNotFound
 }
diff --git a/lib/blocs/file_download/personal_file_download_cubit.dart b/lib/blocs/file_download/personal_file_download_cubit.dart
@@ -42,6 +42,18 @@ class ProfileFileDownloadCubit extends FileDownloadCubit {
         _arfsRepository = arfsRepository,
         super(FileDownloadStarting());
 
+  Future<void> verifyUploadLimitationsAndDownload(SecretKey? cipherKey) async {
+    if (await AppPlatform.isSafari()) {
+      if (_file.size > publicDownloadSafariSizeLimit) {
+        emit(const FileDownloadFailure(
+            FileDownloadFailureReason.browserDoesNotSupportLargeDownloads));
+        return;
+      }
+    }
+
+    download(cipherKey);
+  }
+
   Future<void> download(SecretKey? cipherKey) async {
     try {
       final drive = await _arfsRepository.getDriveById(_file.driveId);
@@ -127,7 +139,9 @@ class ProfileFileDownloadCubit extends FileDownloadCubit {
     String? cipherIvTag;
     SecretKey? fileKey;
 
-    if (drive.drivePrivacy == DrivePrivacy.private) {
+    final isPinFile = _file.pinnedDataOwnerAddress != null;
+
+    if (drive.drivePrivacy == DrivePrivacy.private && !isPinFile) {
       SecretKey? driveKey;
 
       if (cipherKey != null) {
@@ -138,9 +152,11 @@ class ProfileFileDownloadCubit extends FileDownloadCubit {
       } else {
         driveKey = await _driveDao.getDriveKeyFromMemory(_file.driveId);
       }
+
       if (driveKey == null) {
         throw StateError('Drive Key not found');
       }
+
       fileKey = await _driveDao.getFileKey(_file.id, driveKey);
 
       final dataTx = await (_arweave.getTransactionDetails(_file.txId));
@@ -158,6 +174,7 @@ class ProfileFileDownloadCubit extends FileDownloadCubit {
       fileName: _file.name,
       fileSize: _file.size,
       lastModifiedDate: _file.lastModifiedDate,
+      isManifest: _file.contentType == ContentType.manifest,
       contentType:
           _file.contentType ?? lookupMimeTypeWithDefaultType(_file.name),
       cipher: cipher,
@@ -170,13 +187,6 @@ class ProfileFileDownloadCubit extends FileDownloadCubit {
         return;
       }
 
-      if (progress == 100) {
-        emit(FileDownloadFinishedWithSuccess(fileName: _file.name));
-        return;
-      }
-
-      logger.d('Download progress: $progress');
-
       emit(
         FileDownloadWithProgress(
           fileName: _file.name,
@@ -186,6 +196,7 @@ class ProfileFileDownloadCubit extends FileDownloadCubit {
               _file.contentType ?? lookupMimeTypeWithDefaultType(_file.name),
         ),
       );
+
       _downloadProgress.sink.add(FileDownloadProgress(progress / 100));
     }
 

diff --git a/lib/blocs/pin_file/pin_file_bloc.dart b/lib/blocs/pin_file/pin_file_bloc.dart
@@ -4,7 +4,6 @@ import 'package:ardrive/blocs/blocs.dart';
 import 'package:ardrive/core/arfs/entities/arfs_entities.dart';
 import 'package:ardrive/core/crypto/crypto.dart';
 import 'package:ardrive/entities/entities.dart' show FileEntity;
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/misc/misc.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/services/services.dart';

diff --git a/lib/blocs/shared_file/shared_file_cubit.dart b/lib/blocs/shared_file/shared_file_cubit.dart
@@ -1,5 +1,4 @@
 import 'package:ardrive/entities/entities.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/services/services.dart';
 import 'package:ardrive/utils/logger/logger.dart';

diff --git a/lib/blocs/sync/sync_cubit.dart b/lib/blocs/sync/sync_cubit.dart
@@ -7,7 +7,6 @@ import 'package:ardrive/blocs/constants.dart';
 import 'package:ardrive/blocs/sync/ghost_folder.dart';
 import 'package:ardrive/core/activity_tracker.dart';
 import 'package:ardrive/entities/entities.dart';
-import 'package:ardrive/entities/string_types.dart';
 import 'package:ardrive/models/models.dart';
 import 'package:ardrive/services/services.dart';
 import 'package:ardrive/utils/logger/logger.dart';

diff --git a/lib/blocs/sync/utils/add_file_entity_revisions.dart b/lib/blocs/sync/utils/add_file_entity_revisions.dart
@@ -29,17 +29,20 @@ Future<List<FileRevisionsCompanion>> _addNewFileEntityRevisions({
       continue;
     }
     // If Parent-Folder-Id is missing for a file, put it in the root folder
+    try {
+      entity.parentFolderId = entity.parentFolderId ?? rootPath;
+      final revision =
+          entity.toRevisionCompanion(performedAction: revisionPerformedAction);
 
-    entity.parentFolderId = entity.parentFolderId ?? rootPath;
-    final revision =
-        entity.toRevisionCompanion(performedAction: revisionPerformedAction);
+      if (revision.action.value.isEmpty) {
+        continue;
+      }
 
-    if (revision.action.value.isEmpty) {
-      continue;
+      newRevisions.add(revision);
+      latestRevisions[entity.id!] = revision;
+    } catch (e) {
+      logger.e('Error adding revision for entity: ${entity.id}', e);
     }
-
-    newRevisions.add(revision);
-    latestRevisions[entity.id!] = revision;
   }
 
   await database.batch((b) {

diff --git a/lib/blocs/upload/limits.dart b/lib/blocs/upload/limits.dart
@@ -1,16 +1,20 @@
-import 'package:ardrive/utils/data_size.dart';
-import 'package:flutter/foundation.dart';
+import 'package:ardrive_utils/ardrive_utils.dart';
 
-final privateFileSizeLimit = const MiB(100000).size;
+final privateFileSizeLimit = const GiB(65).size;
+
+final largeFileUploadSizeThreshold = const MiB(500).size;
 
 final mobilePrivateFileSizeLimit = const GiB(10).size;
 
 final publicFileSafeSizeLimit = const GiB(5).size;
+final nonChromeBrowserUploadSafeLimitUsingTurbo = const MiB(500).size;
 
-final bundleSizeLimit = kIsWeb ? webBundleSizeLimit : mobileBundleSizeLimit;
+int getBundleSizeLimit(bool isTurbo) =>
+    isTurbo ? turboBundleSizeLimit : d2nBundleSizeLimit;
 
-final webBundleSizeLimit = const MiB(65000).size;
-final mobileBundleSizeLimit = const MiB(65000).size;
+final d2nBundleSizeLimit = const GiB(65).size;
+final turboBundleSizeLimit = const GiB(2).size;
+final mobileBundleSizeLimit = const GiB(65).size;
 const maxBundleDataItemCount = 500;
 const maxFilesPerBundle = maxBundleDataItemCount ~/ 2;
 const maxFilesSizePerBundleUsingTurbo = 1;
diff --git a/lib/blocs/upload/models/upload_plan.dart b/lib/blocs/upload/models/upload_plan.dart
@@ -5,7 +5,6 @@ import 'package:ardrive/blocs/upload/upload_handles/upload_handle.dart';
 import 'package:ardrive/turbo/services/upload_service.dart';
 import 'package:ardrive/utils/bundles/next_fit_bundle_packer.dart';
 import 'package:ardrive/utils/logger/logger.dart';
-import 'package:flutter/foundation.dart';
 
 import '../upload_handles/file_data_item_upload_handle.dart';
 import '../upload_handles/file_v2_upload_handle.dart';
@@ -30,6 +29,7 @@ class UploadPlan {
         folderDataItemUploadHandles,
     required TurboUploadService turboUploadService,
     required int maxDataItemCount,
+    required bool useTurbo,
   }) async {
     final uploadPlan = UploadPlan._create(
       fileV2UploadHandles: fileV2UploadHandles,
@@ -43,13 +43,15 @@ class UploadPlan {
         folderDataItemUploadHandles: folderDataItemUploadHandles,
         turboUploadService: turboUploadService,
         maxDataItemCount: maxDataItemCount,
+        useTurbo: useTurbo,
       );
     }
 
     return uploadPlan;
   }
 
   Future<void> createBundleHandlesFromDataItemHandles({
+    required bool useTurbo,
     Map<String, FileDataItemUploadHandle> fileDataItemUploadHandles = const {},
     Map<String, FolderDataItemUploadHandle> folderDataItemUploadHandles =
         const {},
@@ -58,8 +60,7 @@ class UploadPlan {
   }) async {
     logger.i(
         'Creating bundle handles from data item handles with a max number of files of $maxDataItemCount');
-    final int maxBundleSize =
-        (kIsWeb ? bundleSizeLimit : mobileBundleSizeLimit);
+    final int maxBundleSize = getBundleSizeLimit(useTurbo);
 
     final folderItems = await NextFitBundlePacker<UploadHandle>(
       maxBundleSize: maxBundleSize,