Skip to content

Commit

Permalink
Merge pull request #1559 from ardriveapp/PE-5397
Browse files Browse the repository at this point in the history
PE-5387: chunk uploads to turbo
  • Loading branch information
thiagocarvalhodev authored Feb 20, 2024
2 parents 597896d + 802c133 commit b6038fb
Show file tree
Hide file tree
Showing 27 changed files with 611 additions and 630 deletions.
20 changes: 7 additions & 13 deletions lib/blocs/upload/limits.dart
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
import 'package:ardrive_utils/ardrive_utils.dart';

final privateFileSizeLimit = const GiB(65).size;
final fileSizeLimit = const GiB(65).size;
final fileSizeWarning = const GiB(5).size;

final largeFileUploadSizeThreshold = const MiB(500).size;

final mobilePrivateFileSizeLimit = const GiB(10).size;
const maxBundleDataItemCount = 500;
const maxFilesPerBundle = maxBundleDataItemCount ~/ 2;
const maxFilesSizePerBundleUsingTurbo = 1;

final publicFileSafeSizeLimit = const GiB(5).size;
final nonChromeBrowserUploadSafeLimitUsingTurbo = const MiB(500).size;
final d2nBundleSizeLimit = const GiB(65).size;
final turboBundleSizeLimit = const GiB(10).size;

int getBundleSizeLimit(bool isTurbo) =>
isTurbo ? turboBundleSizeLimit : d2nBundleSizeLimit;

final d2nBundleSizeLimit = const GiB(65).size;
final turboBundleSizeLimit = const GiB(10).size;
final mobileBundleSizeLimit = const GiB(65).size;
const maxBundleDataItemCount = 500;
const maxFilesPerBundle = maxBundleDataItemCount ~/ 2;
const maxFilesSizePerBundleUsingTurbo = 1;
45 changes: 17 additions & 28 deletions lib/blocs/upload/upload_cubit.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import 'dart:async';

import 'package:ardrive/authentication/ardrive_auth.dart';
import 'package:ardrive/blocs/blocs.dart';
import 'package:ardrive/blocs/upload/limits.dart';
import 'package:ardrive/blocs/upload/models/models.dart';
import 'package:ardrive/blocs/upload/models/payment_method_info.dart';
import 'package:ardrive/blocs/upload/upload_file_checker.dart';
Expand Down Expand Up @@ -44,7 +43,7 @@ class UploadCubit extends Cubit<UploadState> {
final ProfileCubit _profileCubit;
final DriveDao _driveDao;
final PstService _pst;
final UploadFileChecker _uploadFileChecker;
final UploadFileSizeChecker _uploadFileSizeChecker;
final ArDriveAuth _auth;
final ActivityTracker _activityTracker;
final LicenseService _licenseService;
Expand Down Expand Up @@ -98,7 +97,7 @@ class UploadCubit extends Cubit<UploadState> {
required ProfileCubit profileCubit,
required DriveDao driveDao,
required PstService pst,
required UploadFileChecker uploadFileChecker,
required UploadFileSizeChecker uploadFileSizeChecker,
required ArDriveAuth auth,
required ArDriveUploadPreparationManager arDriveUploadManager,
required ActivityTracker activityTracker,
Expand All @@ -107,7 +106,7 @@ class UploadCubit extends Cubit<UploadState> {
this.uploadFolders = false,
this.isDragNDrop = false,
}) : _profileCubit = profileCubit,
_uploadFileChecker = uploadFileChecker,
_uploadFileSizeChecker = uploadFileSizeChecker,
_driveDao = driveDao,
_pst = pst,
_auth = auth,
Expand Down Expand Up @@ -186,14 +185,14 @@ class UploadCubit extends Cubit<UploadState> {

Future<void> checkFilesAboveLimit() async {
if (_isAPrivateUpload()) {
final tooLargeFiles = await _uploadFileChecker
.checkAndReturnFilesAbovePrivateLimit(files: files);
final largeFiles =
await _uploadFileSizeChecker.getFilesAboveSizeLimit(files: files);

if (tooLargeFiles.isNotEmpty) {
if (largeFiles.isNotEmpty) {
emit(
UploadFileTooLarge(
hasFilesToUpload: files.length > tooLargeFiles.length,
tooLargeFileNames: tooLargeFiles,
hasFilesToUpload: files.length > largeFiles.length,
tooLargeFileNames: largeFiles,
isPrivate: _isAPrivateUpload(),
),
);
Expand Down Expand Up @@ -402,13 +401,9 @@ class UploadCubit extends Cubit<UploadState> {

if (_uploadMethod == UploadMethod.turbo) {
await _verifyIfUploadContainsLargeFilesUsingTurbo();
if ((_containsLargeTurboUpload ?? false) &&
!hasEmittedWarning &&
kIsWeb &&
!await AppPlatform.isChrome()) {
if ((_containsLargeTurboUpload ?? false) && !hasEmittedWarning) {
emit(
UploadShowingWarning(
reason: UploadWarningReason.fileTooLargeOnNonChromeBrowser,
uploadPlanForAR: uploadPlanForAr,
uploadPlanForTurbo: uploadPlanForTurbo,
),
Expand Down Expand Up @@ -688,11 +683,9 @@ class UploadCubit extends Cubit<UploadState> {
if (_containsLargeTurboUpload == null) {
_containsLargeTurboUpload = false;

for (var file in files) {
if (await file.ioFile.length >= largeFileUploadSizeThreshold) {
_containsLargeTurboUpload = true;
break;
}
if (await _uploadFileSizeChecker.hasFileAboveSizeLimit(files: files)) {
_containsLargeTurboUpload = true;
return;
}
}
}
Expand Down Expand Up @@ -814,8 +807,8 @@ class UploadCubit extends Cubit<UploadState> {

Future<void> skipLargeFilesAndCheckForConflicts() async {
emit(UploadPreparationInProgress());
final List<String> filesToSkip = await _uploadFileChecker
.checkAndReturnFilesAbovePrivateLimit(files: files);
final List<String> filesToSkip =
await _uploadFileSizeChecker.getFilesAboveSizeLimit(files: files);

files.removeWhere(
(file) => filesToSkip.contains(file.getIdentifier()),
Expand All @@ -836,20 +829,16 @@ class UploadCubit extends Cubit<UploadState> {

Future<void> verifyFilesAboveWarningLimit() async {
if (!_targetDrive.isPrivate) {
bool fileAboveWarningLimit =
await _uploadFileChecker.hasFileAboveSafePublicSizeLimit(
files: files,
);

if (fileAboveWarningLimit) {
if (await _uploadFileSizeChecker.hasFileAboveWarningSizeLimit(
files: files)) {
emit(UploadShowingWarning(
reason: UploadWarningReason.fileTooLarge,
uploadPlanForAR: null,
uploadPlanForTurbo: null,
));

return;
}

await prepareUploadPlanAndCostEstimates();
}

Expand Down
58 changes: 36 additions & 22 deletions lib/blocs/upload/upload_file_checker.dart
Original file line number Diff line number Diff line change
@@ -1,37 +1,51 @@
import 'package:ardrive/blocs/upload/models/upload_file.dart';

class UploadFileChecker {
UploadFileChecker({
required int publicFileSafeSizeLimit,
required int privateFileSafeSizeLimit,
}) : _publicFileSafeSizeLimit = publicFileSafeSizeLimit,
_privateFileSafeSizeLimit = privateFileSafeSizeLimit;

final int _publicFileSafeSizeLimit;
final int _privateFileSafeSizeLimit;

Future<bool> hasFileAboveSafePublicSizeLimit(
{required List<UploadFile> files}) async {
class UploadFileSizeChecker {
UploadFileSizeChecker({
required int fileSizeLimit,
required int fileSizeWarning,
}) : _fileSizeLimit = fileSizeLimit,
_fileSizeWarning = fileSizeWarning;

final int _fileSizeLimit;
final int _fileSizeWarning;

Future<bool> hasFileAboveSizeLimit({
required List<UploadFile> files,
}) =>
_hasFileAboveLimit(files: files, limit: _fileSizeLimit);

Future<bool> hasFileAboveWarningSizeLimit({
required List<UploadFile> files,
}) =>
_hasFileAboveLimit(files: files, limit: _fileSizeWarning);

Future<List<String>> getFilesAboveSizeLimit({
required List<UploadFile> files,
}) async {
final filesAboveSizeLimit = <String>[];

for (final file in files) {
final fileSize = await file.ioFile.length;
if (fileSize > _publicFileSafeSizeLimit) {
return true;
if (fileSize > _fileSizeLimit) {
filesAboveSizeLimit.add(file.getIdentifier());
}
}
return false;
}

Future<List<String>> checkAndReturnFilesAbovePrivateLimit(
{required List<UploadFile> files}) async {
final filesAbovePrivateLimit = <String>[];
return filesAboveSizeLimit;
}

Future<bool> _hasFileAboveLimit({
required List<UploadFile> files,
required int limit,
}) async {
for (final file in files) {
final fileSize = await file.ioFile.length;
if (fileSize > _privateFileSafeSizeLimit) {
filesAbovePrivateLimit.add(file.getIdentifier());
if (fileSize > limit) {
return true;
}
}

return filesAbovePrivateLimit;
return false;
}
}
11 changes: 0 additions & 11 deletions lib/blocs/upload/upload_state.dart
Original file line number Diff line number Diff line change
Expand Up @@ -179,30 +179,19 @@ class UploadComplete extends UploadState {}
class UploadWalletMismatch extends UploadState {}

class UploadShowingWarning extends UploadState {
final UploadWarningReason reason;
final UploadPlan? uploadPlanForAR;
final UploadPlan? uploadPlanForTurbo;

UploadShowingWarning({
required this.reason,
this.uploadPlanForAR,
this.uploadPlanForTurbo,
});

@override
List<Object> get props => [reason];
}

class UploadCanceled extends UploadState {}

class CancelD2NUploadWarning extends UploadState {}

enum UploadWarningReason {
/// The user is attempting to upload a file that is too large.
fileTooLarge,
fileTooLargeOnNonChromeBrowser,
}

enum UploadErrors {
turboTimeout,
unknown,
Expand Down
Loading

0 comments on commit b6038fb

Please sign in to comment.