From 9d74caedfea94bcb8ef37f45a75184a976fe7dfa Mon Sep 17 00:00:00 2001 From: Ekjot <43255916+ekjotmultani@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:08:41 -0800 Subject: [PATCH] feat(storage): multi bucket list api (#5576) * added bucket option to list api options and added tests * updated tests to account for changes to uploadData api --------- Co-authored-by: ekjotmultani Co-authored-by: NikaHsn Co-authored-by: Tyler-Larkin Co-authored-by: Elijah Quartey --- .../lib/src/types/storage/list_options.dart | 9 +- .../example/integration_test/list_test.dart | 111 ++++++++++++++++-- .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 9 +- .../test/amplify_storage_s3_dart_test.dart | 3 + 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/list_options.dart b/packages/amplify_core/lib/src/types/storage/list_options.dart index c046f90d55..f0a1d80962 100644 --- a/packages/amplify_core/lib/src/types/storage/list_options.dart +++ b/packages/amplify_core/lib/src/types/storage/list_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.list_options} /// Configurable options for `Amplify.Storage.list`. @@ -15,6 +15,7 @@ class StorageListOptions const StorageListOptions({ this.pageSize = 1000, this.nextToken, + this.bucket, this.pluginOptions, }); @@ -27,8 +28,11 @@ class StorageListOptions /// {@macro amplify_core.storage.list_plugin_options} final StorageListPluginOptions? pluginOptions; + /// Optionally specify which bucket to retrieve + final StorageBucket? bucket; + @override - List get props => [pageSize, nextToken, pluginOptions]; + List get props => [pageSize, nextToken, pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageListOptions'; @@ -37,6 +41,7 @@ class StorageListOptions Map toJson() => { 'pageSize': pageSize, 'nextToken': nextToken, + 'bucket': bucket, 'pluginOptions': pluginOptions?.toJson(), }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart index 22e2a2766b..15c3737225 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart @@ -20,18 +20,41 @@ void main() { '$uniquePrefix/file2.txt', '$uniquePrefix/subdir/file3.txt', '$uniquePrefix/subdir2#file4.txt', + '$uniquePrefix/file5.txt', + '$uniquePrefix/file6.txt', + '$uniquePrefix/subdir3/file7.txt', + '$uniquePrefix/subdir4#file8.txt', ]; group('standard config', () { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); setUpAll(() async { await configure(amplifyEnvironments['main']!); - - for (final path in uploadedPaths) { + for (var pathIndex = 0; + pathIndex < uploadedPaths.length ~/ 2; + pathIndex++) { await Amplify.Storage.uploadData( - path: StoragePath.fromString(path), + path: StoragePath.fromString(uploadedPaths[pathIndex]), data: StorageDataPayload.bytes('test content'.codeUnits), + options: StorageUploadDataOptions( + bucket: mainBucket, + ), + ).result; + } + for (var pathIndex = uploadedPaths.length ~/ 2; + pathIndex < uploadedPaths.length; + pathIndex++) { + await Amplify.Storage.uploadData( + path: StoragePath.fromString(uploadedPaths[pathIndex]), + data: StorageDataPayload.bytes('test content'.codeUnits), + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), ).result; } - for (final path in uploadedPaths) { addTearDownPath(StoragePath.fromString(path)); } @@ -39,13 +62,31 @@ void main() { group('list() without options', () { testWidgets('should list all files with unique prefix', (_) async { - final listResult = await Amplify.Storage.list( + // this will use the main bucket by default when no optional bucket is specified + final listResultMainBucket = await Amplify.Storage.list( path: StoragePath.fromString(uniquePrefix), ).result; - - for (final uploadedPath in uploadedPaths) { + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString(uniquePrefix), + options: StorageListOptions( + bucket: secondaryBucket, + ), + ).result; + for (var pathIndex = 0; + pathIndex < uploadedPaths.length ~/ 2; + pathIndex++) { + expect( + listResultMainBucket.items + .any((item) => item.path == uploadedPaths[pathIndex]), + isTrue, + ); + } + for (var pathIndex = uploadedPaths.length ~/ 2; + pathIndex < uploadedPaths.length; + pathIndex++) { expect( - listResult.items.any((item) => item.path == uploadedPath), + listResultSecondaryBucket.items + .any((item) => item.path == uploadedPaths[pathIndex]), isTrue, ); } @@ -101,6 +142,17 @@ void main() { ), ).result as S3ListResult; + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString('$uniquePrefix/'), + options: StorageListOptions( + pluginOptions: const S3ListPluginOptions( + excludeSubPaths: true, + delimiter: '#', + ), + bucket: secondaryBucket, + ), + ).result as S3ListResult; + expect(listResult.items.length, 3); expect(listResult.items.first.path, contains('file1.txt')); @@ -110,6 +162,19 @@ void main() { '$uniquePrefix/subdir2#', ); expect(listResult.metadata.delimiter, '#'); + + expect(listResultSecondaryBucket.items.length, 3); + expect( + listResultSecondaryBucket.items.first.path, + contains('file5.txt'), + ); + + expect(listResultSecondaryBucket.metadata.subPaths.length, 1); + expect( + listResultSecondaryBucket.metadata.subPaths.first, + '$uniquePrefix/subdir4#', + ); + expect(listResultSecondaryBucket.metadata.delimiter, '#'); }); }); @@ -123,6 +188,20 @@ void main() { expect(listResult.items.length, 2); expect(listResult.items.first.path, contains('file1.txt')); + + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString(uniquePrefix), + options: StorageListOptions( + pageSize: 2, + bucket: secondaryBucket, + ), + ).result; + + expect(listResultSecondaryBucket.items.length, 2); + expect( + listResultSecondaryBucket.items.first.path, + contains('file5.txt'), + ); }); testWidgets('should list files with pagination', (_) async { @@ -157,8 +236,22 @@ void main() { ), ).result; - expect(listResult.items.length, uploadedPaths.length); + expect(listResult.items.length, uploadedPaths.length ~/ 2); expect(listResult.nextToken, isNull); + + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString(uniquePrefix), + options: StorageListOptions( + pluginOptions: const S3ListPluginOptions.listAll(), + bucket: secondaryBucket, + ), + ).result; + + expect( + listResultSecondaryBucket.items.length, + uploadedPaths.length ~/ 2, + ); + expect(listResultSecondaryBucket.nextToken, isNull); }); }); }); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 5512a934e2..41cbcc5df8 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -136,6 +136,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageListOptions( pluginOptions: s3PluginOptions, nextToken: options?.nextToken, + bucket: options?.bucket, pageSize: options?.pageSize ?? 1000, ); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 9140551535..783eed55c2 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -125,11 +125,12 @@ class StorageS3Service { const S3ListPluginOptions(); final resolvedPath = await _pathResolver.resolvePath(path: path); + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); if (!s3PluginOptions.listAll) { final request = s3.ListObjectsV2Request.build((builder) { builder - ..bucket = _storageOutputs.bucketName + ..bucket = s3ClientInfo.bucketName ..prefix = resolvedPath ..maxKeys = options.pageSize ..continuationToken = options.nextToken @@ -140,7 +141,7 @@ class StorageS3Service { try { return S3ListResult.fromPaginatedResult( - await _defaultS3Client.listObjectsV2(request).result, + await s3ClientInfo.client.listObjectsV2(request).result, ); } on smithy.UnknownSmithyHttpException catch (error) { // S3Client.headObject may return 403 error @@ -156,14 +157,14 @@ class StorageS3Service { try { final request = s3.ListObjectsV2Request.build((builder) { builder - ..bucket = _storageOutputs.bucketName + ..bucket = s3ClientInfo.bucketName ..prefix = resolvedPath ..delimiter = s3PluginOptions.excludeSubPaths ? s3PluginOptions.delimiter : null; }); - listResult = await _defaultS3Client.listObjectsV2(request).result; + listResult = await s3ClientInfo.client.listObjectsV2(request).result; recursiveResult = S3ListResult.fromPaginatedResult( listResult, ); diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 5d01e5736c..239b97187b 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -140,6 +140,9 @@ void main() { const testOptions = StorageListOptions( pluginOptions: S3ListPluginOptions(excludeSubPaths: true), nextToken: 'next-token-123', + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), pageSize: 2, );