From 9e63b54d236c718f9164c8681167e826727bb259 Mon Sep 17 00:00:00 2001 From: Tyler-Larkin Date: Tue, 19 Nov 2024 15:41:11 -0800 Subject: [PATCH] feat(storage): multi bucket copy (#5674) * chore(core): Added toJson to storage option buckets * feat(storage): multi bucket copy --- .../lib/src/types/storage/bucket_info.dart | 9 +- .../lib/src/types/storage/copy_buckets.dart | 24 ++++++ .../lib/src/types/storage/copy_options.dart | 13 ++- .../types/storage/download_data_options.dart | 2 +- .../types/storage/download_file_options.dart | 2 +- .../types/storage/get_properties_options.dart | 2 +- .../src/types/storage/get_url_options.dart | 2 +- .../lib/src/types/storage/list_options.dart | 2 +- .../types/storage/remove_many_options.dart | 2 +- .../lib/src/types/storage/remove_options.dart | 2 +- .../lib/src/types/storage/storage_bucket.dart | 9 +- .../storage/storage_bucket_from_outputs.dart | 5 ++ .../lib/src/types/storage/storage_types.dart | 1 + .../types/storage/upload_data_options.dart | 2 +- .../types/storage/upload_file_options.dart | 2 +- .../example/integration_test/copy_test.dart | 85 +++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 14 +-- 18 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 packages/amplify_core/lib/src/types/storage/copy_buckets.dart diff --git a/packages/amplify_core/lib/src/types/storage/bucket_info.dart b/packages/amplify_core/lib/src/types/storage/bucket_info.dart index a30811b333..811871f9b3 100644 --- a/packages/amplify_core/lib/src/types/storage/bucket_info.dart +++ b/packages/amplify_core/lib/src/types/storage/bucket_info.dart @@ -3,7 +3,8 @@ import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.bucket_info} /// Presents a storage bucket information. /// {@endtemplate} -class BucketInfo with AWSEquatable { +class BucketInfo + with AWSEquatable, AWSSerializable> { /// {@macro amplify_core.storage.bucket_info} const BucketInfo({required this.bucketName, required this.region}); final String bucketName; @@ -14,4 +15,10 @@ class BucketInfo with AWSEquatable { bucketName, region, ]; + + @override + Map toJson() => { + 'bucketName': bucketName, + 'region': region, + }; } diff --git a/packages/amplify_core/lib/src/types/storage/copy_buckets.dart b/packages/amplify_core/lib/src/types/storage/copy_buckets.dart new file mode 100644 index 0000000000..7bfb83575d --- /dev/null +++ b/packages/amplify_core/lib/src/types/storage/copy_buckets.dart @@ -0,0 +1,24 @@ +import 'package:amplify_core/amplify_core.dart'; + +/// Presents storage buckets for a copy operation. +class CopyBuckets with AWSSerializable> { + /// Creates a [CopyBuckets] with [source] and [destination] buckets. + const CopyBuckets({ + required this.source, + required this.destination, + }); + + /// Creates a [CopyBuckets] with the same [bucket] for the [source] and [destination]. + CopyBuckets.sameBucket(StorageBucket bucket) + : source = bucket, + destination = bucket; + + final StorageBucket source; + final StorageBucket destination; + + @override + Map toJson() => { + 'source': source.toJson(), + 'destination': destination.toJson(), + }; +} diff --git a/packages/amplify_core/lib/src/types/storage/copy_options.dart b/packages/amplify_core/lib/src/types/storage/copy_options.dart index eacd7f4ff1..a9910ecfff 100644 --- a/packages/amplify_core/lib/src/types/storage/copy_options.dart +++ b/packages/amplify_core/lib/src/types/storage/copy_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.copy_options} /// Configurable options for `Amplify.Storage.copy`. @@ -12,13 +12,19 @@ class StorageCopyOptions AWSSerializable>, AWSDebuggable { /// {@macro amplify_core.storage.copy_options} - const StorageCopyOptions({this.pluginOptions}); + const StorageCopyOptions({ + this.pluginOptions, + this.buckets, + }); /// plugin specific options for `Amplify.Storage.copy`. final StorageCopyPluginOptions? pluginOptions; + /// Optionally specify which buckets to target + final CopyBuckets? buckets; + @override - List get props => [pluginOptions]; + List get props => [pluginOptions, buckets]; @override String get runtimeTypeName => 'StorageCopyOptions'; @@ -26,6 +32,7 @@ class StorageCopyOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'buckets': buckets?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/download_data_options.dart b/packages/amplify_core/lib/src/types/storage/download_data_options.dart index 020856dfe1..26a4ea2422 100644 --- a/packages/amplify_core/lib/src/types/storage/download_data_options.dart +++ b/packages/amplify_core/lib/src/types/storage/download_data_options.dart @@ -32,7 +32,7 @@ class StorageDownloadDataOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/download_file_options.dart b/packages/amplify_core/lib/src/types/storage/download_file_options.dart index 23d745f5e6..69be1758cd 100644 --- a/packages/amplify_core/lib/src/types/storage/download_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/download_file_options.dart @@ -32,7 +32,7 @@ class StorageDownloadFileOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart index dbc5b8c112..9a2216b40d 100644 --- a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart +++ b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart @@ -32,7 +32,7 @@ class StorageGetPropertiesOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/get_url_options.dart b/packages/amplify_core/lib/src/types/storage/get_url_options.dart index 62954e8281..161cc2b93b 100644 --- a/packages/amplify_core/lib/src/types/storage/get_url_options.dart +++ b/packages/amplify_core/lib/src/types/storage/get_url_options.dart @@ -35,7 +35,7 @@ class StorageGetUrlOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/list_options.dart b/packages/amplify_core/lib/src/types/storage/list_options.dart index f0a1d80962..72570f262b 100644 --- a/packages/amplify_core/lib/src/types/storage/list_options.dart +++ b/packages/amplify_core/lib/src/types/storage/list_options.dart @@ -41,7 +41,7 @@ class StorageListOptions Map toJson() => { 'pageSize': pageSize, 'nextToken': nextToken, - 'bucket': bucket, + 'bucket': bucket?.toJson(), 'pluginOptions': pluginOptions?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart index 05db2da027..a50c651fc8 100644 --- a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart +++ b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart @@ -35,7 +35,7 @@ class StorageRemoveManyOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/remove_options.dart b/packages/amplify_core/lib/src/types/storage/remove_options.dart index 95be578c1c..13e68cb501 100644 --- a/packages/amplify_core/lib/src/types/storage/remove_options.dart +++ b/packages/amplify_core/lib/src/types/storage/remove_options.dart @@ -32,7 +32,7 @@ class StorageRemoveOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart index 711e30d18b..ccfdd345a9 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_bucket.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart @@ -1,10 +1,10 @@ +import 'package:amplify_core/amplify_core.dart'; import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart'; -import 'package:amplify_core/src/types/storage/bucket_info.dart'; import 'package:amplify_core/src/types/storage/storage_bucket_from_outputs.dart'; import 'package:meta/meta.dart'; /// Presents a storage bucket. -class StorageBucket { +class StorageBucket with AWSSerializable> { /// Creates a [StorageBucket] from [BucketInfo]. const StorageBucket.fromBucketInfo(this._info); @@ -16,4 +16,9 @@ class StorageBucket { @internal BucketInfo resolveBucketInfo(StorageOutputs? storageOutputs) => _info; + + @override + Map toJson() => { + '_info': _info.toJson(), + }; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart index ed22c24aa2..b51d17029d 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart @@ -40,4 +40,9 @@ class StorageBucketFromOutputs implements StorageBucket { region: bucket.awsRegion, ); } + + @override + Map toJson() => { + '_name': _name, + }; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_types.dart b/packages/amplify_core/lib/src/types/storage/storage_types.dart index 1e4bfbfb02..b00b66a362 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_types.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_types.dart @@ -12,6 +12,7 @@ export '../exception/amplify_exception.dart' NetworkException, UnknownException; export 'bucket_info.dart'; +export 'copy_buckets.dart'; export 'copy_operation.dart'; export 'copy_options.dart'; export 'copy_request.dart'; diff --git a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart index 3b7bd9eb7b..83e8f5009c 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart @@ -37,7 +37,7 @@ class StorageUploadDataOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart index 6b025257fb..2422eef043 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart @@ -37,7 +37,7 @@ class StorageUploadFileOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart index 87afda83b3..12e892ea06 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart @@ -138,5 +138,90 @@ void main() { expect(result.copiedItem.path, destinationPath); }); }); + + group('multi bucket', () { + final data = 'copy data'.codeUnits; + final bucket1 = StorageBucket.fromOutputs( + 'Storage Integ Test main bucket', + ); + final bucket2 = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + final bucket1PathSource = 'public/multi-bucket-get-url-${uuid()}'; + final bucket2PathSource = 'public/multi-bucket-get-url-${uuid()}'; + final bucket2PathDestination = 'public/multi-bucket-get-url-${uuid()}'; + final storageBucket1PathSource = + StoragePath.fromString(bucket1PathSource); + final storageBucket2PathSource = + StoragePath.fromString(bucket2PathSource); + final storageBucket2PathDestination = + StoragePath.fromString(bucket2PathDestination); + + setUp(() async { + await configure(amplifyEnvironments['main']!); + addTearDownPath(storageBucket1PathSource); + addTearDownPath(storageBucket2PathSource); + addTearDownPath(storageBucket2PathDestination); + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storageBucket1PathSource, + options: StorageUploadDataOptions( + bucket: bucket1, + ), + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storageBucket2PathSource, + options: StorageUploadDataOptions( + bucket: bucket2, + ), + ).result; + }); + + testWidgets('copy to a different bucket', (_) async { + final result = await Amplify.Storage.copy( + source: storageBucket1PathSource, + destination: storageBucket2PathDestination, + options: StorageCopyOptions( + buckets: CopyBuckets( + source: bucket1, + destination: bucket2, + ), + ), + ).result; + expect(result.copiedItem.path, bucket2PathDestination); + + final downloadResult = await Amplify.Storage.downloadData( + path: storageBucket2PathDestination, + options: StorageDownloadDataOptions(bucket: bucket2), + ).result; + expect( + downloadResult.bytes, + data, + ); + }); + + testWidgets('copy to the same bucket', (_) async { + final result = await Amplify.Storage.copy( + source: storageBucket2PathSource, + destination: storageBucket2PathDestination, + options: StorageCopyOptions( + buckets: CopyBuckets.sameBucket( + bucket2, + ), + ), + ).result; + expect(result.copiedItem.path, bucket2PathDestination); + + final downloadResult = await Amplify.Storage.downloadData( + path: storageBucket2PathDestination, + options: StorageDownloadDataOptions(bucket: bucket2), + ).result; + expect( + downloadResult.bytes, + data, + ); + }); + }); }); } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index f754941b43..11a2c9bc9c 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -364,6 +364,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageCopyOptions( pluginOptions: s3PluginOptions, + buckets: options?.buckets, ); return S3CopyOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 05b71ea05f..d3d8939f39 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -412,6 +412,10 @@ class StorageS3Service { }) async { final s3PluginOptions = options.pluginOptions as S3CopyPluginOptions? ?? const S3CopyPluginOptions(); + final s3ClientInfoSource = + getS3ClientInfo(storageBucket: options.buckets?.source); + final s3ClientInfoDestination = + getS3ClientInfo(storageBucket: options.buckets?.destination); final [sourcePath, destinationPath] = await _pathResolver.resolvePaths( paths: [source, destination], @@ -419,14 +423,14 @@ class StorageS3Service { final copyRequest = s3.CopyObjectRequest.build((builder) { builder - ..bucket = _storageOutputs.bucketName - ..copySource = '${_storageOutputs.bucketName}/$sourcePath' + ..bucket = s3ClientInfoDestination.bucketName + ..copySource = '${s3ClientInfoSource.bucketName}/$sourcePath' ..key = destinationPath ..metadataDirective = s3.MetadataDirective.copy; }); try { - await _defaultS3Client.copyObject(copyRequest).result; + await s3ClientInfoDestination.client.copyObject(copyRequest).result; } on smithy.UnknownSmithyHttpException catch (error) { // S3Client.copyObject may return 403 or 404 error throw error.toStorageException(); @@ -438,8 +442,8 @@ class StorageS3Service { copiedItem: s3PluginOptions.getProperties ? S3Item.fromHeadObjectOutput( await headObject( - s3client: _defaultS3Client, - bucket: _storageOutputs.bucketName, + s3client: s3ClientInfoDestination.client, + bucket: s3ClientInfoDestination.bucketName, key: destinationPath, ), path: destinationPath,