feat(storage): multi bucket copy (#5674)

* chore(core): Added toJson to storage option buckets * feat(storage): multi bucket copy
aws-amplify · Nov 20, 2024 · 9e63b54 · 9e63b54
1 parent 1d9c618
commit 9e63b54
Show file tree

Hide file tree

Showing 18 changed files with 159 additions and 20 deletions.
diff --git a/packages/amplify_core/lib/src/types/storage/bucket_info.dart b/packages/amplify_core/lib/src/types/storage/bucket_info.dart
@@ -3,7 +3,8 @@ import 'package:amplify_core/amplify_core.dart';
 /// {@template amplify_core.storage.bucket_info}
 /// Presents a storage bucket information.
 /// {@endtemplate}
-class BucketInfo with AWSEquatable<BucketInfo> {
+class BucketInfo
+    with AWSEquatable<BucketInfo>, AWSSerializable<Map<String, Object?>> {
   /// {@macro amplify_core.storage.bucket_info}
   const BucketInfo({required this.bucketName, required this.region});
   final String bucketName;
@@ -14,4 +15,10 @@ class BucketInfo with AWSEquatable<BucketInfo> {
         bucketName,
         region,
       ];
+
+  @override
+  Map<String, Object?> toJson() => {
+        'bucketName': bucketName,
+        'region': region,
+      };
 }
diff --git a/packages/amplify_core/lib/src/types/storage/copy_buckets.dart b/packages/amplify_core/lib/src/types/storage/copy_buckets.dart
@@ -0,0 +1,24 @@
+import 'package:amplify_core/amplify_core.dart';
+
+/// Presents storage buckets for a copy operation.
+class CopyBuckets with AWSSerializable<Map<String, Object?>> {
+  /// Creates a [CopyBuckets] with [source] and [destination] buckets.
+  const CopyBuckets({
+    required this.source,
+    required this.destination,
+  });
+
+  /// Creates a [CopyBuckets] with the same [bucket] for the [source] and [destination].
+  CopyBuckets.sameBucket(StorageBucket bucket)
+      : source = bucket,
+        destination = bucket;
+
+  final StorageBucket source;
+  final StorageBucket destination;
+
+  @override
+  Map<String, Object?> toJson() => {
+        'source': source.toJson(),
+        'destination': destination.toJson(),
+      };
+}
diff --git a/packages/amplify_core/lib/src/types/storage/copy_options.dart b/packages/amplify_core/lib/src/types/storage/copy_options.dart
@@ -1,7 +1,7 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import 'package:aws_common/aws_common.dart';
+import 'package:amplify_core/amplify_core.dart';
 
 /// {@template amplify_core.storage.copy_options}
 /// Configurable options for `Amplify.Storage.copy`.
@@ -12,20 +12,27 @@ class StorageCopyOptions
         AWSSerializable<Map<String, Object?>>,
         AWSDebuggable {
   /// {@macro amplify_core.storage.copy_options}
-  const StorageCopyOptions({this.pluginOptions});
+  const StorageCopyOptions({
+    this.pluginOptions,
+    this.buckets,
+  });
 
   /// plugin specific options for `Amplify.Storage.copy`.
   final StorageCopyPluginOptions? pluginOptions;
 
+  /// Optionally specify which buckets to target
+  final CopyBuckets? buckets;
+
   @override
-  List<Object?> get props => [pluginOptions];
+  List<Object?> get props => [pluginOptions, buckets];
 
   @override
   String get runtimeTypeName => 'StorageCopyOptions';
 
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
+        'buckets': buckets?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/download_data_options.dart b/packages/amplify_core/lib/src/types/storage/download_data_options.dart
@@ -32,7 +32,7 @@ class StorageDownloadDataOptions
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/download_file_options.dart b/packages/amplify_core/lib/src/types/storage/download_file_options.dart
@@ -32,7 +32,7 @@ class StorageDownloadFileOptions
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart
@@ -32,7 +32,7 @@ class StorageGetPropertiesOptions
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/get_url_options.dart b/packages/amplify_core/lib/src/types/storage/get_url_options.dart
@@ -35,7 +35,7 @@ class StorageGetUrlOptions
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/list_options.dart b/packages/amplify_core/lib/src/types/storage/list_options.dart
@@ -41,7 +41,7 @@ class StorageListOptions
   Map<String, Object?> toJson() => {
         'pageSize': pageSize,
         'nextToken': nextToken,
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
         'pluginOptions': pluginOptions?.toJson(),
       };
 }

diff --git a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart
@@ -35,7 +35,7 @@ class StorageRemoveManyOptions
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/remove_options.dart b/packages/amplify_core/lib/src/types/storage/remove_options.dart
@@ -32,7 +32,7 @@ class StorageRemoveOptions
   @override
   Map<String, Object?> toJson() => {
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart
@@ -1,10 +1,10 @@
+import 'package:amplify_core/amplify_core.dart';
 import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart';
-import 'package:amplify_core/src/types/storage/bucket_info.dart';
 import 'package:amplify_core/src/types/storage/storage_bucket_from_outputs.dart';
 import 'package:meta/meta.dart';
 
 /// Presents a storage bucket.
-class StorageBucket {
+class StorageBucket with AWSSerializable<Map<String, Object?>> {
   /// Creates a [StorageBucket] from [BucketInfo].
   const StorageBucket.fromBucketInfo(this._info);
 
@@ -16,4 +16,9 @@ class StorageBucket {
 
   @internal
   BucketInfo resolveBucketInfo(StorageOutputs? storageOutputs) => _info;
+
+  @override
+  Map<String, Object?> toJson() => {
+        '_info': _info.toJson(),
+      };
 }
diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart
@@ -40,4 +40,9 @@ class StorageBucketFromOutputs implements StorageBucket {
       region: bucket.awsRegion,
     );
   }
+
+  @override
+  Map<String, Object?> toJson() => {
+        '_name': _name,
+      };
 }
diff --git a/packages/amplify_core/lib/src/types/storage/storage_types.dart b/packages/amplify_core/lib/src/types/storage/storage_types.dart
@@ -12,6 +12,7 @@ export '../exception/amplify_exception.dart'
         NetworkException,
         UnknownException;
 export 'bucket_info.dart';
+export 'copy_buckets.dart';
 export 'copy_operation.dart';
 export 'copy_options.dart';
 export 'copy_request.dart';

diff --git a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart
@@ -37,7 +37,7 @@ class StorageUploadDataOptions
   Map<String, Object?> toJson() => {
         'metadata': metadata,
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart
@@ -37,7 +37,7 @@ class StorageUploadFileOptions
   Map<String, Object?> toJson() => {
         'metadata': metadata,
         'pluginOptions': pluginOptions?.toJson(),
-        'bucket': bucket,
+        'bucket': bucket?.toJson(),
       };
 }
 

diff --git a/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart
@@ -138,5 +138,90 @@ void main() {
         expect(result.copiedItem.path, destinationPath);
       });
     });
+
+    group('multi bucket', () {
+      final data = 'copy data'.codeUnits;
+      final bucket1 = StorageBucket.fromOutputs(
+        'Storage Integ Test main bucket',
+      );
+      final bucket2 = StorageBucket.fromOutputs(
+        'Storage Integ Test secondary bucket',
+      );
+      final bucket1PathSource = 'public/multi-bucket-get-url-${uuid()}';
+      final bucket2PathSource = 'public/multi-bucket-get-url-${uuid()}';
+      final bucket2PathDestination = 'public/multi-bucket-get-url-${uuid()}';
+      final storageBucket1PathSource =
+          StoragePath.fromString(bucket1PathSource);
+      final storageBucket2PathSource =
+          StoragePath.fromString(bucket2PathSource);
+      final storageBucket2PathDestination =
+          StoragePath.fromString(bucket2PathDestination);
+
+      setUp(() async {
+        await configure(amplifyEnvironments['main']!);
+        addTearDownPath(storageBucket1PathSource);
+        addTearDownPath(storageBucket2PathSource);
+        addTearDownPath(storageBucket2PathDestination);
+        await Amplify.Storage.uploadData(
+          data: StorageDataPayload.bytes(data),
+          path: storageBucket1PathSource,
+          options: StorageUploadDataOptions(
+            bucket: bucket1,
+          ),
+        ).result;
+        await Amplify.Storage.uploadData(
+          data: StorageDataPayload.bytes(data),
+          path: storageBucket2PathSource,
+          options: StorageUploadDataOptions(
+            bucket: bucket2,
+          ),
+        ).result;
+      });
+
+      testWidgets('copy to a different bucket', (_) async {
+        final result = await Amplify.Storage.copy(
+          source: storageBucket1PathSource,
+          destination: storageBucket2PathDestination,
+          options: StorageCopyOptions(
+            buckets: CopyBuckets(
+              source: bucket1,
+              destination: bucket2,
+            ),
+          ),
+        ).result;
+        expect(result.copiedItem.path, bucket2PathDestination);
+
+        final downloadResult = await Amplify.Storage.downloadData(
+          path: storageBucket2PathDestination,
+          options: StorageDownloadDataOptions(bucket: bucket2),
+        ).result;
+        expect(
+          downloadResult.bytes,
+          data,
+        );
+      });
+
+      testWidgets('copy to the same bucket', (_) async {
+        final result = await Amplify.Storage.copy(
+          source: storageBucket2PathSource,
+          destination: storageBucket2PathDestination,
+          options: StorageCopyOptions(
+            buckets: CopyBuckets.sameBucket(
+              bucket2,
+            ),
+          ),
+        ).result;
+        expect(result.copiedItem.path, bucket2PathDestination);
+
+        final downloadResult = await Amplify.Storage.downloadData(
+          path: storageBucket2PathDestination,
+          options: StorageDownloadDataOptions(bucket: bucket2),
+        ).result;
+        expect(
+          downloadResult.bytes,
+          data,
+        );
+      });
+    });
   });
 }
diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart
@@ -364,6 +364,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface
 
     final s3Options = StorageCopyOptions(
       pluginOptions: s3PluginOptions,
+      buckets: options?.buckets,
     );
 
     return S3CopyOperation(

diff --git a/...e/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/...e/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart
@@ -412,21 +412,25 @@ class StorageS3Service {
   }) async {
     final s3PluginOptions = options.pluginOptions as S3CopyPluginOptions? ??
         const S3CopyPluginOptions();
+    final s3ClientInfoSource =
+        getS3ClientInfo(storageBucket: options.buckets?.source);
+    final s3ClientInfoDestination =
+        getS3ClientInfo(storageBucket: options.buckets?.destination);
 
     final [sourcePath, destinationPath] = await _pathResolver.resolvePaths(
       paths: [source, destination],
     );
 
     final copyRequest = s3.CopyObjectRequest.build((builder) {
       builder
-        ..bucket = _storageOutputs.bucketName
-        ..copySource = '${_storageOutputs.bucketName}/$sourcePath'
+        ..bucket = s3ClientInfoDestination.bucketName
+        ..copySource = '${s3ClientInfoSource.bucketName}/$sourcePath'
         ..key = destinationPath
         ..metadataDirective = s3.MetadataDirective.copy;
     });
 
     try {
-      await _defaultS3Client.copyObject(copyRequest).result;
+      await s3ClientInfoDestination.client.copyObject(copyRequest).result;
     } on smithy.UnknownSmithyHttpException catch (error) {
       // S3Client.copyObject may return 403 or 404 error
       throw error.toStorageException();
@@ -438,8 +442,8 @@ class StorageS3Service {
       copiedItem: s3PluginOptions.getProperties
           ? S3Item.fromHeadObjectOutput(
               await headObject(
-                s3client: _defaultS3Client,
-                bucket: _storageOutputs.bucketName,
+                s3client: s3ClientInfoDestination.client,
+                bucket: s3ClientInfoDestination.bucketName,
                 key: destinationPath,
               ),
               path: destinationPath,