Merge pull request #64 from gocardless/allow-file-uploads

Support streaming uploads and downloads

CHANGELOG.md

@@ -1,3 +1,8 @@
+v0.6.0
+------
+- [BREAKING] Remove support for Ruby 2.6
+- Add support for streaming uploads / downloads
+
 v0.5.0
 ------
 - Support escaping of URI keys. Fixes #46.

README.md

@@ -135,18 +135,34 @@ in mind:
 
 ## Examples
 
-### Uploading a file to a bucket
+### Uploading a string to a bucket
 ```ruby
 BucketStore.for("inmemory://bucket/path/file.xml").upload!("hello world")
 => "inmemory://bucket/path/file.xml"
 ```
 
-### Accessing a file in a bucket
+### Accessing a string in a bucket
 ```ruby
 BucketStore.for("inmemory://bucket/path/file.xml").download
 => {:bucket=>"bucket", :key=>"path/file.xml", :content=>"hello world"}
 ```
 
+### Uploading a file-like object to a bucket
+```ruby
+buffer = StringIO.new("This could also be an actual file")
+BucketStore.for("inmemory://bucket/path/file.xml").stream.upload!(file: buffer)
+=> "inmemory://bucket/path/file.xml"
+```
+
+### Downloading to a file-like object from a bucket
+```ruby
+buffer = StringIO.new
+BucketStore.for("inmemory://bucket/path/file.xml").stream.download(file: buffer)
+=> {:bucket=>"bucket", :key=>"path/file.xml", :file=>buffer}
+buffer.string 
+=> "This could also be an actual file"
+```
+
 ### Listing all keys under a prefix
 ```ruby
 BucketStore.for("inmemory://bucket/path/").list

docker-compose.yml

@@ -12,6 +12,7 @@ services:
       MINIO_REGION_NAME: us-east-1
     ports:
       - "9000:9000"
+      - "9001:9001"
   gcp-simulator:
     image: fsouza/fake-gcs-server
     hostname: gcp

lib/bucket_store/disk.rb

@@ -13,23 +13,22 @@ def initialize(base_dir)
       @base_dir = File.expand_path(base_dir)
     end
 
-    def upload!(bucket:, key:, content:)
-      File.open(key_path(bucket, key), "w") do |file|
-        file.write(content)
+    def upload!(bucket:, key:, file:)
+      File.open(key_path(bucket, key), "w") do |output_file|
+        output_file.write(file.read)
+        output_file.rewind
       end
+
       {
         bucket: bucket,
         key: key,
       }
     end
 
-    def download(bucket:, key:)
-      File.open(key_path(bucket, key), "r") do |file|
-        {
-          bucket: bucket,
-          key: key,
-          content: file.read,
-        }
+    def download(bucket:, key:, file:)
+      File.open(key_path(bucket, key), "r") do |saved_file|
+        file.write(saved_file.read)
+        file.rewind
       end
     end
 

lib/bucket_store/gcs.rb

@@ -33,27 +33,23 @@ def initialize(timeout_seconds)
                  end
     end
 
-    def upload!(bucket:, key:, content:)
-      buffer = StringIO.new(content)
-      get_bucket(bucket).create_file(buffer, key)
+    def upload!(bucket:, key:, file:)
+      get_bucket(bucket).create_file(file, key)
 
       {
         bucket: bucket,
         key: key,
       }
     end
 
-    def download(bucket:, key:)
-      file = get_bucket(bucket).file(key)
+    def download(bucket:, key:, file:)
+      file.tap do |f|
+        get_bucket(bucket).
+          file(key).
+          download(f)
 
-      buffer = StringIO.new
-      file.download(buffer)
-
-      {
-        bucket: bucket,
-        key: key,
-        content: buffer.string,
-      }
+        f.rewind
+      end
     end
 
     def list(bucket:, key:, page_size:)

lib/bucket_store/in_memory.rb

@@ -24,21 +24,22 @@ def reset!
       @buckets = Hash.new { |hash, key| hash[key] = {} }
     end
 
-    def upload!(bucket:, key:, content:)
-      @buckets[bucket][key] = content
+    def upload!(bucket:, key:, file:)
+      file.tap do |f|
+        @buckets[bucket][key] = f.read
+      end
 
       {
         bucket: bucket,
         key: key,
       }
     end
 
-    def download(bucket:, key:)
-      {
-        bucket: bucket,
-        key: key,
-        content: @buckets[bucket].fetch(key),
-      }
+    def download(bucket:, key:, file:)
+      file.tap do |f|
+        f.write(@buckets[bucket].fetch(key))
+        f.rewind
+      end
     end
 
     def list(bucket:, key:, page_size:)

lib/bucket_store/key_storage.rb

@@ -15,6 +15,90 @@ class KeyStorage
       disk: Disk,
     }.freeze
 
+    # Defines a streaming interface for download and upload operations.
+    #
+    # Note that individual adapters may require additional configuration for the correct
+    # behavior of the streaming interface.
+    class KeyStreamer
+      attr_reader :bucket, :key, :adapter_type
+
+      def initialize(adapter:, adapter_type:, bucket:, key:)
+        @adapter = adapter
+        @adapter_type = adapter_type
+        @bucket = bucket
+        @key = key
+      end
+
+      # Streams the content of the reference key into a file-like object
+      # @param [IO] file a writeable IO instance, or a file-like object such as `StringIO`
+      # @return hash containing the bucket, the key and file like object passed in as input
+      #
+      # @see KeyStorage#download
+      # @example Download a key
+      #   buffer = StringIO.new
+      #   BucketStore.for("inmemory://bucket/file.xml").stream.download(file: buffer)
+      #   buffer.string == "Imagine I'm a 2GB file"
+      def download(file:)
+        BucketStore.logger.info(event: "key_storage.download_started")
+
+        start = BucketStore::Timing.monotonic_now
+        adapter.download(
+          bucket: bucket,
+          key: key,
+          file: file,
+        )
+
+        BucketStore.logger.info(event: "key_storage.download_finished",
+                                duration: BucketStore::Timing.monotonic_now - start)
+
+        {
+          bucket: bucket,
+          key: key,
+          file: file,
+        }
+      end
+
+      # Performs a streaming upload to the backing object store
+      # @param [IO] file a readable IO instance, or a file-like object such as `StringIO`
+      # @return the generated key for the new object
+      #
+      # @see KeyStorage#upload!
+      # @example Upload a key
+      #   buffer = StringIO.new("Imagine I'm a 2GB file")
+      #   BucketStore.for("inmemory://bucket/file.xml").stream.upload!(file: buffer)
+      def upload!(file:)
+        raise ArgumentError, "Key cannot be empty" if key.empty?
+
+        BucketStore.logger.info(event: "key_storage.upload_started",
+                                **log_context)
+
+        start = BucketStore::Timing.monotonic_now
+        adapter.upload!(
+          bucket: bucket,
+          key: key,
+          file: file,
+        )
+
+        BucketStore.logger.info(event: "key_storage.upload_finished",
+                                duration: BucketStore::Timing.monotonic_now - start,
+                                **log_context)
+
+        "#{adapter_type}://#{bucket}/#{key}"
+      end
+
+      private
+
+      attr_reader :adapter
+
+      def log_context
+        {
+          bucket: bucket,
+          key: key,
+          adapter_type: adapter_type,
+        }.compact
+      end
+    end
+
     attr_reader :bucket, :key, :adapter_type
 
     def initialize(adapter:, bucket:, key:)
@@ -40,45 +124,32 @@ def filename
     # @example Download a key
     #   BucketStore.for("inmemory://bucket/file.xml").download
     def download
-      raise ArgumentError, "Key cannot be empty" if key.empty?
-
-      BucketStore.logger.info(event: "key_storage.download_started")
-
-      start = BucketStore::Timing.monotonic_now
-      result = adapter.download(bucket: bucket, key: key)
-
-      BucketStore.logger.info(event: "key_storage.download_finished",
-                              duration: BucketStore::Timing.monotonic_now - start)
-
-      result
+      buffer = StringIO.new
+      stream.download(file: buffer).tap do |result|
+        result.delete(:file)
+        result[:content] = buffer.string
+      end
     end
 
-    # Uploads the given content to the reference key location.
+    # Uploads the given file to the reference key location.
     #
     # If the `key` already exists, its content will be replaced by the one in input.
     #
-    # @param [String] content The content to upload
+    # @param [String] content Contents of the file
     # @return [String] The final `key` where the content has been uploaded
     # @example Upload a file
-    #   BucketStore.for("inmemory://bucket/file.xml").upload("hello world")
+    #   BucketStore.for("inmemory://bucket/file.xml").upload!("hello world")
     def upload!(content)
-      raise ArgumentError, "Key cannot be empty" if key.empty?
-
-      BucketStore.logger.info(event: "key_storage.upload_started",
-                              **log_context)
-
-      start = BucketStore::Timing.monotonic_now
-      result = adapter.upload!(
-        bucket: bucket,
-        key: key,
-        content: content,
-      )
+      stream.upload!(file: StringIO.new(content))
+    end
 
-      BucketStore.logger.info(event: "key_storage.upload_finished",
-                              duration: BucketStore::Timing.monotonic_now - start,
-                              **log_context)
+    # Returns an interface for streaming operations
+    #
+    # @return [KeyStreamer] An interface for streaming operations
+    def stream
+      raise ArgumentError, "Key cannot be empty" if key.empty?
 
-      "#{adapter_type}://#{result[:bucket]}/#{result[:key]}"
+      KeyStreamer.new(adapter: adapter, adapter_type: adapter_type, bucket: bucket, key: key)
     end
 
     # Lists all keys for the current adapter that have the reference key as prefix
@@ -159,13 +230,5 @@ def exists?
     private
 
     attr_reader :adapter
-
-    def log_context
-      {
-        bucket: bucket,
-        key: key,
-        adapter_type: adapter_type,
-      }.compact
-    end
   end
 end

lib/bucket_store/s3.rb

@@ -20,11 +20,11 @@ def initialize(open_timeout_seconds, read_timeout_seconds)
       )
     end
 
-    def upload!(bucket:, key:, content:)
+    def upload!(bucket:, key:, file:)
       storage.put_object(
         bucket: bucket,
         key: key,
-        body: content,
+        body: file,
       )
 
       {
@@ -33,17 +33,12 @@ def upload!(bucket:, key:, content:)
       }
     end
 
-    def download(bucket:, key:)
-      file = storage.get_object(
+    def download(bucket:, key:, file:)
+      storage.get_object(
+        response_target: file,
         bucket: bucket,
         key: key,
       )
-
-      {
-        bucket: bucket,
-        key: key,
-        content: file.body.read,
-      }
     end
 
     def list(bucket:, key:, page_size:)

lib/bucket_store/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BucketStore
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end