From 831dd05980592684fef04bc447f96702426e83f9 Mon Sep 17 00:00:00 2001 From: Joseph Southan Date: Mon, 8 Feb 2021 12:23:08 +0000 Subject: [PATCH] Support moving files between uris Bump version v0.3.0 --- .circleci/config.yml | 2 + .gitignore | 1 + CHANGELOG.md | 5 ++ README.md | 9 ++ lib/file_storage/disk.rb | 9 ++ lib/file_storage/gcs.rb | 20 ++++- lib/file_storage/in_memory.rb | 8 ++ lib/file_storage/key_storage.rb | 37 ++++++++ lib/file_storage/version.rb | 2 +- spec/file_storage/disk_spec.rb | 40 +++++++++ spec/file_storage/gcs_spec.rb | 120 ++++++++++++++++++++++++++ spec/file_storage/in_memory_spec.rb | 40 +++++++++ spec/file_storage/key_storage_spec.rb | 27 ++++++ 13 files changed, 315 insertions(+), 5 deletions(-) create mode 100644 spec/file_storage/gcs_spec.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index b105f64..bf382b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,8 @@ jobs: type: string docker: - image: circleci/ruby:<< parameters.ruby_version >> + - image: fsouza/fake-gcs-server + command: "-scheme http" steps: - add_ssh_keys - checkout diff --git a/.gitignore b/.gitignore index 4a45838..0752cb4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ tmp/ .bundle/ .rspec_status +.yardoc/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b706e..e5f13b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v0.3.0 +------ + +- Add `move!` to move files + v0.2.0 ------ diff --git a/README.md b/README.md index 22c8e90..96cfe52 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,15 @@ FileStorage.for("inmemory://bucket/path/").list => ["inmemory://bucket/path/file.xml"] ``` +### Moving a file + +_Note: Moving a file is only supported between the same adapter type_ + +```ruby +FileStorage.for("inmemory://bucket/path/file.xml").move!("inmemory://bucket/path/file2.xml") +=> "inmemory://bucket/path/file2.xml" +``` + ### Delete a file ```ruby FileStorage.for("inmemory://bucket/path/file.xml").delete! diff --git a/lib/file_storage/disk.rb b/lib/file_storage/disk.rb index 16543f4..470f563 100644 --- a/lib/file_storage/disk.rb +++ b/lib/file_storage/disk.rb @@ -55,6 +55,15 @@ def delete!(bucket:, key:) true end + def move!(bucket:, key:, new_bucket:, new_key:) + FileUtils.mv(key_path(bucket, key), key_path(new_bucket, new_key)) + + { + bucket: new_bucket, + key: new_key, + } + end + private attr_reader :base_dir diff --git a/lib/file_storage/gcs.rb b/lib/file_storage/gcs.rb index 4d8c969..263f634 100644 --- a/lib/file_storage/gcs.rb +++ b/lib/file_storage/gcs.rb @@ -9,13 +9,13 @@ module FileStorage class Gcs DEFAULT_TIMEOUT_SECONDS = 30 - def self.build(timeout_seconds = DEFAULT_TIMEOUT_SECONDS) - Gcs.new(timeout_seconds) + def self.build(timeout_seconds = DEFAULT_TIMEOUT_SECONDS, gcs_config = {}) + Gcs.new(timeout: timeout_seconds, **gcs_config) end - def initialize(timeout_seconds) + def initialize(**gcs_config) @storage = Google::Cloud::Storage.new( - timeout: timeout_seconds, + **gcs_config, ) end @@ -66,6 +66,18 @@ def delete!(bucket:, key:) true end + def move!(bucket:, key:, new_bucket:, new_key:) + old_file = get_bucket(bucket).file(key) + destination_bucket = get_bucket(new_bucket) + old_file.copy(destination_bucket.name, new_key) + old_file.delete + + { + bucket: new_bucket, + key: new_key, + } + end + private attr_reader :storage diff --git a/lib/file_storage/in_memory.rb b/lib/file_storage/in_memory.rb index 58759a2..43b08ba 100644 --- a/lib/file_storage/in_memory.rb +++ b/lib/file_storage/in_memory.rb @@ -58,5 +58,13 @@ def delete!(bucket:, key:) true end + + def move!(bucket:, key:, new_bucket:, new_key:) + @buckets[new_bucket][new_key] = @buckets.fetch(bucket).delete(key) + { + bucket: new_bucket, + key: new_key, + } + end end end diff --git a/lib/file_storage/key_storage.rb b/lib/file_storage/key_storage.rb index c8180ae..7c4dd01 100644 --- a/lib/file_storage/key_storage.rb +++ b/lib/file_storage/key_storage.rb @@ -121,6 +121,43 @@ def list(page_size: 1000) end end + # Moves the existing file to a new file path + # + # @param [String] new_key The new key to move the file to + # @return [String] A URI to the file's new path + # @example Move a file + # FileStorage.for("inmemory://bucket1/foo").move!("inmemory://bucket2/bar") + def move!(new_key) + raise ArgumentError, "Key cannot be empty" if key.empty? + + new_key_ctx = FileStorage.for(new_key) + + unless new_key_ctx.adapter_type == adapter_type + raise ArgumentError, "Adapter type must be the same" + end + raise ArgumentError, "Destination key cannot be empty" if new_key_ctx.key.empty? + + start = FileStorage::Timing.monotonic_now + result = adapter.move!( + bucket: bucket, + key: key, + new_bucket: new_key_ctx.bucket, + new_key: new_key_ctx.key, + ) + + old_key = "#{adapter_type}://#{bucket}/#{key}" + new_key = "#{adapter_type}://#{result[:bucket]}/#{result[:key]}" + + FileStorage.logger.info( + event: "key_storage.moved", + duration: FileStorage::Timing.monotonic_now - start, + old_key: old_key, + new_key: new_key, + ) + + new_key + end + # Deletes the referenced key. # # Note that this method will always return true. diff --git a/lib/file_storage/version.rb b/lib/file_storage/version.rb index 0471daf..152cbb5 100644 --- a/lib/file_storage/version.rb +++ b/lib/file_storage/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module FileStorage - VERSION = "0.2.0" + VERSION = "0.3.0" end diff --git a/spec/file_storage/disk_spec.rb b/spec/file_storage/disk_spec.rb index c9b2ab3..4b6ecee 100644 --- a/spec/file_storage/disk_spec.rb +++ b/spec/file_storage/disk_spec.rb @@ -156,4 +156,44 @@ to raise_error(Errno::ENOENT, /No such file or directory/) end end + + describe "#move!" do + subject(:move) do + instance.move!(bucket: bucket, key: key, new_bucket: new_bucket, new_key: new_key) + end + + let(:new_bucket) { "cake" } + + context "when the 'existing' file doesn't exist" do + let(:key) { "foobar" } + let(:new_key) { "barbaz" } + + it "raises a File error" do + expect { move }.to raise_error(Errno::ENOENT, /No such file or directory/) + end + end + + context "when the file does exist" do + let(:key) { "2021-02-08/hello1" } + let(:new_key) { "2021-02-08/hello2" } + let(:content) { "world" } + + before { instance.upload!(bucket: bucket, key: key, content: content) } + + it "moves the file" do + move + + expect(instance.download(bucket: new_bucket, key: new_key)[:content]).to eq(content) + expect { instance.download(bucket: bucket, key: key)[:content] }. + to raise_error(Errno::ENOENT, /No such file or directory/) + end + + it "returns the expected payload" do + expect(move).to eq( + bucket: new_bucket, + key: new_key, + ) + end + end + end end diff --git a/spec/file_storage/gcs_spec.rb b/spec/file_storage/gcs_spec.rb new file mode 100644 index 0000000..350617f --- /dev/null +++ b/spec/file_storage/gcs_spec.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require "spec_helper" + +require "file_storage/gcs" + +RSpec.describe FileStorage::Gcs do + subject(:instance) { described_class.new(**gcs_config) } + + let(:bucket_name) { "bucket" } + let(:gcs_config) do + { + endpoint: "http://0.0.0.0:4443/", + project_id: "test", + } + end + + let(:storage) do + @storage = Google::Cloud::Storage.new(**gcs_config) + end + let(:test_bucket) { storage.bucket(bucket_name) } + + before do + storage.create_bucket(bucket_name) + end + + after do + test_bucket.files.each(&:delete) + test_bucket.delete + end + + describe "#upload!" do + subject(:upload!) do + instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something") + end + + it "uploads the file" do + expect { upload! }.to change { test_bucket.files.count }.from(0).to(1) + end + + it "uploads the content" do + upload! + + buffer = StringIO.new + test_bucket.file("some-file.json").download(buffer) + + expect(buffer.string).to eq("something") + end + end + + describe "#download" do + subject(:download) { instance.download(bucket: bucket_name, key: "some-file.json") } + + before do + instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something") + end + + it "returns the correct information" do + expect(download).to eq(bucket: bucket_name, key: "some-file.json", content: "something") + end + end + + describe "#list" do + subject(:list) { instance.list(bucket: bucket_name, key: "", page_size: 1000) } + + before do + instance.upload!(bucket: bucket_name, key: "some-file1.json", content: "something") + instance.upload!(bucket: bucket_name, key: "some-file2.json", content: "something") + end + + it "returns an enumerator" do + expect(list.to_a).to eq([ + { bucket: bucket_name, keys: ["some-file1.json", "some-file2.json"] }, + ]) + end + end + + describe "#delete!" do + subject(:delete!) { instance.delete!(bucket: bucket_name, key: "some-file.json") } + + before do + instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something") + end + + it "deletes the file" do + expect { delete! }.to change { test_bucket.files.count }.from(1).to(0) + end + end + + describe "#move!" do + subject(:move!) do + instance.move!( + bucket: bucket_name, + key: "some-file.json", + new_bucket: bucket_2_name, + new_key: "some-file-moved.json", + ) + end + + let(:bucket_2_name) { "bucket-2" } + let(:bucket_2) { storage.bucket(bucket_2_name) } + + before do + storage.create_bucket(bucket_2_name) + instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something") + end + + after do + bucket_2.files.each(&:delete) + bucket_2.delete + end + + it "moves the file to the new directory" do + move! + + expect(test_bucket.files.count).to eq(0) + expect(bucket_2.file("some-file-moved.json")).to_not be_nil + end + end +end diff --git a/spec/file_storage/in_memory_spec.rb b/spec/file_storage/in_memory_spec.rb index 6588143..77a9c03 100644 --- a/spec/file_storage/in_memory_spec.rb +++ b/spec/file_storage/in_memory_spec.rb @@ -118,6 +118,46 @@ end end + describe "#move!" do + subject(:move) do + instance.move!(bucket: bucket, key: key, new_bucket: new_bucket, new_key: new_key) + end + + let(:new_bucket) { "cake" } + + context "when the 'existing' file doesn't exist" do + let(:key) { "foobar" } + let(:new_key) { "barbaz" } + + it "raises a KeyError" do + expect { move }.to raise_error(KeyError, /#{bucket}/) + end + end + + context "when the file does exist" do + let(:key) { "2021-02-08/hello1" } + let(:new_key) { "2021-02-08/hello2" } + let(:content) { "world" } + + before { instance.upload!(bucket: bucket, key: key, content: content) } + + it "moves the file" do + move + + expect(instance.download(bucket: new_bucket, key: new_key)[:content]).to eq(content) + expect { instance.download(bucket: bucket, key: key)[:content] }. + to raise_error(KeyError, /key not found/) + end + + it "returns the expected payload" do + expect(move).to eq( + bucket: new_bucket, + key: new_key, + ) + end + end + end + describe "#reset!" do let(:bucket2) { "bucket2" } diff --git a/spec/file_storage/key_storage_spec.rb b/spec/file_storage/key_storage_spec.rb index f111b2d..d3670e4 100644 --- a/spec/file_storage/key_storage_spec.rb +++ b/spec/file_storage/key_storage_spec.rb @@ -117,6 +117,33 @@ def build_for(key) end end + describe "#move!" do + subject(:move) { build_for(old_key).move!(new_key) } + + let(:old_key) { "inmemory://bucket/file1" } + let(:new_key) { "inmemory://bucket2/file2" } + + before { build_for(old_key).upload!("hello") } + + it "returns the new path's uri" do + expect(move).to eq(new_key) + end + + it "moves the file" do + move + + expect(build_for(new_key).download[:content]).to eq("hello") + end + + context "with a different adapter" do + let(:new_key) { "disk://bucket2/file2" } + + it "raises an error" do + expect { move }.to raise_error(ArgumentError, /Adapter type/) + end + end + end + describe "delete!" do before do build_for("inmemory://bucket/file1").upload!("content1")