Skip to content

Commit

Permalink
Support moving files between uris
Browse files Browse the repository at this point in the history
Bump version v0.3.0
  • Loading branch information
JoeSouthan committed May 26, 2021
1 parent 3ad68aa commit 831dd05
Show file tree
Hide file tree
Showing 13 changed files with 315 additions and 5 deletions.
2 changes: 2 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ jobs:
type: string
docker:
- image: circleci/ruby:<< parameters.ruby_version >>
- image: fsouza/fake-gcs-server
command: "-scheme http"
steps:
- add_ssh_keys
- checkout
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ tmp/

.bundle/
.rspec_status
.yardoc/
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
v0.3.0
------

- Add `move!` to move files

v0.2.0
------

Expand Down
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,15 @@ FileStorage.for("inmemory://bucket/path/").list
=> ["inmemory://bucket/path/file.xml"]
```

### Moving a file

_Note: Moving a file is only supported between the same adapter type_

```ruby
FileStorage.for("inmemory://bucket/path/file.xml").move!("inmemory://bucket/path/file2.xml")
=> "inmemory://bucket/path/file2.xml"
```

### Delete a file
```ruby
FileStorage.for("inmemory://bucket/path/file.xml").delete!
Expand Down
9 changes: 9 additions & 0 deletions lib/file_storage/disk.rb
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ def delete!(bucket:, key:)
true
end

def move!(bucket:, key:, new_bucket:, new_key:)
FileUtils.mv(key_path(bucket, key), key_path(new_bucket, new_key))

{
bucket: new_bucket,
key: new_key,
}
end

private

attr_reader :base_dir
Expand Down
20 changes: 16 additions & 4 deletions lib/file_storage/gcs.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@ module FileStorage
class Gcs
DEFAULT_TIMEOUT_SECONDS = 30

def self.build(timeout_seconds = DEFAULT_TIMEOUT_SECONDS)
Gcs.new(timeout_seconds)
def self.build(timeout_seconds = DEFAULT_TIMEOUT_SECONDS, gcs_config = {})
Gcs.new(timeout: timeout_seconds, **gcs_config)
end

def initialize(timeout_seconds)
def initialize(**gcs_config)
@storage = Google::Cloud::Storage.new(
timeout: timeout_seconds,
**gcs_config,
)
end

Expand Down Expand Up @@ -66,6 +66,18 @@ def delete!(bucket:, key:)
true
end

def move!(bucket:, key:, new_bucket:, new_key:)
old_file = get_bucket(bucket).file(key)
destination_bucket = get_bucket(new_bucket)
old_file.copy(destination_bucket.name, new_key)
old_file.delete

{
bucket: new_bucket,
key: new_key,
}
end

private

attr_reader :storage
Expand Down
8 changes: 8 additions & 0 deletions lib/file_storage/in_memory.rb
Original file line number Diff line number Diff line change
Expand Up @@ -58,5 +58,13 @@ def delete!(bucket:, key:)

true
end

def move!(bucket:, key:, new_bucket:, new_key:)
@buckets[new_bucket][new_key] = @buckets.fetch(bucket).delete(key)
{
bucket: new_bucket,
key: new_key,
}
end
end
end
37 changes: 37 additions & 0 deletions lib/file_storage/key_storage.rb
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,43 @@ def list(page_size: 1000)
end
end

# Moves the existing file to a new file path
#
# @param [String] new_key The new key to move the file to
# @return [String] A URI to the file's new path
# @example Move a file
# FileStorage.for("inmemory://bucket1/foo").move!("inmemory://bucket2/bar")
def move!(new_key)
raise ArgumentError, "Key cannot be empty" if key.empty?

new_key_ctx = FileStorage.for(new_key)

unless new_key_ctx.adapter_type == adapter_type
raise ArgumentError, "Adapter type must be the same"
end
raise ArgumentError, "Destination key cannot be empty" if new_key_ctx.key.empty?

start = FileStorage::Timing.monotonic_now
result = adapter.move!(
bucket: bucket,
key: key,
new_bucket: new_key_ctx.bucket,
new_key: new_key_ctx.key,
)

old_key = "#{adapter_type}://#{bucket}/#{key}"
new_key = "#{adapter_type}://#{result[:bucket]}/#{result[:key]}"

FileStorage.logger.info(
event: "key_storage.moved",
duration: FileStorage::Timing.monotonic_now - start,
old_key: old_key,
new_key: new_key,
)

new_key
end

# Deletes the referenced key.
#
# Note that this method will always return true.
Expand Down
2 changes: 1 addition & 1 deletion lib/file_storage/version.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# frozen_string_literal: true

module FileStorage
VERSION = "0.2.0"
VERSION = "0.3.0"
end
40 changes: 40 additions & 0 deletions spec/file_storage/disk_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,44 @@
to raise_error(Errno::ENOENT, /No such file or directory/)
end
end

describe "#move!" do
subject(:move) do
instance.move!(bucket: bucket, key: key, new_bucket: new_bucket, new_key: new_key)
end

let(:new_bucket) { "cake" }

context "when the 'existing' file doesn't exist" do
let(:key) { "foobar" }
let(:new_key) { "barbaz" }

it "raises a File error" do
expect { move }.to raise_error(Errno::ENOENT, /No such file or directory/)
end
end

context "when the file does exist" do
let(:key) { "2021-02-08/hello1" }
let(:new_key) { "2021-02-08/hello2" }
let(:content) { "world" }

before { instance.upload!(bucket: bucket, key: key, content: content) }

it "moves the file" do
move

expect(instance.download(bucket: new_bucket, key: new_key)[:content]).to eq(content)
expect { instance.download(bucket: bucket, key: key)[:content] }.
to raise_error(Errno::ENOENT, /No such file or directory/)
end

it "returns the expected payload" do
expect(move).to eq(
bucket: new_bucket,
key: new_key,
)
end
end
end
end
120 changes: 120 additions & 0 deletions spec/file_storage/gcs_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# frozen_string_literal: true

require "spec_helper"

require "file_storage/gcs"

RSpec.describe FileStorage::Gcs do
subject(:instance) { described_class.new(**gcs_config) }

let(:bucket_name) { "bucket" }
let(:gcs_config) do
{
endpoint: "http://0.0.0.0:4443/",
project_id: "test",
}
end

let(:storage) do
@storage = Google::Cloud::Storage.new(**gcs_config)
end
let(:test_bucket) { storage.bucket(bucket_name) }

before do
storage.create_bucket(bucket_name)
end

after do
test_bucket.files.each(&:delete)
test_bucket.delete
end

describe "#upload!" do
subject(:upload!) do
instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something")
end

it "uploads the file" do
expect { upload! }.to change { test_bucket.files.count }.from(0).to(1)
end

it "uploads the content" do
upload!

buffer = StringIO.new
test_bucket.file("some-file.json").download(buffer)

expect(buffer.string).to eq("something")
end
end

describe "#download" do
subject(:download) { instance.download(bucket: bucket_name, key: "some-file.json") }

before do
instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something")
end

it "returns the correct information" do
expect(download).to eq(bucket: bucket_name, key: "some-file.json", content: "something")
end
end

describe "#list" do
subject(:list) { instance.list(bucket: bucket_name, key: "", page_size: 1000) }

before do
instance.upload!(bucket: bucket_name, key: "some-file1.json", content: "something")
instance.upload!(bucket: bucket_name, key: "some-file2.json", content: "something")
end

it "returns an enumerator" do
expect(list.to_a).to eq([
{ bucket: bucket_name, keys: ["some-file1.json", "some-file2.json"] },
])
end
end

describe "#delete!" do
subject(:delete!) { instance.delete!(bucket: bucket_name, key: "some-file.json") }

before do
instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something")
end

it "deletes the file" do
expect { delete! }.to change { test_bucket.files.count }.from(1).to(0)
end
end

describe "#move!" do
subject(:move!) do
instance.move!(
bucket: bucket_name,
key: "some-file.json",
new_bucket: bucket_2_name,
new_key: "some-file-moved.json",
)
end

let(:bucket_2_name) { "bucket-2" }
let(:bucket_2) { storage.bucket(bucket_2_name) }

before do
storage.create_bucket(bucket_2_name)
instance.upload!(bucket: bucket_name, key: "some-file.json", content: "something")
end

after do
bucket_2.files.each(&:delete)
bucket_2.delete
end

it "moves the file to the new directory" do
move!

expect(test_bucket.files.count).to eq(0)
expect(bucket_2.file("some-file-moved.json")).to_not be_nil
end
end
end
40 changes: 40 additions & 0 deletions spec/file_storage/in_memory_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,46 @@
end
end

describe "#move!" do
subject(:move) do
instance.move!(bucket: bucket, key: key, new_bucket: new_bucket, new_key: new_key)
end

let(:new_bucket) { "cake" }

context "when the 'existing' file doesn't exist" do
let(:key) { "foobar" }
let(:new_key) { "barbaz" }

it "raises a KeyError" do
expect { move }.to raise_error(KeyError, /#{bucket}/)
end
end

context "when the file does exist" do
let(:key) { "2021-02-08/hello1" }
let(:new_key) { "2021-02-08/hello2" }
let(:content) { "world" }

before { instance.upload!(bucket: bucket, key: key, content: content) }

it "moves the file" do
move

expect(instance.download(bucket: new_bucket, key: new_key)[:content]).to eq(content)
expect { instance.download(bucket: bucket, key: key)[:content] }.
to raise_error(KeyError, /key not found/)
end

it "returns the expected payload" do
expect(move).to eq(
bucket: new_bucket,
key: new_key,
)
end
end
end

describe "#reset!" do
let(:bucket2) { "bucket2" }

Expand Down
Loading

0 comments on commit 831dd05

Please sign in to comment.