Skip to content

Commit

Permalink
Merge pull request #25 from saalfeldlab/perf/skipIsFile
Browse files Browse the repository at this point in the history 
perf: throw N5NoSuchKeyException if no key when reading
  • Loading branch information
@bogovicj
bogovicj authored Jul 16, 2024
2 parents 0b2c58d + 5a96559 commit 0a0aaa4
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 28 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.scijava</groupId>
<artifactId>pom-scijava</artifactId>
<version>37.0.0</version>
<version>38.0.1</version>
<relativePath />
</parent>

Expand Down Expand Up @@ -123,7 +123,7 @@
<!-- NB: Deploy releases to the SciJava Maven repository. -->
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>

<n5.version>3.2.0</n5.version>
<n5.version>3.3.0</n5.version>
</properties>

<dependencies>
Expand Down
139 changes: 113 additions & 26 deletions src/main/java/org/janelia/saalfeldlab/n5/googlecloud/GoogleCloudStorageKeyValueAccess.java
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,9 @@
package org.janelia.saalfeldlab.n5.googlecloud;

import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobField;
import com.google.cloud.storage.Storage.BlobListOption;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
import org.janelia.saalfeldlab.n5.LockedChannel;
import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.N5URI;

import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
Expand All @@ -35,6 +20,25 @@
import java.util.List;
import java.util.stream.Collectors;

import com.google.cloud.ReadChannel;
import com.google.cloud.storage.StorageException;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
import org.janelia.saalfeldlab.n5.LockedChannel;
import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.N5URI;

import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobField;
import com.google.cloud.storage.Storage.BlobListOption;

public class GoogleCloudStorageKeyValueAccess implements KeyValueAccess {

private final Storage storage;
Expand All @@ -45,7 +49,7 @@ protected static GoogleCloudStorageURI uncheckedContainerLocationStringToGoogleU

try {
return new GoogleCloudStorageURI(uri);
} catch (Exception e) {
} catch (final Exception e) {
throw new N5Exception("Container location " + uri + " is an invalid URI", e);
}
}
Expand Down Expand Up @@ -167,7 +171,7 @@ public String relativize(final String path, final String base) {
* case, since we only care about the relative portion of `path` to `base`, so the result always
* ignores the absolute prefix anyway. */
return GoogleCloudUtils.getGoogleCloudStorageKey(normalize(uri("/" + base).relativize(uri("/" + path)).getPath()));
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception("Cannot relativize path (" + path + ") with base (" + base + ")", e);
}
}
Expand Down Expand Up @@ -247,6 +251,7 @@ private boolean keyExists(final String key) {

private static boolean blobExists(final Blob blob) {

// TODO document this
return blob != null && blob.exists();
}

Expand Down Expand Up @@ -438,10 +443,8 @@ private void checkWritable() {
@Override
public InputStream newInputStream() {

final Blob blob = storage.get(BlobId.of(bucketName, path));
if (!blobExists(blob))
return null;
final InputStream in = Channels.newInputStream(blob.reader());
final ReadChannel channel = storage.reader(bucketName, path);
final InputStream in = new NoSuchKeyWrappedInputStream(Channels.newInputStream(channel));
synchronized (resources) {
resources.add(in);
}
Expand All @@ -451,10 +454,7 @@ public InputStream newInputStream() {
@Override
public Reader newReader() {

final Blob blob = storage.get(BlobId.of(bucketName, path));
if (!blobExists(blob))
return null;
final Reader in = Channels.newReader(blob.reader(), StandardCharsets.UTF_8.name());
final Reader in = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8);
synchronized (resources) {
resources.add(in);
}
Expand Down Expand Up @@ -494,5 +494,92 @@ public void close() throws IOException {
resources.clear();
}
}

private class NoSuchKeyWrappedInputStream extends InputStream {

private final InputStream in;

public NoSuchKeyWrappedInputStream(InputStream in) {

this.in = in;
}

private IOException rethrowOrNoSuchKeyException(IOException e) {

if (e.getCause() instanceof StorageException && ((StorageException) e.getCause()).getCode() == 404)
throw new N5Exception.N5NoSuchKeyException(e);
return e;
}

@Override public int read() throws IOException {

try {
return in.read();
} catch (final IOException e) {
throw rethrowOrNoSuchKeyException(e);
}
}

@Override public int read(byte[] b) throws IOException {

try {
return in.read(b);
} catch (final IOException e) {
throw rethrowOrNoSuchKeyException(e);
}
}

@Override public int read(byte[] b, int off, int len) throws IOException {

try {
return in.read(b, off, len);
} catch (final IOException e) {
throw rethrowOrNoSuchKeyException(e);
}
}

@Override public long skip(long n) throws IOException {

try {
return in.skip(n);
} catch (final IOException e) {
throw rethrowOrNoSuchKeyException(e);
}
}

@Override public int available() throws IOException {

try {
return in.available();
} catch (final IOException e) {
throw rethrowOrNoSuchKeyException(e);
}

}

@Override public void close() throws IOException {

in.close();
}

@Override public void mark(int readlimit) {

in.mark(readlimit);
}

@Override public void reset() throws IOException {

try {
in.reset();
} catch (final IOException e) {
throw rethrowOrNoSuchKeyException(e);
}
}

@Override public boolean markSupported() {

return in.markSupported();
}
}
}
}

0 comments on commit 0a0aaa4

Please sign in to comment.