From f65a4b71636725c8f0d27ca82481539068ba8479 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Ritschard Date: Wed, 2 Feb 2022 10:53:20 +0100 Subject: [PATCH] support bytebuffer updates on streaming hashes In some cases ByteBuffers do not provide access to an underlying byte array without incurring copies, this is notably the case for off-heap direct bytebuffers. This patch provides a way to call update on streaming XXHASH instances against a ByteBuffer, which avoids the extra copy needed into a byte array. --- .../xxhash32_streaming.template | 7 +++- .../xxhash64_streaming.template | 7 +++- .../net/jpountz/xxhash/StreamingXXHash32.java | 10 +++++ .../jpountz/xxhash/StreamingXXHash32JNI.java | 18 +++++++++ .../net/jpountz/xxhash/StreamingXXHash64.java | 10 +++++ .../jpountz/xxhash/StreamingXXHash64JNI.java | 18 +++++++++ src/java/net/jpountz/xxhash/XXHashJNI.java | 2 + src/jni/net_jpountz_xxhash_XXHashJNI.c | 40 +++++++++++++++++++ 8 files changed, 110 insertions(+), 2 deletions(-) diff --git a/src/build/source_templates/xxhash32_streaming.template b/src/build/source_templates/xxhash32_streaming.template index 6166758e..311c2fb3 100644 --- a/src/build/source_templates/xxhash32_streaming.template +++ b/src/build/source_templates/xxhash32_streaming.template @@ -6,6 +6,7 @@ import static net.jpountz.xxhash.XXHashConstants.*; import static net.jpountz.util.${type}Utils.*; import static net.jpountz.util.SafeUtils.checkRange; import static java.lang.Integer.rotateLeft; +import java.nio.ByteBuffer; /** * Streaming xxhash. @@ -60,6 +61,11 @@ final class StreamingXXHash32Java${type} extends AbstractStreamingXXHash32Java { return h32; } + @Override + public void update(ByteBuffer buf, int off, int len) { + throw new RuntimeException("unimplemented"); + } + @Override public void update(byte[] buf, int off, int len) { checkRange(buf, off, len); @@ -139,4 +145,3 @@ final class StreamingXXHash32Java${type} extends AbstractStreamingXXHash32Java { } } - diff --git a/src/build/source_templates/xxhash64_streaming.template b/src/build/source_templates/xxhash64_streaming.template index 2789ae0a..00231340 100644 --- a/src/build/source_templates/xxhash64_streaming.template +++ b/src/build/source_templates/xxhash64_streaming.template @@ -6,6 +6,7 @@ import static net.jpountz.xxhash.XXHashConstants.*; import static net.jpountz.util.${type}Utils.*; import static net.jpountz.util.SafeUtils.checkRange; import static java.lang.Long.rotateLeft; +import java.nio.ByteBuffer; /** * Streaming xxhash. @@ -84,6 +85,11 @@ final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java { return h64; } + @Override + public void update(ByteBuffer buf, int off, int len) { + throw new RuntimeException("unimplemented"); + } + @Override public void update(byte[] buf, int off, int len) { checkRange(buf, off, len); @@ -163,4 +169,3 @@ final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java { } } - diff --git a/src/java/net/jpountz/xxhash/StreamingXXHash32.java b/src/java/net/jpountz/xxhash/StreamingXXHash32.java index 81434331..a5bbb595 100644 --- a/src/java/net/jpountz/xxhash/StreamingXXHash32.java +++ b/src/java/net/jpountz/xxhash/StreamingXXHash32.java @@ -2,6 +2,7 @@ import java.util.zip.Checksum; import java.io.Closeable; +import java.nio.ByteBuffer; /* * Copyright 2020 Adrien Grand and the lz4-java contributors. @@ -71,6 +72,15 @@ interface Factory { */ public abstract void update(byte[] buf, int off, int len); + /** + * Updates the value of the hash with buf[off:off+len]. + * + * @param buf the input data + * @param off the start offset in buf + * @param len the number of bytes to hash + */ + public abstract void update(ByteBuffer buf, int off, int len); + /** * Resets this instance to the state it had right after instantiation. The * seed remains unchanged. diff --git a/src/java/net/jpountz/xxhash/StreamingXXHash32JNI.java b/src/java/net/jpountz/xxhash/StreamingXXHash32JNI.java index a03840f5..97ca77a8 100644 --- a/src/java/net/jpountz/xxhash/StreamingXXHash32JNI.java +++ b/src/java/net/jpountz/xxhash/StreamingXXHash32JNI.java @@ -1,5 +1,9 @@ package net.jpountz.xxhash; +import static net.jpountz.util.ByteBufferUtils.checkRange; +import static net.jpountz.util.SafeUtils.checkRange; +import java.nio.ByteBuffer; + /* * Copyright 2020 Adrien Grand and the lz4-java contributors. * @@ -69,6 +73,20 @@ public synchronized void update(byte[] bytes, int off, int len) { XXHashJNI.XXH32_update(state, bytes, off, len); } + public synchronized void update(ByteBuffer buf, int off, int len) { + checkState(); + if (buf.isDirect()) { + checkRange(buf, off, len); + XXHashJNI.XXH32BB_update(state, buf, off, len); + } else if (buf.hasArray()) { + XXHashJNI.XXH32_update(state, buf.array(), off + buf.arrayOffset(), len); + } else { + // XXX: What to do here? + throw new RuntimeException("unsupported"); + } + + } + @Override public synchronized void close() { if (state != 0) { diff --git a/src/java/net/jpountz/xxhash/StreamingXXHash64.java b/src/java/net/jpountz/xxhash/StreamingXXHash64.java index db1e88b4..064ded7a 100644 --- a/src/java/net/jpountz/xxhash/StreamingXXHash64.java +++ b/src/java/net/jpountz/xxhash/StreamingXXHash64.java @@ -1,6 +1,7 @@ package net.jpountz.xxhash; import java.util.zip.Checksum; +import java.nio.ByteBuffer; import java.io.Closeable; /* @@ -71,6 +72,15 @@ interface Factory { */ public abstract void update(byte[] buf, int off, int len); + /** + * Updates the value of the hash with buf[off:off+len]. + * + * @param buf the input data + * @param off the start offset in buf + * @param len the number of bytes to hash + */ + public abstract void update(ByteBuffer buf, int off, int len); + /** * Resets this instance to the state it had right after instantiation. The * seed remains unchanged. diff --git a/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java b/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java index 73713e4b..9f746e7b 100644 --- a/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java +++ b/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java @@ -1,5 +1,9 @@ package net.jpountz.xxhash; +import static net.jpountz.util.ByteBufferUtils.checkRange; +import static net.jpountz.util.SafeUtils.checkRange; +import java.nio.ByteBuffer; + /* * Copyright 2020 Linnaea Von Lavia and the lz4-java contributors. * @@ -70,6 +74,20 @@ public synchronized void update(byte[] bytes, int off, int len) { XXHashJNI.XXH64_update(state, bytes, off, len); } + @Override + public synchronized void update(ByteBuffer buf, int off, int len) { + checkState(); + if (buf.isDirect()) { + checkRange(buf, off, len); + XXHashJNI.XXH64BB_update(state, buf, off, len); + } else if (buf.hasArray()) { + XXHashJNI.XXH64_update(state, buf.array(), off + buf.arrayOffset(), len); + } else { + // XXX: What to do here? + throw new RuntimeException("unsupported"); + } + } + @Override public synchronized void close() { if (state != 0) { diff --git a/src/java/net/jpountz/xxhash/XXHashJNI.java b/src/java/net/jpountz/xxhash/XXHashJNI.java index f39769a0..28c26931 100644 --- a/src/java/net/jpountz/xxhash/XXHashJNI.java +++ b/src/java/net/jpountz/xxhash/XXHashJNI.java @@ -33,6 +33,7 @@ enum XXHashJNI { static native int XXH32BB(ByteBuffer input, int offset, int len, int seed); static native long XXH32_init(int seed); static native void XXH32_update(long state, byte[] input, int offset, int len); + static native void XXH32BB_update(long state, ByteBuffer input, int offset, int len); static native int XXH32_digest(long state); static native void XXH32_free(long state); @@ -40,6 +41,7 @@ enum XXHashJNI { static native long XXH64BB(ByteBuffer input, int offset, int len, long seed); static native long XXH64_init(long seed); static native void XXH64_update(long state, byte[] input, int offset, int len); + static native void XXH64BB_update(long state, ByteBuffer input, int offset, int len); static native long XXH64_digest(long state); static native void XXH64_free(long state); } diff --git a/src/jni/net_jpountz_xxhash_XXHashJNI.c b/src/jni/net_jpountz_xxhash_XXHashJNI.c index d7d8a663..84bb2c57 100644 --- a/src/jni/net_jpountz_xxhash_XXHashJNI.c +++ b/src/jni/net_jpountz_xxhash_XXHashJNI.c @@ -120,6 +120,26 @@ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1update } +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH32BB_update + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32BB_1update +(JNIEnv *env, jclass cls, jlong state, jobject src, jint off, jint len) { + + char* in; + jlong h64; + + in = (char*) (*env)->GetDirectBufferAddress(env, src); + if (in == NULL) { + throw_OOM(env); + return; + } + + XXH32_update((XXH32_state_t*) state, in + off, len); +} + /* * Class: net_jpountz_xxhash_XXHashJNI * Method: XXH32_digest @@ -230,6 +250,26 @@ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1update } +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64BB_update + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64BB_1update +(JNIEnv *env, jclass cls, jlong state, jobject src, jint off, jint len) { + + char* in; + + in = (char*) (*env)->GetDirectBufferAddress(env, src); + if (in == NULL) { + printf("I AM NULLLLL?\n"); + throw_OOM(env); + return; + } + + XXH64_update((XXH64_state_t*) state, in + off, len); +} + /* * Class: net_jpountz_xxhash_XXHashJNI * Method: XXH64_digest