From b4540ed5d4b611488f28e48b5e63839e2b2fe802 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 18 Oct 2023 02:33:42 +0530 Subject: [PATCH] Optimize the reading of numerical frame arrays in MSQ (#15175) --- .../frame/field/DoubleArrayFieldReader.java | 5 +- .../frame/field/FloatArrayFieldReader.java | 5 +- .../frame/field/LongArrayFieldReader.java | 5 +- .../field/NumericArrayFieldSelector.java | 57 +++++++-------- .../frame/field/ReadableFieldPointer.java | 5 ++ .../frame/field/RowMemoryFieldPointer.java | 71 +++++++++++++++++++ .../frame/field/SettableFieldPointer.java | 16 ++++- .../segment/row/ReadableFrameRowPointer.java | 6 ++ .../frame/field/ComplexFieldReaderTest.java | 4 +- .../frame/field/ConstantFieldPointer.java | 10 ++- .../field/DoubleArrayFieldReaderTest.java | 37 ++++++---- .../frame/field/DoubleFieldReaderTest.java | 12 ++-- .../field/FloatArrayFieldReaderTest.java | 26 +++---- .../frame/field/FloatFieldReaderTest.java | 12 ++-- .../frame/field/IndexArrayFieldPointer.java | 10 ++- .../frame/field/LongArrayFieldReaderTest.java | 26 +++---- .../frame/field/LongFieldReaderTest.java | 12 ++-- .../field/StringArrayFieldWriterTest.java | 2 +- .../frame/field/StringFieldReaderTest.java | 26 ++++--- .../frame/field/StringFieldWriterTest.java | 2 +- 20 files changed, 241 insertions(+), 108 deletions(-) rename processing/src/{main => test}/java/org/apache/druid/frame/field/ConstantFieldPointer.java (85%) diff --git a/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java index 48f2a5117ee8..ec7de095e12c 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java +++ b/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java @@ -37,6 +37,7 @@ public ColumnValueSelector makeColumnValueSelector( { return new NumericArrayFieldSelector(memory, fieldPointer) { + private static final int FIELD_SIZE = Byte.BYTES + Double.BYTES; final SettableFieldPointer fieldPointer = new SettableFieldPointer(); final ColumnValueSelector columnValueSelector = DoubleFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer); @@ -45,7 +46,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Double getIndividualValueAtMemory(long position) { - fieldPointer.setPosition(position); + fieldPointer.setPositionAndLength(position, FIELD_SIZE); if (columnValueSelector.isNull()) { return null; } @@ -55,7 +56,7 @@ public Double getIndividualValueAtMemory(long position) @Override public int getIndividualFieldSize() { - return Byte.BYTES + Double.BYTES; + return FIELD_SIZE; } }; } diff --git a/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java index fcbe407bdb26..e97af071824e 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java +++ b/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java @@ -37,6 +37,7 @@ public ColumnValueSelector makeColumnValueSelector( { return new NumericArrayFieldSelector(memory, fieldPointer) { + private static final int FIELD_SIZE = Byte.BYTES + Float.BYTES; final SettableFieldPointer fieldPointer = new SettableFieldPointer(); final ColumnValueSelector columnValueSelector = FloatFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer); @@ -45,7 +46,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Float getIndividualValueAtMemory(long position) { - fieldPointer.setPosition(position); + fieldPointer.setPositionAndLength(position, FIELD_SIZE); if (columnValueSelector.isNull()) { return null; } @@ -55,7 +56,7 @@ public Float getIndividualValueAtMemory(long position) @Override public int getIndividualFieldSize() { - return Byte.BYTES + Float.BYTES; + return FIELD_SIZE; } }; } diff --git a/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java index b52b39d13c47..8f7578c07d38 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java +++ b/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java @@ -37,6 +37,7 @@ public ColumnValueSelector makeColumnValueSelector( { return new NumericArrayFieldSelector(memory, fieldPointer) { + private static final int FIELD_SIZE = Byte.BYTES + Long.BYTES; final SettableFieldPointer fieldPointer = new SettableFieldPointer(); final ColumnValueSelector columnValueSelector = LongFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer); @@ -45,7 +46,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Long getIndividualValueAtMemory(long position) { - fieldPointer.setPosition(position); + fieldPointer.setPositionAndLength(position, FIELD_SIZE); if (columnValueSelector.isNull()) { return null; } @@ -55,7 +56,7 @@ public Long getIndividualValueAtMemory(long position) @Override public int getIndividualFieldSize() { - return Byte.BYTES + Long.BYTES; + return FIELD_SIZE; } }; } diff --git a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java index 1871aef06e08..f15361d47ea1 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java +++ b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java @@ -25,8 +25,6 @@ import org.apache.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; /** * Base implementation of the column value selector that the concrete numeric field reader implementations inherit from. @@ -66,12 +64,8 @@ public abstract class NumericArrayFieldSelector impl /** * Value of the row at the location beginning at {@link #currentFieldPosition} */ - private final List currentRow = new ArrayList<>(); - - /** - * Nullity of the row at the location beginning at {@link #currentFieldPosition} - */ - private boolean currentRowIsNull; + @Nullable + private Number[] currentRow = null; public NumericArrayFieldSelector(final Memory memory, final ReadableFieldPointer fieldPointer) { @@ -89,13 +83,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public Object getObject() { - final List currentArray = computeCurrentArray(); - - if (currentArray == null) { - return null; - } - - return currentArray.toArray(); + return computeCurrentArray(); } @Override @@ -143,34 +131,29 @@ public boolean isNull() public abstract int getIndividualFieldSize(); @Nullable - private List computeCurrentArray() + private Number[] computeCurrentArray() { final long fieldPosition = fieldPointer.position(); + final long fieldLength = fieldPointer.length(); if (fieldPosition != currentFieldPosition) { - updateCurrentArray(fieldPosition); + updateCurrentArray(fieldPosition, fieldLength); } this.currentFieldPosition = fieldPosition; - - if (currentRowIsNull) { - return null; - } return currentRow; - } - private void updateCurrentArray(final long fieldPosition) + private void updateCurrentArray(final long fieldPosition, final long fieldLength) { - currentRow.clear(); - currentRowIsNull = false; + currentRow = null; long position = fieldPosition; long limit = memory.getCapacity(); // Check the first byte, and if it is null, update the current value to null and return if (isNull()) { - currentRowIsNull = true; + // Already set the currentRow to null return; } @@ -179,9 +162,13 @@ private void updateCurrentArray(final long fieldPosition) position++; } + int numElements = numElements(fieldLength); + currentRow = new Number[numElements]; + // Sanity check, to make sure that we see the rowTerminator at the end boolean rowTerminatorSeen = false; + int curElement = 0; while (position < limit) { final byte kind = memory.getByte(position); @@ -193,12 +180,26 @@ private void updateCurrentArray(final long fieldPosition) // If terminator not seen, then read the field at that location, and increment the position by the element's field // size to read the next element. - currentRow.add(getIndividualValueAtMemory(position)); + currentRow[curElement] = getIndividualValueAtMemory(position); position += getIndividualFieldSize(); + curElement++; } - if (!rowTerminatorSeen) { + if (!rowTerminatorSeen || curElement != numElements) { throw DruidException.defensive("Unexpected end of field"); } } + + int numElements(long fieldSize) + { + if (fieldSize <= 1) { + throw DruidException.defensive("fieldSize should be greater than 1 for non null array elements"); + } + // Remove one byte for the nullity byte, and one for the array terminator + long cumulativeFieldSize = fieldSize - Byte.BYTES - Byte.BYTES; + if (cumulativeFieldSize % getIndividualFieldSize() != 0) { + throw DruidException.defensive("cumulativeFieldSize should be a multiple of the individual fieldSize"); + } + return Math.toIntExact(cumulativeFieldSize / getIndividualFieldSize()); + } } diff --git a/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java b/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java index 29412b507af8..09de051a691e 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java @@ -31,4 +31,9 @@ public interface ReadableFieldPointer * Starting position of the field. */ long position(); + + /** + * Length of the field. + */ + long length(); } diff --git a/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java b/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java index 881c9adb04f9..d90eb3eaa22a 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java @@ -24,6 +24,10 @@ /** * A {@link ReadableFieldPointer} that is derived from a row-based frame. + * + * Returns the position and the length of a field at a particular position for the row that the rowPointer is pointing + * to at the time. It caches the values of the position and the length based on position of the rowPointer. + * This method is not thread-safe */ public class RowMemoryFieldPointer implements ReadableFieldPointer { @@ -32,6 +36,16 @@ public class RowMemoryFieldPointer implements ReadableFieldPointer private final int fieldNumber; private final int fieldCount; + // Caching of position() calls + private long rowWithCachedPosition = -1L; + private long cachedPosition = -1L; + + // Caching of length() calls + // We cache the length() calls separately, because not all field types call length(), therefore it's wasteful to + // compute and cache length() if we are not reading it + private long rowWithCachedLength = -1L; + private long cachedLength = -1L; + public RowMemoryFieldPointer( final Memory memory, final ReadableFrameRowPointer rowPointer, @@ -47,6 +61,63 @@ public RowMemoryFieldPointer( @Override public long position() + { + updatePosition(); + return cachedPosition; + } + + @Override + public long length() + { + updatePositionAndLength(); + return cachedLength; + } + + private void updatePosition() + { + long rowPointerPosition = rowPointer.position(); + if (rowPointerPosition == rowWithCachedPosition) { + return; + } + // Update the cached position for position() + rowWithCachedPosition = rowPointerPosition; + + // Update the start position + cachedPosition = startPosition(fieldNumber); + } + + // Not all field types call length(), and therefore there's no need to cache the length of the field. This method + // updates both the position and the length. + private void updatePositionAndLength() + { + updatePosition(); + + // rowPointerPosition = rowPointer.position() = rowWithCachedPosition, since that was updated in the call to update + // position above + long rowPointerPosition = rowWithCachedPosition; + + if (rowPointerPosition == rowWithCachedLength) { + return; + } + // Update the cached position for length() + rowWithCachedLength = rowPointerPosition; + + if (fieldNumber == fieldCount - 1) { + // If the field is the last field in the row, then the length of the field would be the end of the row minus the + // start position of the field. End of the row is the start of the row plus the length of the row. + cachedLength = (rowPointerPosition + rowPointer.length()) - cachedPosition; + } else { + // Else the length of the field would be the difference between the start position of the given field and + // the subsequent field + cachedLength = startPosition(fieldNumber + 1) - cachedPosition; + } + } + + /** + * Given a fieldNumber, computes the start position of the field. Requires a memory access to read the start position, + * therefore callers should cache the value for better efficiency. + */ + private long startPosition(int fieldNumber) { if (fieldNumber == 0) { // First field starts after the field end pointers -- one integer per field. diff --git a/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java b/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java index d26f84f251de..0242b412ca08 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java @@ -20,16 +20,20 @@ package org.apache.druid.frame.field; /** - * A simple {@link ReadableFieldPointer} that returns the position that was set on its object. + * A simple {@link ReadableFieldPointer} that returns the position and the length that was set on its object. */ public class SettableFieldPointer implements ReadableFieldPointer { - long position = 0; + long length = -1; - public void setPosition(long position) + /** + * Sets the position and the length to be returned when interface's methods are called. + */ + public void setPositionAndLength(long position, long length) { this.position = position; + this.length = length; } @Override @@ -37,4 +41,10 @@ public long position() { return position; } + + @Override + public long length() + { + return length; + } } diff --git a/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java b/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java index 714195396c8e..8276e26b28c3 100644 --- a/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java @@ -30,7 +30,13 @@ */ public interface ReadableFrameRowPointer { + /** + * Position of the start of the row relative to the start of the Frame + */ long position(); + /** + * Length of the row (in bytes) + */ long length(); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java index 598e7d28edc1..d4970d9bd0cd 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java @@ -123,7 +123,7 @@ public void test_makeColumnValueSelector_null() writeToMemory(null); final ColumnValueSelector readSelector = - new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertNull(readSelector.getObject()); } @@ -134,7 +134,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory("foo"); final ColumnValueSelector readSelector = - new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals("foo", readSelector.getObject()); } diff --git a/processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java b/processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java similarity index 85% rename from processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java rename to processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java index 17ff1058794b..0079246c2623 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java +++ b/processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java @@ -22,10 +22,12 @@ public class ConstantFieldPointer implements ReadableFieldPointer { private final long position; + private final long length; - public ConstantFieldPointer(long position) + public ConstantFieldPointer(long position, long length) { this.position = position; + this.length = length; } @Override @@ -33,4 +35,10 @@ public long position() { return position; } + + @Override + public long length() + { + return length; + } } diff --git a/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java index 6381138f62db..c931b7aac904 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java @@ -149,10 +149,13 @@ public void test_isNull_arrayWithSingleNullElement() @Test public void test_makeColumnValueSelector_null() { - writeToMemory(null, MEMORY_POSITION); + long sz = writeToMemory(null, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader().makeColumnValueSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, sz) + ); Assert.assertTrue(readSelector.isNull()); } @@ -160,10 +163,14 @@ public void test_makeColumnValueSelector_null() @Test public void test_makeColumnValueSelector_aValue() { - writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION); + long sz = writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader() + .makeColumnValueSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, sz) + ); assertResults(DOUBLES_LIST_1, readSelector.getObject()); } @@ -172,15 +179,15 @@ public void test_makeColumnValueSelector_aValue() public void test_makeColumnValueSelector_multipleValues() { long sz = writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION); - writeToMemory(DOUBLES_ARRAY_2, MEMORY_POSITION + sz); - IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz)); - + long sz2 = writeToMemory(DOUBLES_ARRAY_2, MEMORY_POSITION + sz); + IndexArrayFieldPointer pointer = new IndexArrayFieldPointer( + ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz), + ImmutableList.of(sz, sz2) + ); final ColumnValueSelector readSelector = new DoubleArrayFieldReader().makeColumnValueSelector(memory, pointer); - pointer.setPointer(0); assertResults(DOUBLES_LIST_1, readSelector.getObject()); - pointer.setPointer(1); assertResults(DOUBLES_LIST_2, readSelector.getObject()); } @@ -188,10 +195,11 @@ public void test_makeColumnValueSelector_multipleValues() @Test public void test_makeColumnValueSelector_emptyArray() { - writeToMemory(new Object[]{}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader() + .makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.emptyList(), readSelector.getObject()); } @@ -199,10 +207,13 @@ public void test_makeColumnValueSelector_emptyArray() @Test public void test_makeColumnValueSelector_arrayWithSingleNullElement() { - writeToMemory(new Object[]{null}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader().makeColumnValueSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, sz) + ); assertResults(Collections.singletonList(null), readSelector.getObject()); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java index a18b7cc70ec9..c4a300b5bc3c 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java @@ -89,7 +89,7 @@ public void test_makeColumnValueSelector_defaultOrNull() writeToMemory(NullHandling.defaultDoubleValue()); final ColumnValueSelector readSelector = - DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull()); @@ -104,7 +104,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory(5.1d); final ColumnValueSelector readSelector = - DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(5.1d, readSelector.getObject()); } @@ -115,7 +115,8 @@ public void test_makeDimensionSelector_defaultOrNull() writeToMemory(NullHandling.defaultDoubleValue()); final DimensionSelector readSelector = - DoubleFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + DoubleFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -149,7 +150,8 @@ public void test_makeDimensionSelector_aValue() writeToMemory(5.1d); final DimensionSelector readSelector = - DoubleFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + DoubleFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -178,7 +180,7 @@ public void test_makeDimensionSelector_aValue_extractionFn() final DimensionSelector readSelector = DoubleFieldReader.forPrimitive().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java index e61e40db1cb1..3c404ed28dd9 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java @@ -150,10 +150,10 @@ public void test_isNull_arrayWithSingleNullElement() @Test public void test_makeColumnValueSelector_null() { - writeToMemory(null, MEMORY_POSITION); + long sz = writeToMemory(null, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); Assert.assertTrue(readSelector.isNull()); } @@ -161,10 +161,10 @@ public void test_makeColumnValueSelector_null() @Test public void test_makeColumnValueSelector_aValue() { - writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION); + long sz = writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(FLOATS_LIST_1, readSelector.getObject()); } @@ -173,15 +173,15 @@ public void test_makeColumnValueSelector_aValue() public void test_makeColumnValueSelector_multipleValues() { long sz = writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION); - writeToMemory(FLOATS_ARRAY_2, MEMORY_POSITION + sz); - IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz)); - + long sz2 = writeToMemory(FLOATS_ARRAY_2, MEMORY_POSITION + sz); + IndexArrayFieldPointer pointer = new IndexArrayFieldPointer( + ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz), + ImmutableList.of(sz, sz2) + ); final ColumnValueSelector readSelector = new FloatArrayFieldReader().makeColumnValueSelector(memory, pointer); - pointer.setPointer(0); assertResults(FLOATS_LIST_1, readSelector.getObject()); - pointer.setPointer(1); assertResults(FLOATS_LIST_2, readSelector.getObject()); } @@ -189,10 +189,10 @@ public void test_makeColumnValueSelector_multipleValues() @Test public void test_makeColumnValueSelector_emptyArray() { - writeToMemory(new Object[]{}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.emptyList(), readSelector.getObject()); } @@ -200,10 +200,10 @@ public void test_makeColumnValueSelector_emptyArray() @Test public void test_makeColumnValueSelector_arrayWithSingleNullElement() { - writeToMemory(new Object[]{null}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.singletonList(null), readSelector.getObject()); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java index 6bae52f1c50d..f25845f9f4ed 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java @@ -89,7 +89,7 @@ public void test_makeColumnValueSelector_defaultOrNull() writeToMemory(NullHandling.defaultFloatValue()); final ColumnValueSelector readSelector = - FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull()); @@ -104,7 +104,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory(5.1f); final ColumnValueSelector readSelector = - FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(5.1f, readSelector.getObject()); } @@ -115,7 +115,8 @@ public void test_makeDimensionSelector_defaultOrNull() writeToMemory(NullHandling.defaultFloatValue()); final DimensionSelector readSelector = - FloatFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + FloatFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -149,7 +150,8 @@ public void test_makeDimensionSelector_aValue() writeToMemory(5.1f); final DimensionSelector readSelector = - FloatFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + FloatFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -178,7 +180,7 @@ public void test_makeDimensionSelector_aValue_extractionFn() final DimensionSelector readSelector = FloatFieldReader.forPrimitive().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java b/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java index 1e115f48e3c5..22fdbacd1b71 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java +++ b/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java @@ -30,11 +30,13 @@ public class IndexArrayFieldPointer implements ReadableFieldPointer { private final LongArrayList indices; + private final LongArrayList lengths; private int pointer = 0; - public IndexArrayFieldPointer(final List indices) + public IndexArrayFieldPointer(final List indices, final List lengths) { this.indices = new LongArrayList(indices); + this.lengths = new LongArrayList(lengths); } private int numIndices() @@ -53,4 +55,10 @@ public long position() { return indices.getLong(pointer); } + + @Override + public long length() + { + return lengths.getLong(pointer); + } } diff --git a/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java index aa34cd6afaf3..f679bb2b3ed9 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java @@ -126,10 +126,10 @@ public void test_isNull_arrayWithSingleNullElement() @Test public void test_makeColumnValueSelector_null() { - writeToMemory(null, MEMORY_POSITION); + long sz = writeToMemory(null, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); Assert.assertTrue(readSelector.isNull()); } @@ -137,10 +137,10 @@ public void test_makeColumnValueSelector_null() @Test public void test_makeColumnValueSelector_aValue() { - writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION); + long sz = writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(LONGS_LIST_1, readSelector.getObject()); } @@ -149,15 +149,15 @@ public void test_makeColumnValueSelector_aValue() public void test_makeColumnValueSelector_multipleValues() { long sz = writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION); - writeToMemory(LONGS_ARRAY_2, MEMORY_POSITION + sz); - IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz)); - + long sz2 = writeToMemory(LONGS_ARRAY_2, MEMORY_POSITION + sz); + IndexArrayFieldPointer pointer = new IndexArrayFieldPointer( + ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz), + ImmutableList.of(sz, sz2) + ); final ColumnValueSelector readSelector = new LongArrayFieldReader().makeColumnValueSelector(memory, pointer); - pointer.setPointer(0); assertResults(LONGS_LIST_1, readSelector.getObject()); - pointer.setPointer(1); assertResults(LONGS_LIST_2, readSelector.getObject()); } @@ -165,10 +165,10 @@ public void test_makeColumnValueSelector_multipleValues() @Test public void test_makeColumnValueSelector_emptyArray() { - writeToMemory(new Object[]{}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.emptyList(), readSelector.getObject()); } @@ -176,10 +176,10 @@ public void test_makeColumnValueSelector_emptyArray() @Test public void test_makeColumnValueSelector_arrayWithSingleNullElement() { - writeToMemory(new Object[]{null}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.singletonList(null), readSelector.getObject()); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java index 643846ee0e3a..8a201394083d 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java @@ -89,7 +89,7 @@ public void test_makeColumnValueSelector_defaultOrNull() writeToMemory(NullHandling.defaultLongValue()); final ColumnValueSelector readSelector = - LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull()); @@ -104,7 +104,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory(5L); final ColumnValueSelector readSelector = - LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(5L, readSelector.getObject()); } @@ -115,7 +115,8 @@ public void test_makeDimensionSelector_defaultOrNull() writeToMemory(NullHandling.defaultLongValue()); final DimensionSelector readSelector = - LongFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + LongFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -149,7 +150,8 @@ public void test_makeDimensionSelector_aValue() writeToMemory(5L); final DimensionSelector readSelector = - LongFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + LongFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -178,7 +180,7 @@ public void test_makeDimensionSelector_aValue_extractionFn() final DimensionSelector readSelector = LongFieldReader.forPrimitive().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java index 63dd03d48bf0..02d4d44cbfcc 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java @@ -140,7 +140,7 @@ private List readFromMemory(final long written) final FieldReader fieldReader = FieldReaders.create("columnNameDoesntMatterHere", ColumnType.STRING_ARRAY); final ColumnValueSelector selector = - fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final Object o = selector.getObject(); //noinspection rawtypes,unchecked diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java index a682e658ca76..04296cb78c3c 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java @@ -143,9 +143,9 @@ public void test_makeColumnValueSelector_singleString_notArray() writeToMemory(Collections.singletonList("foo")); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals("foo", readSelector.getObject()); Assert.assertArrayEquals(new Object[]{"foo"}, (Object[]) readSelectorAsArray.getObject()); @@ -157,9 +157,9 @@ public void test_makeColumnValueSelector_multiString() writeToMemory(ImmutableList.of("foo", "bar")); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(ImmutableList.of("foo", "bar"), readSelector.getObject()); Assert.assertArrayEquals(new Object[]{"foo", "bar"}, (Object[]) readSelectorAsArray.getObject()); @@ -171,9 +171,9 @@ public void test_makeColumnValueSelector_null() writeToMemory(Collections.singletonList(null)); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertNull(readSelector.getObject()); Assert.assertArrayEquals(new Object[]{null}, (Object[]) readSelectorAsArray.getObject()); @@ -185,9 +185,9 @@ public void test_makeColumnValueSelector_empty() writeToMemory(Collections.emptyList()); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertNull(readSelector.getObject()); Assert.assertArrayEquals(ObjectArrays.EMPTY_ARRAY, (Object[]) readSelectorAsArray.getObject()); @@ -200,7 +200,11 @@ public void test_makeDimensionSelector_multiString_asArray() final IllegalStateException e = Assert.assertThrows( IllegalStateException.class, - () -> new StringArrayFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null) + () -> new StringArrayFieldReader().makeDimensionSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, -1), + null + ) ); MatcherAssert.assertThat( @@ -215,7 +219,7 @@ public void test_makeDimensionSelector_multiString() writeToMemory(ImmutableList.of("foo", "bar")); final DimensionSelector readSelector = - new StringFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + new StringFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -247,7 +251,7 @@ public void test_makeDimensionSelector_multiString_withExtractionFn() final DimensionSelector readSelector = new StringFieldReader().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java index 12bbf8238bf2..f44b69e2810f 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java @@ -184,7 +184,7 @@ private Object[] readFromMemory(final long written) final FieldReader fieldReader = FieldReaders.create("columnNameDoesntMatterHere", ColumnType.STRING_ARRAY); final ColumnValueSelector selector = - fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); return (Object[]) selector.getObject(); }