From 32bca40e3de2de0f820a6a492c4a4990c1a93937 Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Wed, 21 Jun 2023 14:32:44 -0700 Subject: [PATCH 01/14] Add support for first/last for double/float/long #10702 --- .../wikipedia_merge_index_queries.json | 38 ++++++- .../indexer/wikipedia_merge_index_task.json | 30 +++++ ...merge_reindex_druid_input_source_task.json | 30 +++++ .../indexer/wikipedia_merge_reindex_task.json | 30 +++++ .../druid/jackson/AggregatorsModule.java | 7 ++ ...stractSerializableLongObjectPairSerde.java | 82 ++++++++++++++ .../SerializablePairLongDouble.java | 35 ++++++ ...zablePairLongDoubleComplexMetricSerde.java | 99 ++++++++++++++++ .../SerializablePairLongFloat.java | 35 ++++++ ...izablePairLongFloatComplexMetricSerde.java | 99 ++++++++++++++++ .../aggregation/SerializablePairLongLong.java | 35 ++++++ ...lizablePairLongLongComplexMetricSerde.java | 99 ++++++++++++++++ .../first/DoubleFirstAggregator.java | 20 ++-- .../first/DoubleFirstAggregatorFactory.java | 107 +++++------------- .../first/DoubleFirstBufferAggregator.java | 23 ++-- .../first/FloatFirstAggregator.java | 21 ++-- .../first/FloatFirstAggregatorFactory.java | 105 +++++------------ .../first/FloatFirstBufferAggregator.java | 23 ++-- .../first/GenericFirstAggregateCombiner.java | 66 +++++++++++ .../first/LongFirstAggregator.java | 19 +++- .../first/LongFirstAggregatorFactory.java | 104 +++++------------ .../first/LongFirstBufferAggregator.java | 19 +++- .../first/NumericFirstAggregator.java | 40 ++++++- .../first/NumericFirstBufferAggregator.java | 48 ++++++-- .../first/StringFirstAggregatorFactory.java | 4 +- .../first/StringFirstLastUtils.java | 7 +- .../last/DoubleLastAggregator.java | 20 ++-- .../last/DoubleLastAggregatorFactory.java | 106 +++++------------ .../last/DoubleLastBufferAggregator.java | 21 ++-- .../last/DoubleLastVectorAggregator.java | 4 +- .../aggregation/last/FloatLastAggregator.java | 21 ++-- .../last/FloatLastAggregatorFactory.java | 104 +++++------------ .../last/FloatLastBufferAggregator.java | 22 ++-- .../last/FloatLastVectorAggregator.java | 4 +- .../last/GenericLastAggregateCombiner.java | 66 +++++++++++ .../aggregation/last/LongLastAggregator.java | 19 +++- .../last/LongLastAggregatorFactory.java | 105 +++++------------ .../last/LongLastBufferAggregator.java | 19 +++- .../last/LongLastVectorAggregator.java | 4 +- .../last/NumericLastAggregator.java | 47 ++++++-- .../last/NumericLastBufferAggregator.java | 49 ++++++-- .../last/StringLastAggregatorFactory.java | 4 +- .../aggregation/AggregatorFactoryTest.java | 12 +- .../first/DoubleFirstAggregationTest.java | 53 +++++++-- .../first/FloatFirstAggregationTest.java | 49 ++++++-- .../first/LongFirstAggregationTest.java | 47 ++++++-- .../last/DoubleLastAggregationTest.java | 52 +++++++-- .../last/FloatLastAggregationTest.java | 51 +++++++-- .../last/LongLastAggregationTest.java | 52 +++++++-- .../GroupByQueryQueryToolChestTest.java | 7 +- .../topn/TopNQueryQueryToolChestTest.java | 7 +- .../druid/segment/IndexMergerRollupTest.java | 32 ++++-- 52 files changed, 1542 insertions(+), 660 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json index ab4674999b5d..25ea61496165 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json @@ -26,6 +26,36 @@ "type":"stringLast", "name":"latest_user", "fieldName":"last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longFirst", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ] }, @@ -35,7 +65,13 @@ "event" : { "continent":"Asia", "earliest_user":"masterYi", - "latest_user":"stringer" + "latest_user":"stringer", + "double_first_delta": 111.0, + "double_last_delta": -9.0, + "long_first_delta": 111, + "long_last_delta": -9, + "float_first_delta": 111.0, + "float_last_delta": -9.0 } } ] } diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json index 268a3aef4a85..0af62eb49884 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json @@ -40,6 +40,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "delta" } ], "granularitySpec": { diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json index 9daae62c8d42..348aff886455 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json @@ -56,6 +56,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ] } diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json index 040fff005ce7..9c69c825b9e7 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json @@ -37,6 +37,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ], "granularitySpec": { diff --git a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java index 3130fefb85d3..93d6afb1dd98 100644 --- a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java @@ -39,6 +39,9 @@ import org.apache.druid.query.aggregation.LongMinAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDoubleComplexMetricSerde; +import org.apache.druid.query.aggregation.SerializablePairLongFloatComplexMetricSerde; +import org.apache.druid.query.aggregation.SerializablePairLongLongComplexMetricSerde; import org.apache.druid.query.aggregation.SerializablePairLongStringComplexMetricSerde; import org.apache.druid.query.aggregation.any.DoubleAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.FloatAnyAggregatorFactory; @@ -83,6 +86,10 @@ public AggregatorsModule() ComplexMetrics.registerSerde(PreComputedHyperUniquesSerde.TYPE_NAME, new PreComputedHyperUniquesSerde()); ComplexMetrics.registerSerde(SerializablePairLongStringComplexMetricSerde.TYPE_NAME, new SerializablePairLongStringComplexMetricSerde()); + ComplexMetrics.registerSerde(SerializablePairLongFloatComplexMetricSerde.TYPE_NAME, new SerializablePairLongFloatComplexMetricSerde()); + ComplexMetrics.registerSerde(SerializablePairLongDoubleComplexMetricSerde.TYPE_NAME, new SerializablePairLongDoubleComplexMetricSerde()); + ComplexMetrics.registerSerde(SerializablePairLongLongComplexMetricSerde.TYPE_NAME, new SerializablePairLongLongComplexMetricSerde()); + setMixInAnnotation(AggregatorFactory.class, AggregatorFactoryMixin.class); setMixInAnnotation(PostAggregator.class, PostAggregatorMixin.class); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java new file mode 100644 index 000000000000..e754fa8c2f58 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public abstract class AbstractSerializableLongObjectPairSerde> extends + ComplexMetricSerde +{ + private final Class pairClass; + + public AbstractSerializableLongObjectPairSerde(Class pairClass) + { + this.pairClass = pairClass; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + @Override + public Class extractedClass() + { + return pairClass; + } + + @Nullable + @Override + public Object extractValue(InputRow inputRow, String metricName) + { + return inputRow.getMetric(metricName); + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create( + segmentWriteOutMedium, + column, + getObjectStrategy() + ); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java new file mode 100644 index 000000000000..e811d6ac4f91 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.collections.SerializablePair; + +import javax.annotation.Nullable; + +public class SerializablePairLongDouble extends SerializablePair +{ + @JsonCreator + public SerializablePairLongDouble(@JsonProperty("lhs") Long lhs, @JsonProperty("rhs") @Nullable Double rhs) + { + super(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java new file mode 100644 index 000000000000..8ebaa2322939 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Comparator; + +public class SerializablePairLongDoubleComplexMetricSerde extends AbstractSerializableLongObjectPairSerde +{ + public static final String TYPE_NAME = "serializablePairLongDouble"; + + private static final Comparator> COMPARATOR = SerializablePair.createNullHandlingComparator( + Double::compare, + true + ); + + public SerializablePairLongDoubleComplexMetricSerde() + { + super(SerializablePairLongDouble.class); + } + + @Override + public String getTypeName() + { + return TYPE_NAME; + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return new ObjectStrategy() + { + @Override + public int compare(SerializablePairLongDouble o1, SerializablePairLongDouble o2) + { + return COMPARATOR.compare(o1, o2); + } + + @Override + public Class getClazz() + { + return SerializablePairLongDouble.class; + } + + @Override + public SerializablePairLongDouble fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); + } else { + return new SerializablePairLongDouble(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongDouble inPair) + { + if (inPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); + bbuf.putLong(inPair.lhs); + if (inPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putDouble(inPair.rhs); + } + return bbuf.array(); + } + }; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java new file mode 100644 index 000000000000..619f2e1528ec --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.collections.SerializablePair; + +import javax.annotation.Nullable; + +public class SerializablePairLongFloat extends SerializablePair +{ + @JsonCreator + public SerializablePairLongFloat(@JsonProperty("lhs") Long lhs, @JsonProperty("rhs") @Nullable Float rhs) + { + super(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java new file mode 100644 index 000000000000..7578c3d7c52d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Comparator; + +public class SerializablePairLongFloatComplexMetricSerde extends AbstractSerializableLongObjectPairSerde +{ + public static final String TYPE_NAME = "serializablePairLongFloat"; + + private static final Comparator> COMPARATOR = SerializablePair.createNullHandlingComparator( + Float::compare, + true + ); + + public SerializablePairLongFloatComplexMetricSerde() + { + super(SerializablePairLongFloat.class); + } + + @Override + public String getTypeName() + { + return TYPE_NAME; + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return new ObjectStrategy() + { + @Override + public int compare(SerializablePairLongFloat o1, SerializablePairLongFloat o2) + { + return COMPARATOR.compare(o1, o2); + } + + @Override + public Class getClazz() + { + return SerializablePairLongFloat.class; + } + + @Override + public SerializablePairLongFloat fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); + } else { + return new SerializablePairLongFloat(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongFloat inPair) + { + if (inPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); + bbuf.putLong(inPair.lhs); + if (inPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putFloat(inPair.rhs); + } + return bbuf.array(); + } + }; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java new file mode 100644 index 000000000000..af06a8c210b4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.collections.SerializablePair; + +import javax.annotation.Nullable; + +public class SerializablePairLongLong extends SerializablePair +{ + @JsonCreator + public SerializablePairLongLong(@JsonProperty("lhs") Long lhs, @JsonProperty("rhs") @Nullable Long rhs) + { + super(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java new file mode 100644 index 000000000000..37fe5eef183f --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Comparator; + +public class SerializablePairLongLongComplexMetricSerde extends AbstractSerializableLongObjectPairSerde +{ + public static final String TYPE_NAME = "serializablePairLongLong"; + + private static final Comparator> COMPARATOR = SerializablePair.createNullHandlingComparator( + Long::compare, + true + ); + + public SerializablePairLongLongComplexMetricSerde() + { + super(SerializablePairLongLong.class); + } + + @Override + public String getTypeName() + { + return TYPE_NAME; + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return new ObjectStrategy() + { + @Override + public int compare(SerializablePairLongLong o1, SerializablePairLongLong o2) + { + return COMPARATOR.compare(o1, o2); + } + + @Override + public Class getClazz() + { + return SerializablePairLongLong.class; + } + + @Override + public SerializablePairLongLong fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); + } else { + return new SerializablePairLongLong(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongLong inPair) + { + if (inPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); + bbuf.putLong(inPair.lhs); + if (inPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putLong(inPair.rhs); + } + return bbuf.array(); + } + }; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java index 8b4a89d0d72e..f713a15de190 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java @@ -19,30 +19,36 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class DoubleFirstAggregator extends NumericFirstAggregator +public class DoubleFirstAggregator extends NumericFirstAggregator { double firstValue; - public DoubleFirstAggregator(BaseLongColumnValueSelector timeSelector, BaseDoubleColumnValueSelector valueSelector) + public DoubleFirstAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); firstValue = 0; } @Override - void setCurrentValue() + void setFirstValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getDouble(); } + @Override + void setFirstValue(Number firstValue) + { + this.firstValue = firstValue.doubleValue(); + } + @Override public Object get() { - return new SerializablePair<>(firstTime, rhsNull ? null : firstValue); + return new SerializablePairLongDouble(firstTime, rhsNull ? null : firstValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index d575b263f0fb..dc042aa2ab93 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -21,17 +21,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.SerializablePairLongDoubleComplexMetricSerde; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -47,11 +47,15 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("doubleFirst") public class DoubleFirstAggregatorFactory extends AggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongDoubleComplexMetricSerde.TYPE_NAME); + private static final Aggregator NIL_AGGREGATOR = new DoubleFirstAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -63,7 +67,9 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new DoubleFirstBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false + ) { @Override @@ -100,13 +106,18 @@ public DoubleFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } else { return new DoubleFirstAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) ); } } @@ -114,13 +125,18 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; } else { return new DoubleFirstBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) ); } } @@ -153,74 +169,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("DoubleFirstAggregatorFactory is not supported during ingestion for rollup"); + return new GenericFirstAggregateCombiner(SerializablePairLongDouble.class); } @Override public AggregatorFactory getCombiningFactory() { - return new DoubleFirstAggregatorFactory(name, name, timeColumn) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - if (pair.rhs != null) { - firstValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleFirstBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putDouble(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new DoubleFirstAggregatorFactory(name, name, timeColumn); } @Override @@ -234,16 +189,16 @@ public Object deserialize(Object object) { Map map = (Map) object; if (map.get("rhs") == null) { - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), null); + return new SerializablePairLongDouble(((Number) map.get("lhs")).longValue(), null); } - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).doubleValue()); + return new SerializablePairLongDouble(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).doubleValue()); } @Override @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? null : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePairLongDouble) object).rhs; } @Override @@ -284,7 +239,7 @@ public byte[] getCacheKey() public ColumnType getIntermediateType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? ColumnType.FLOAT : ColumnType.DOUBLE; + return TYPE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java index dabade475369..c00472e923c9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java @@ -19,20 +19,21 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class DoubleFirstBufferAggregator extends NumericFirstBufferAggregator +public class DoubleFirstBufferAggregator extends NumericFirstBufferAggregator { public DoubleFirstBufferAggregator( BaseLongColumnValueSelector timeSelector, - BaseDoubleColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -42,16 +43,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSector) { - buf.putDouble(position, valueSelector.getDouble()); + buf.putDouble(position, valueSector.getDouble()); + } + + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putDouble(position, value.doubleValue()); } @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java index 2c0f62934ffe..d02bfdb922cd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java @@ -19,33 +19,40 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class FloatFirstAggregator extends NumericFirstAggregator +public class FloatFirstAggregator extends NumericFirstAggregator { float firstValue; public FloatFirstAggregator( BaseLongColumnValueSelector timeSelector, - BaseFloatColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); firstValue = 0; } @Override - void setCurrentValue() + void setFirstValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getFloat(); } + @Override + void setFirstValue(Number firstValue) + { + this.firstValue = firstValue.floatValue(); + } + @Override public Object get() { - return new SerializablePair<>(firstTime, rhsNull ? null : firstValue); + return new SerializablePairLongFloat(firstTime, rhsNull ? null : firstValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index be6a0f6aad97..7884ea4a9280 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -21,17 +21,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.SerializablePairLongFloatComplexMetricSerde; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -47,11 +47,15 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("floatFirst") public class FloatFirstAggregatorFactory extends AggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongFloatComplexMetricSerde.TYPE_NAME); + private static final Aggregator NIL_AGGREGATOR = new FloatFirstAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -63,7 +67,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new FloatFirstBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -98,13 +103,18 @@ public FloatFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } else { return new FloatFirstAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) ); } } @@ -112,13 +122,18 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; } else { return new FloatFirstBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) ); } } @@ -151,73 +166,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("FloatFirstAggregatorFactory is not supported during ingestion for rollup"); + return new GenericFirstAggregateCombiner(SerializablePairLongFloat.class); } @Override public AggregatorFactory getCombiningFactory() { - - return new FloatFirstAggregatorFactory(name, name, timeColumn) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - if (pair.rhs != null) { - firstValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putFloat(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new FloatFirstAggregatorFactory(name, name, timeColumn); } @Override @@ -231,16 +186,16 @@ public Object deserialize(Object object) { Map map = (Map) object; if (map.get("rhs") == null) { - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), null); + return new SerializablePairLongFloat(((Number) map.get("lhs")).longValue(), null); } - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).floatValue()); + return new SerializablePairLongFloat(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).floatValue()); } @Override @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? null : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePairLongFloat) object).rhs; } @Override @@ -281,7 +236,7 @@ public byte[] getCacheKey() public ColumnType getIntermediateType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ColumnType.FLOAT; + return TYPE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java index cf7d272b0085..b8881ee9500f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java @@ -19,20 +19,21 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class FloatFirstBufferAggregator extends NumericFirstBufferAggregator +public class FloatFirstBufferAggregator extends NumericFirstBufferAggregator { public FloatFirstBufferAggregator( BaseLongColumnValueSelector timeSelector, - BaseFloatColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -42,16 +43,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSector) { - buf.putFloat(position, valueSelector.getFloat()); + buf.putFloat(position, valueSector.getFloat()); + } + + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putFloat(position, value.floatValue()); } @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java new file mode 100644 index 000000000000..cd72c306b798 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.ObjectAggregateCombiner; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; + +public class GenericFirstAggregateCombiner> extends ObjectAggregateCombiner +{ + private T firstValue; + private final Class pairClass; + + public GenericFirstAggregateCombiner(Class pairClass) + { + this.pairClass = pairClass; + } + + @Override + public void reset(ColumnValueSelector selector) + { + firstValue = (T) selector.getObject(); + } + + @Override + public void fold(ColumnValueSelector selector) + { + T newValue = (T) selector.getObject(); + if (Longs.compare(firstValue.lhs, newValue.lhs) > 0) { + firstValue = newValue; + } + } + + @Nullable + @Override + public T getObject() + { + return firstValue; + } + + @Override + public Class classOfObject() + { + return pairClass; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java index 8cda544521ea..af070dbb5e8c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java @@ -19,29 +19,36 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class LongFirstAggregator extends NumericFirstAggregator +public class LongFirstAggregator extends NumericFirstAggregator { long firstValue; - public LongFirstAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongFirstAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); firstValue = 0; } @Override - void setCurrentValue() + void setFirstValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getLong(); } + @Override + void setFirstValue(Number firstValue) + { + this.firstValue = firstValue.longValue(); + } + @Override public Object get() { - return new SerializablePair<>(firstTime, rhsNull ? null : firstValue); + return new SerializablePairLongLong(firstTime, rhsNull ? null : firstValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 695d01b3a4a7..3ecaabf15971 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -21,17 +21,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLong; +import org.apache.druid.query.aggregation.SerializablePairLongLongComplexMetricSerde; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -46,11 +46,15 @@ import java.util.List; import java.util.Map; +@JsonTypeName("longFirst") public class LongFirstAggregatorFactory extends AggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongLongComplexMetricSerde.TYPE_NAME); + private static final Aggregator NIL_AGGREGATOR = new LongFirstAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -62,7 +66,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new LongFirstBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -97,13 +102,18 @@ public LongFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } else { return new LongFirstAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) ); } } @@ -111,13 +121,18 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; } else { return new LongFirstBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) ); } } @@ -150,72 +165,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("LongFirstAggregatorFactory is not supported during ingestion for rollup"); + return new GenericFirstAggregateCombiner(SerializablePairLongLong.class); } @Override public AggregatorFactory getCombiningFactory() { - return new LongFirstAggregatorFactory(name, name, timeColumn) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - if (pair.rhs != null) { - firstValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putLong(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new LongFirstAggregatorFactory(name, name, timeColumn); } @Override @@ -229,16 +185,16 @@ public Object deserialize(Object object) { Map map = (Map) object; if (map.get("rhs") == null) { - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), null); + return new SerializablePairLongLong(((Number) map.get("lhs")).longValue(), null); } - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).longValue()); + return new SerializablePairLongLong(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).longValue()); } @Override @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? null : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePairLongLong) object).rhs; } @Override @@ -279,7 +235,7 @@ public byte[] getCacheKey() public ColumnType getIntermediateType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ColumnType.LONG; + return TYPE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java index 582cda160153..426d4c64816d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java @@ -19,16 +19,17 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class LongFirstBufferAggregator extends NumericFirstBufferAggregator +public class LongFirstBufferAggregator extends NumericFirstBufferAggregator { - public LongFirstBufferAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongFirstBufferAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -38,16 +39,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putLong(position, valueSelector.getLong()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putLong(position, value.longValue()); + } + @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java index c8a537438b01..b7f88f963fed 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java @@ -19,28 +19,31 @@ package org.apache.druid.query.aggregation.first; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; /** * Base type for on heap 'first' aggregator for primitive numeric column selectors */ -public abstract class NumericFirstAggregator implements Aggregator +public abstract class NumericFirstAggregator implements Aggregator { private final boolean useDefault = NullHandling.replaceWithDefault(); private final BaseLongColumnValueSelector timeSelector; + private final boolean needsFoldCheck; - final TSelector valueSelector; + final ColumnValueSelector valueSelector; long firstTime; boolean rhsNull; - public NumericFirstAggregator(BaseLongColumnValueSelector timeSelector, TSelector valueSelector) + public NumericFirstAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.needsFoldCheck = needsFoldCheck; firstTime = Long.MAX_VALUE; rhsNull = !useDefault; @@ -49,7 +52,12 @@ public NumericFirstAggregator(BaseLongColumnValueSelector timeSelector, TSelecto /** * Store the current primitive typed 'first' value */ - abstract void setCurrentValue(); + abstract void setFirstValue(ColumnValueSelector valueSelector); + + /** + * Store a non-null first value + */ + abstract void setFirstValue(Number firstValue); @Override public void aggregate() @@ -57,13 +65,33 @@ public void aggregate() if (timeSelector.isNull()) { return; } + + if (needsFoldCheck) { + final Object object = valueSelector.getObject(); + if (object instanceof SerializablePair) { + SerializablePair inPair = (SerializablePair) object; + + if (inPair != null && inPair.lhs < firstTime) { + firstTime = inPair.lhs; + if (inPair.rhs == null) { + rhsNull = true; + } else { + rhsNull = false; + setFirstValue(inPair.rhs); + } + } + return; + } + } + long time = timeSelector.getLong(); if (time < firstTime) { firstTime = time; if (useDefault || !valueSelector.isNull()) { - setCurrentValue(); + setFirstValue(valueSelector); rhsNull = false; } else { + setFirstValue(0); rhsNull = true; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java index 159c6e1317b3..c9911399ed11 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java @@ -19,32 +19,34 @@ package org.apache.druid.query.aggregation.first; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; /** * Base type for buffer based 'first' aggregator for primitive numeric column selectors */ -public abstract class NumericFirstBufferAggregator - implements BufferAggregator +public abstract class NumericFirstBufferAggregator implements BufferAggregator { static final int NULL_OFFSET = Long.BYTES; static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; private final boolean useDefault = NullHandling.replaceWithDefault(); private final BaseLongColumnValueSelector timeSelector; + private final boolean needsFoldCheck; - final TSelector valueSelector; + final ColumnValueSelector valueSelector; - public NumericFirstBufferAggregator(BaseLongColumnValueSelector timeSelector, TSelector valueSelector) + public NumericFirstBufferAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.needsFoldCheck = needsFoldCheck; } /** @@ -55,13 +57,22 @@ public NumericFirstBufferAggregator(BaseLongColumnValueSelector timeSelector, TS /** * Place the primitive value in the buffer at the position of {@link #VALUE_OFFSET} */ - abstract void putValue(ByteBuffer buf, int position); + abstract void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSector); - void updateTimeWithValue(ByteBuffer buf, int position, long time) + abstract void putValue(ByteBuffer buf, int position, Number value); + + void updateTimeWithValue(ByteBuffer buf, int position, long time, ColumnValueSelector valueSelector) { buf.putLong(position, time); buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET); + putValue(buf, position + VALUE_OFFSET, valueSelector); + } + + void updateTimeWithValue(ByteBuffer buf, int position, long time, Number value) + { + buf.putLong(position, time); + buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + putValue(buf, position + VALUE_OFFSET, value); } void updateTimeWithNull(ByteBuffer buf, int position, long time) @@ -89,11 +100,28 @@ public void aggregate(ByteBuffer buf, int position) if (timeSelector.isNull()) { return; } - long time = timeSelector.getLong(); + long firstTime = buf.getLong(position); + if (needsFoldCheck) { + final Object object = valueSelector.getObject(); + if (object instanceof SerializablePair) { + final SerializablePair inPair = (SerializablePair) object; + if (inPair != null && inPair.lhs < firstTime) { + if (inPair.rhs == null) { + updateTimeWithNull(buf, position, inPair.lhs); + } else { + updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + } + } + return; + } + } + + long time = timeSelector.getLong(); + if (time < firstTime) { if (useDefault || !valueSelector.isNull()) { - updateTimeWithValue(buf, position, time); + updateTimeWithValue(buf, position, time, valueSelector); } else { updateTimeWithNull(buf, position, time); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index f7624f4541b3..ed880f50f610 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -133,7 +133,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } @@ -149,7 +149,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java index 6b93be7d7080..0298867bc77b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java @@ -42,7 +42,8 @@ public class StringFirstLastUtils */ public static boolean selectorNeedsFoldCheck( final BaseObjectColumnValueSelector valueSelector, - @Nullable final ColumnCapabilities valueSelectorCapabilities + @Nullable final ColumnCapabilities valueSelectorCapabilities, + Class pairClass ) { if (valueSelectorCapabilities != null && !valueSelectorCapabilities.is(ValueType.COMPLEX)) { @@ -57,8 +58,8 @@ public static boolean selectorNeedsFoldCheck( // Check if the selector class could possibly be a SerializablePairLongString (either a superclass or subclass). final Class clazz = valueSelector.classOfObject(); - return clazz.isAssignableFrom(SerializablePairLongString.class) - || SerializablePairLongString.class.isAssignableFrom(clazz); + return clazz.isAssignableFrom(pairClass) + || pairClass.isAssignableFrom(clazz); } /** diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java index 3f6a1506bad6..e5eca666986d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java @@ -19,30 +19,36 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class DoubleLastAggregator extends NumericLastAggregator +public class DoubleLastAggregator extends NumericLastAggregator { double lastValue; - public DoubleLastAggregator(BaseLongColumnValueSelector timeSelector, BaseDoubleColumnValueSelector valueSelector) + public DoubleLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue() + void setLastValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getDouble(); } + @Override + void setLastValue(Number lastValue) + { + this.lastValue = lastValue.doubleValue(); + } + @Override public Object get() { - return new SerializablePair<>(lastTime, rhsNull ? null : lastValue); + return new SerializablePairLongDouble(lastTime, rhsNull ? null : lastValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 5e3fa6667928..de6e707310f7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -21,21 +21,22 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.SerializablePairLongDoubleComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NumericNilVectorAggregator; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; @@ -56,11 +57,14 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("lastDouble") public class DoubleLastAggregatorFactory extends AggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongDoubleComplexMetricSerde.TYPE_NAME); private static final Aggregator NIL_AGGREGATOR = new DoubleLastAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -72,7 +76,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new DoubleLastBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -105,13 +110,18 @@ public DoubleLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } else { return new DoubleLastAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) ); } } @@ -142,13 +152,18 @@ public VectorAggregator factorizeVector( @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; } else { return new DoubleLastBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) ); } } @@ -181,74 +196,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("DoubleLastAggregatorFactory is not supported during ingestion for rollup"); + return new GenericLastAggregateCombiner(SerializablePairLongDouble.class); } @Override public AggregatorFactory getCombiningFactory() { - return new DoubleLastAggregatorFactory(name, name, timeColumn) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - if (pair.rhs != null) { - lastValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleLastBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putDouble(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new DoubleLastAggregatorFactory(name, name, timeColumn); } @Override @@ -262,16 +216,16 @@ public Object deserialize(Object object) { Map map = (Map) object; if (map.get("rhs") == null) { - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), null); + return new SerializablePairLongDouble(((Number) map.get("lhs")).longValue(), null); } - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).doubleValue()); + return new SerializablePairLongDouble(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).doubleValue()); } @Override @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? null : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePairLongDouble) object).rhs; } @Override @@ -312,7 +266,7 @@ public byte[] getCacheKey() public ColumnType getIntermediateType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? ColumnType.FLOAT : ColumnType.DOUBLE; + return TYPE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java index 8acddce53a83..d605180951c4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java @@ -19,20 +19,21 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class DoubleLastBufferAggregator extends NumericLastBufferAggregator +public class DoubleLastBufferAggregator extends NumericLastBufferAggregator { public DoubleLastBufferAggregator( BaseLongColumnValueSelector timeSelector, - BaseDoubleColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -42,16 +43,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putDouble(position, valueSelector.getDouble()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putDouble(position, value.doubleValue()); + } + @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java index d155bbfae84f..a5e9cf9d324e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -57,7 +57,7 @@ public void initValue(ByteBuffer buf, int position) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java index 1381ccb18b7a..c55fbb997baa 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java @@ -19,30 +19,37 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class FloatLastAggregator extends NumericLastAggregator +public class FloatLastAggregator extends NumericLastAggregator { float lastValue; - public FloatLastAggregator(BaseLongColumnValueSelector timeSelector, BaseFloatColumnValueSelector valueSelector) + public FloatLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue() + void setLastValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getFloat(); } + @Override + void setLastValue(Number lastValue) + { + this.lastValue = lastValue.floatValue(); + } + + @Override public Object get() { - return new SerializablePair<>(lastTime, rhsNull ? null : lastValue); + return new SerializablePairLongFloat(lastTime, rhsNull ? null : lastValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index ff23c3d96dc4..a42491013dbb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -21,21 +21,22 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.SerializablePairLongFloatComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NumericNilVectorAggregator; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; @@ -56,11 +57,14 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("lastFloat") public class FloatLastAggregatorFactory extends AggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongFloatComplexMetricSerde.TYPE_NAME); private static final Aggregator NIL_AGGREGATOR = new FloatLastAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -72,7 +76,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new FloatLastBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -103,13 +108,18 @@ public FloatLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } else { return new FloatLastAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) ); } } @@ -117,13 +127,18 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; } else { return new FloatLastBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) ); } } @@ -179,72 +194,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("FloatLastAggregatorFactory is not supported during ingestion for rollup"); + return new GenericLastAggregateCombiner(SerializablePairLongFloat.class); } @Override public AggregatorFactory getCombiningFactory() { - return new FloatLastAggregatorFactory(name, name, timeColumn) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - if (pair.rhs != null) { - lastValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putFloat(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new FloatLastAggregatorFactory(name, name, timeColumn); } @Override @@ -258,16 +214,16 @@ public Object deserialize(Object object) { Map map = (Map) object; if (map.get("rhs") == null) { - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), null); + return new SerializablePairLongFloat(((Number) map.get("lhs")).longValue(), null); } - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).floatValue()); + return new SerializablePairLongFloat(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).floatValue()); } @Override @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? null : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePairLongFloat) object).rhs; } @Override @@ -309,7 +265,7 @@ public byte[] getCacheKey() public ColumnType getIntermediateType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ColumnType.FLOAT; + return TYPE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java index 95ad6fe5c5ee..68affc9cba90 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java @@ -19,20 +19,22 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class FloatLastBufferAggregator extends NumericLastBufferAggregator +public class FloatLastBufferAggregator extends NumericLastBufferAggregator { public FloatLastBufferAggregator( BaseLongColumnValueSelector timeSelector, - BaseFloatColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck + ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -42,16 +44,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putFloat(position, valueSelector.getFloat()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putFloat(position, value.floatValue()); + } + @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java index 2cb468a09295..b1814b5dc07e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -58,7 +58,7 @@ public void initValue(ByteBuffer buf, int position) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java new file mode 100644 index 000000000000..7ad091199df4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.last; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.ObjectAggregateCombiner; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; + +public class GenericLastAggregateCombiner> extends ObjectAggregateCombiner +{ + private T lastValue; + private final Class pairClass; + + public GenericLastAggregateCombiner(Class pairClass) + { + this.pairClass = pairClass; + } + + @Override + public void reset(ColumnValueSelector selector) + { + lastValue = (T) selector.getObject(); + } + + @Override + public void fold(ColumnValueSelector selector) + { + T newValue = (T) selector.getObject(); + if (Longs.compare(lastValue.lhs, newValue.lhs) <= 0) { + lastValue = newValue; + } + } + + @Nullable + @Override + public T getObject() + { + return lastValue; + } + + @Override + public Class classOfObject() + { + return pairClass; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java index 59a159d2d875..b07c3784aaeb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java @@ -19,29 +19,36 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class LongLastAggregator extends NumericLastAggregator +public class LongLastAggregator extends NumericLastAggregator { long lastValue; - public LongLastAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue() + void setLastValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getLong(); } + @Override + void setLastValue(Number lastValue) + { + this.lastValue = lastValue.longValue(); + } + @Override public Object get() { - return new SerializablePair<>(lastTime, rhsNull ? null : lastValue); + return new SerializablePairLongLong(lastTime, rhsNull ? null : lastValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index a5304fe10927..51063bdb5d94 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -21,20 +21,21 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLong; +import org.apache.druid.query.aggregation.SerializablePairLongLongComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NumericNilVectorAggregator; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; @@ -55,11 +56,15 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("longLast") public class LongLastAggregatorFactory extends AggregatorFactory { + public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongLongComplexMetricSerde.TYPE_NAME); + private static final Aggregator NIL_AGGREGATOR = new LongLastAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -71,7 +76,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new LongLastBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -102,13 +108,18 @@ public LongLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } else { return new LongLastAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) ); } } @@ -116,13 +127,18 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; } else { return new LongLastBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), - valueSelector + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) ); } } @@ -177,72 +193,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("LongLastAggregatorFactory is not supported during ingestion for rollup"); + return new GenericLastAggregateCombiner(SerializablePairLongLong.class); } @Override public AggregatorFactory getCombiningFactory() { - return new LongLastAggregatorFactory(name, name, timeColumn) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - if (pair.rhs != null) { - lastValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongLastBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putLong(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new LongLastAggregatorFactory(name, name, timeColumn); } @Override @@ -256,16 +213,16 @@ public Object deserialize(Object object) { Map map = (Map) object; if (map.get("rhs") == null) { - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), null); + return new SerializablePairLongLong(((Number) map.get("lhs")).longValue(), null); } - return new SerializablePair<>(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).longValue()); + return new SerializablePairLongLong(((Number) map.get("lhs")).longValue(), ((Number) map.get("rhs")).longValue()); } @Override @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? null : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePairLongLong) object).rhs; } @Override @@ -306,7 +263,7 @@ public byte[] getCacheKey() public ColumnType getIntermediateType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ColumnType.LONG; + return TYPE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java index 981ba3e2f665..a4a318329b77 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java @@ -19,16 +19,17 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class LongLastBufferAggregator extends NumericLastBufferAggregator +public class LongLastBufferAggregator extends NumericLastBufferAggregator { - public LongLastBufferAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongLastBufferAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -38,16 +39,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putLong(position, valueSelector.getLong()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putLong(position, value.longValue()); + } + @Override public Object get(ByteBuffer buf, int position) { boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java index 60565f0cdce0..ea91430c80f1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -61,6 +61,6 @@ void putValue(ByteBuffer buf, int position, int index) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java index bc1794f1338e..8d36d534a872 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java @@ -19,47 +19,79 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; /** * Base type for on heap 'last' aggregator for primitive numeric column selectors.. * * This could probably share a base class with {@link org.apache.druid.query.aggregation.first.NumericFirstAggregator} */ -public abstract class NumericLastAggregator implements Aggregator +public abstract class NumericLastAggregator implements Aggregator { private final boolean useDefault = NullHandling.replaceWithDefault(); + private final BaseLongColumnValueSelector timeSelector; + final ColumnValueSelector valueSelector; + final boolean needsFoldCheck; - final TSelector valueSelector; long lastTime; boolean rhsNull; - public NumericLastAggregator(BaseLongColumnValueSelector timeSelector, TSelector valueSelector) + public NumericLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, boolean needsFoldCheck) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.needsFoldCheck = needsFoldCheck; lastTime = Long.MIN_VALUE; rhsNull = !useDefault; } + /** + * Store the current primitive typed 'last' value + */ + abstract void setLastValue(ColumnValueSelector valueSelector); + + abstract void setLastValue(Number lastValue); + @Override public void aggregate() { if (timeSelector.isNull()) { return; } + + if (needsFoldCheck) { + final Object object = valueSelector.getObject(); + if (object instanceof SerializablePair) { + final SerializablePair inPair = (SerializablePair) object; + + if (inPair != null && inPair.lhs >= lastTime) { + lastTime = inPair.lhs; + + if (inPair.rhs == null) { + rhsNull = true; + } else { + rhsNull = false; + setLastValue(inPair.rhs); + + } + } + return; + } + } long time = timeSelector.getLong(); if (time >= lastTime) { lastTime = time; if (useDefault || !valueSelector.isNull()) { - setCurrentValue(); + setLastValue(valueSelector); rhsNull = false; } else { + setLastValue(0); rhsNull = true; } } @@ -70,9 +102,4 @@ public void close() { // nothing to close } - - /** - * Store the current primitive typed 'last' value - */ - abstract void setCurrentValue(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java index 4b741e00dd26..e6f4e6a893b4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java @@ -19,11 +19,12 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; @@ -33,8 +34,7 @@ * This could probably share a base type with * {@link org.apache.druid.query.aggregation.first.NumericFirstBufferAggregator} ... */ -public abstract class NumericLastBufferAggregator - implements BufferAggregator +public abstract class NumericLastBufferAggregator implements BufferAggregator { static final int NULL_OFFSET = Long.BYTES; static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; @@ -42,12 +42,14 @@ public abstract class NumericLastBufferAggregator inPair = (SerializablePair) object; + + if (inPair != null && inPair.lhs >= lastTime) { + if (inPair.rhs == null) { + updateTimeWithNull(buf, position, inPair.lhs); + } else { + updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + } + } + return; + } + } + + long time = timeSelector.getLong(); + if (time >= lastTime) { if (useDefault || !valueSelector.isNull()) { - updateTimeWithValue(buf, position, time); + updateTimeWithValue(buf, position, time, valueSelector); } else { updateTimeWithNull(buf, position, time); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index e1b39edc4add..d53d03892903 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -126,7 +126,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } @@ -142,7 +142,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java index 87d0e3dfdd8e..eb2d99854ae1 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java @@ -210,21 +210,21 @@ public void testResultArraySignature() .add("longSum", ColumnType.LONG) .add("longMin", ColumnType.LONG) .add("longMax", ColumnType.LONG) - .add("longFirst", ColumnType.LONG) - .add("longLast", ColumnType.LONG) + .add("longFirst", null) + .add("longLast", null) .add("longAny", ColumnType.LONG) .add("doubleSum", ColumnType.DOUBLE) .add("doubleMin", ColumnType.DOUBLE) .add("doubleMax", ColumnType.DOUBLE) - .add("doubleFirst", ColumnType.DOUBLE) - .add("doubleLast", ColumnType.DOUBLE) + .add("doubleFirst", null) + .add("doubleLast", null) .add("doubleAny", ColumnType.DOUBLE) .add("doubleMean", null) .add("floatSum", ColumnType.FLOAT) .add("floatMin", ColumnType.FLOAT) .add("floatMax", ColumnType.FLOAT) - .add("floatFirst", ColumnType.FLOAT) - .add("floatLast", ColumnType.FLOAT) + .add("floatFirst", null) + .add("floatLast", null) .add("floatAny", ColumnType.FLOAT) .add("stringFirst", null) .add("stringLast", null) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java index ebe628ab1215..7109a0a47961 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -19,17 +19,20 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.TestDoubleColumnSelectorImpl; import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -52,11 +55,11 @@ public class DoubleFirstAggregationTest extends InitializedNullHandlingTest private double[] doubleValues = {1.1d, 2.7d, 3.5d, 1.3d}; private long[] times = {12, 10, 5344, 7899999}; private long[] customTimes = {2, 1, 3, 4}; - private SerializablePair[] pairs = { - new SerializablePair<>(1467225096L, 134.3d), - new SerializablePair<>(23163L, 1232.212d), - new SerializablePair<>(742L, 18d), - new SerializablePair<>(111111L, 233.5232d) + private SerializablePairLongDouble[] pairs = { + new SerializablePairLongDouble(1467225096L, 134.3d), + new SerializablePairLongDouble(23163L, 1232.212d), + new SerializablePairLongDouble(742L, 18d), + new SerializablePairLongDouble(111111L, 233.5232d) }; @Before @@ -73,6 +76,10 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("customTime")).andReturn(customTimeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")) + .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.DOUBLE)); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(null); + EasyMock.replay(colSelectorFactory); } @@ -158,16 +165,16 @@ public void testDoubleFirstBufferAggregatorWithTimeColumn() @Test public void testCombine() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 785.4); + SerializablePairLongDouble pair1 = new SerializablePairLongDouble(1467225000L, 3.621); + SerializablePairLongDouble pair2 = new SerializablePairLongDouble(1467240000L, 785.4); Assert.assertEquals(pair1, doubleFirstAggFactory.combine(pair1, pair2)); } @Test public void testComparator() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 785.4); + SerializablePairLongDouble pair1 = new SerializablePairLongDouble(1467225000L, 3.621); + SerializablePairLongDouble pair2 = new SerializablePairLongDouble(1467240000L, 785.4); Comparator comparator = doubleFirstAggFactory.getComparator(); Assert.assertEquals(-1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -178,8 +185,8 @@ public void testComparator() @Test public void testComparatorWithNulls() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621); - SerializablePair pair2 = new SerializablePair<>(1467240000L, null); + SerializablePairLongDouble pair1 = new SerializablePairLongDouble(1467225000L, 3.621); + SerializablePairLongDouble pair2 = new SerializablePairLongDouble(1467240000L, null); Comparator comparator = doubleFirstAggFactory.getComparator(); Assert.assertEquals(1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -241,6 +248,28 @@ public void testSerde() throws Exception Assert.assertArrayEquals(doubleFirstAggFactory.getCacheKey(), deserialized.getCacheKey()); } + @Test + public void testDoubleFirstAggregateCombiner() + { + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(pairs); + AggregateCombiner doubleFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + doubleFirstAggregateCombiner.reset(columnSelector); + + Assert.assertEquals(pairs[0], doubleFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + doubleFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[1], doubleFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + doubleFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[2], doubleFirstAggregateCombiner.getObject()); + + doubleFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(pairs[2], doubleFirstAggregateCombiner.getObject()); + } + + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java index bd9bc11d2f1e..0df142677437 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -19,17 +19,20 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.TestFloatColumnSelector; import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -52,11 +55,11 @@ public class FloatFirstAggregationTest extends InitializedNullHandlingTest private float[] floats = {1.1f, 2.7f, 3.5f, 1.3f}; private long[] times = {12, 10, 5344, 7899999}; private long[] customTimes = {2, 1, 3, 4}; - private SerializablePair[] pairs = { - new SerializablePair<>(1467225096L, 134.3f), - new SerializablePair<>(23163L, 1232.212f), - new SerializablePair<>(742L, 18f), - new SerializablePair<>(111111L, 233.5232f) + private SerializablePairLongFloat[] pairs = { + new SerializablePairLongFloat(1467225096L, 134.3f), + new SerializablePairLongFloat(23163L, 1232.212f), + new SerializablePairLongFloat(742L, 18f), + new SerializablePairLongFloat(111111L, 233.5232f) }; @Before @@ -73,6 +76,10 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("customTime")).andReturn(customTimeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector).atLeastOnce(); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector).atLeastOnce(); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")) + .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.FLOAT)); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(null); + EasyMock.replay(colSelectorFactory); } @@ -158,16 +165,16 @@ public void testFloatFirstBufferAggregatorWithTimeColumn() @Test public void testCombine() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621f); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 785.4f); + SerializablePairLongFloat pair1 = new SerializablePairLongFloat(1467225000L, 3.621f); + SerializablePairLongFloat pair2 = new SerializablePairLongFloat(1467240000L, 785.4f); Assert.assertEquals(pair1, floatFirstAggregatorFactory.combine(pair1, pair2)); } @Test public void testComparatorWithNulls() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621f); - SerializablePair pair2 = new SerializablePair<>(1467240000L, null); + SerializablePairLongFloat pair1 = new SerializablePairLongFloat(1467225000L, 3.621f); + SerializablePairLongFloat pair2 = new SerializablePairLongFloat(1467240000L, null); Comparator comparator = floatFirstAggregatorFactory.getComparator(); Assert.assertEquals(1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -228,6 +235,28 @@ public void testSerde() throws Exception Assert.assertArrayEquals(floatFirstAggregatorFactory.getCacheKey(), deserialized.getCacheKey()); } + @Test + public void testFloatFirstAggregateCombiner() + { + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(pairs); + AggregateCombiner floatFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + floatFirstAggregateCombiner.reset(columnSelector); + + Assert.assertEquals(pairs[0], floatFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[1], floatFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[2], floatFirstAggregateCombiner.getObject()); + + floatFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(pairs[2], floatFirstAggregateCombiner.getObject()); + } + + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java index d73eb5511e94..104dbcfa60dd 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -19,16 +19,19 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -51,11 +54,11 @@ public class LongFirstAggregationTest extends InitializedNullHandlingTest private long[] longValues = {185, -216, -128751132, Long.MIN_VALUE}; private long[] times = {1123126751, 1784247991, 1854329816, 1000000000}; private long[] customTimes = {2, 1, 3, 4}; - private SerializablePair[] pairs = { - new SerializablePair<>(1L, 113267L), - new SerializablePair<>(1L, 5437384L), - new SerializablePair<>(6L, 34583458L), - new SerializablePair<>(88L, 34583452L) + private SerializablePairLongLong[] pairs = { + new SerializablePairLongLong(1L, 113267L), + new SerializablePairLongLong(1L, 5437384L), + new SerializablePairLongLong(6L, 34583458L), + new SerializablePairLongLong(88L, 34583452L) }; @Before @@ -72,6 +75,9 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("customTime")).andReturn(customTimeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")) + .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.LONG)); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(null); EasyMock.replay(colSelectorFactory); } @@ -157,16 +163,16 @@ public void testLongFirstBufferAggregatorWithTimeColumn() @Test public void testCombine() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 1263L); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 752713L); + SerializablePairLongLong pair1 = new SerializablePairLongLong(1467225000L, 1263L); + SerializablePairLongLong pair2 = new SerializablePairLongLong(1467240000L, 752713L); Assert.assertEquals(pair1, longFirstAggFactory.combine(pair1, pair2)); } @Test public void testComparatorWithNulls() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 1263L); - SerializablePair pair2 = new SerializablePair<>(1467240000L, null); + SerializablePairLongLong pair1 = new SerializablePairLongLong(1467225000L, 1263L); + SerializablePairLongLong pair2 = new SerializablePairLongLong(1467240000L, null); Comparator comparator = longFirstAggFactory.getComparator(); Assert.assertEquals(1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -227,6 +233,27 @@ public void testSerde() throws Exception Assert.assertArrayEquals(longFirstAggFactory.getCacheKey(), deserialized.getCacheKey()); } + @Test + public void testLongFirstAggregeCombiner() + { + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(pairs); + AggregateCombiner longFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + longFirstAggregateCombiner.reset(columnSelector); + + Assert.assertEquals(pairs[0], longFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + longFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[0], longFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + longFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[0], longFirstAggregateCombiner.getObject()); + + longFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(pairs[2], longFirstAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java index 3648ddc5fea6..bbcf9be37a2b 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -19,17 +19,20 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.TestDoubleColumnSelectorImpl; import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -52,11 +55,11 @@ public class DoubleLastAggregationTest extends InitializedNullHandlingTest private double[] doubles = {1.1897d, 0.001d, 86.23d, 166.228d}; private long[] times = {8224, 6879, 2436, 7888}; private long[] customTimes = {1, 4, 3, 2}; - private SerializablePair[] pairs = { - new SerializablePair<>(52782L, 134.3d), - new SerializablePair<>(65492L, 1232.212d), - new SerializablePair<>(69134L, 18.1233d), - new SerializablePair<>(11111L, 233.5232d) + private SerializablePairLongDouble[] pairs = { + new SerializablePairLongDouble(52782L, 134.3d), + new SerializablePairLongDouble(65492L, 1232.212d), + new SerializablePairLongDouble(69134L, 18.1233d), + new SerializablePairLongDouble(11111L, 233.5232d) }; @Before @@ -73,6 +76,9 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("customTime")).andReturn(customTimeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")) + .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.DOUBLE)); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(null); EasyMock.replay(colSelectorFactory); } @@ -159,16 +165,16 @@ public void testDoubleLastBufferAggregatorWithTimeColumn() @Test public void testCombine() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 785.4); + SerializablePairLongDouble pair1 = new SerializablePairLongDouble(1467225000L, 3.621); + SerializablePairLongDouble pair2 = new SerializablePairLongDouble(1467240000L, 785.4); Assert.assertEquals(pair2, doubleLastAggFactory.combine(pair1, pair2)); } @Test public void testComparatorWithNulls() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621); - SerializablePair pair2 = new SerializablePair<>(1467240000L, null); + SerializablePairLongDouble pair1 = new SerializablePairLongDouble(1467225000L, 3.621); + SerializablePairLongDouble pair2 = new SerializablePairLongDouble(1467240000L, null); Comparator comparator = doubleLastAggFactory.getComparator(); Assert.assertEquals(1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -229,6 +235,32 @@ public void testSerde() throws Exception Assert.assertArrayEquals(doubleLastAggFactory.getCacheKey(), deserialized.getCacheKey()); } + @Test + public void testDoubleLastAggregateCombiner() + { + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(pairs); + AggregateCombiner doubleLastAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + doubleLastAggregateCombiner.reset(columnSelector); + + Assert.assertEquals(pairs[0], doubleLastAggregateCombiner.getObject()); + + columnSelector.increment(); + doubleLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[1], doubleLastAggregateCombiner.getObject()); + + columnSelector.increment(); + doubleLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[2], doubleLastAggregateCombiner.getObject()); + + columnSelector.increment(); + doubleLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[2], doubleLastAggregateCombiner.getObject()); + + doubleLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(pairs[3], doubleLastAggregateCombiner.getObject()); + } + + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java index c1e148abe52c..7030f6671e1b 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -19,17 +19,20 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.TestFloatColumnSelector; import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -52,11 +55,11 @@ public class FloatLastAggregationTest extends InitializedNullHandlingTest private float[] floats = {1.1897f, 0.001f, 86.23f, 166.228f}; private long[] times = {8224, 6879, 2436, 7888}; private long[] customTimes = {1, 4, 3, 2}; - private SerializablePair[] pairs = { - new SerializablePair<>(52782L, 134.3f), - new SerializablePair<>(65492L, 1232.212f), - new SerializablePair<>(69134L, 18.1233f), - new SerializablePair<>(11111L, 233.5232f) + private SerializablePairLongFloat[] pairs = { + new SerializablePairLongFloat(52782L, 134.3f), + new SerializablePairLongFloat(65492L, 1232.212f), + new SerializablePairLongFloat(69134L, 18.1233f), + new SerializablePairLongFloat(11111L, 233.5232f) }; @Before @@ -73,6 +76,9 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("customTime")).andReturn(customTimeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")) + .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.FLOAT)); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(null); EasyMock.replay(colSelectorFactory); } @@ -159,16 +165,16 @@ public void testFloatLastBufferAggregatorWithTimeColumn() @Test public void testCombine() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621f); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 785.4f); + SerializablePairLongFloat pair1 = new SerializablePairLongFloat(1467225000L, 3.621f); + SerializablePairLongFloat pair2 = new SerializablePairLongFloat(1467240000L, 785.4f); Assert.assertEquals(pair2, floatLastAggregatorFactory.combine(pair1, pair2)); } @Test public void testComparatorWithNulls() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 3.621f); - SerializablePair pair2 = new SerializablePair<>(1467240000L, null); + SerializablePairLongFloat pair1 = new SerializablePairLongFloat(1467225000L, 3.621f); + SerializablePairLongFloat pair2 = new SerializablePairLongFloat(1467240000L, null); Comparator comparator = floatLastAggregatorFactory.getComparator(); Assert.assertEquals(1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -229,6 +235,31 @@ public void testSerde() throws Exception Assert.assertArrayEquals(floatLastAggregatorFactory.getCacheKey(), deserialized.getCacheKey()); } + @Test + public void testFloatLastAggregateCombiner() + { + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(pairs); + AggregateCombiner floatLastAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + floatLastAggregateCombiner.reset(columnSelector); + + Assert.assertEquals(pairs[0], floatLastAggregateCombiner.getObject()); + + columnSelector.increment(); + floatLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[1], floatLastAggregateCombiner.getObject()); + + columnSelector.increment(); + floatLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[2], floatLastAggregateCombiner.getObject()); + + columnSelector.increment(); + floatLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[2], floatLastAggregateCombiner.getObject()); + + floatLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(pairs[3], floatLastAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java index cacd905ee054..111f86884251 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java @@ -19,16 +19,19 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -51,11 +54,12 @@ public class LongLastAggregationTest extends InitializedNullHandlingTest private long[] longValues = {23216, 8635, 1547123, Long.MAX_VALUE}; private long[] times = {1467935723, 1467225653, 1601848932, 72515}; private long[] customTimes = {1, 4, 3, 2}; - private SerializablePair[] pairs = { - new SerializablePair<>(12531L, 113267L), - new SerializablePair<>(123L, 5437384L), - new SerializablePair<>(125755L, 34583458L), - new SerializablePair<>(124L, 34283452L) + private SerializablePairLongLong[] pairs = { + new SerializablePairLongLong(12531L, 113267L), + new SerializablePairLongLong(12534L, null), + new SerializablePairLongLong(123L, 5437384L), + new SerializablePairLongLong(125755L, 34583458L), + new SerializablePairLongLong(124L, 34283452L) }; @Before @@ -72,6 +76,9 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("customTime")).andReturn(customTimeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")) + .andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.LONG)); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(null); EasyMock.replay(colSelectorFactory); } @@ -158,16 +165,16 @@ public void testLongLastBufferAggregatorWithTimeColumn() @Test public void testCombine() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 64432L); - SerializablePair pair2 = new SerializablePair<>(1467240000L, 99999L); + SerializablePairLongLong pair1 = new SerializablePairLongLong(1467225000L, 64432L); + SerializablePairLongLong pair2 = new SerializablePairLongLong(1467240000L, 99999L); Assert.assertEquals(pair2, longLastAggFactory.combine(pair1, pair2)); } @Test public void testComparatorWithNulls() { - SerializablePair pair1 = new SerializablePair<>(1467225000L, 1263L); - SerializablePair pair2 = new SerializablePair<>(1467240000L, null); + SerializablePairLongLong pair1 = new SerializablePairLongLong(1467225000L, 1263L); + SerializablePairLongLong pair2 = new SerializablePairLongLong(1467240000L, null); Comparator comparator = longLastAggFactory.getComparator(); Assert.assertEquals(1, comparator.compare(pair1, pair2)); Assert.assertEquals(0, comparator.compare(pair1, pair1)); @@ -186,7 +193,7 @@ public void testLongLastCombiningAggregator() aggregate(agg); Pair result = (Pair) agg.get(); - Pair expected = (Pair) pairs[2]; + Pair expected = (Pair) pairs[3]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs); @@ -209,7 +216,7 @@ public void testLongLastCombiningBufferAggregator() aggregate(agg, buffer, 0); Pair result = (Pair) agg.get(buffer, 0); - Pair expected = (Pair) pairs[2]; + Pair expected = (Pair) pairs[3]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs); @@ -228,6 +235,27 @@ public void testSerde() throws Exception Assert.assertArrayEquals(longLastAggFactory.getCacheKey(), deserialized.getCacheKey()); } + @Test + public void testLongLastAggregateCombiner() + { + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(pairs); + AggregateCombiner longLastAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + longLastAggregateCombiner.reset(columnSelector); + + Assert.assertEquals(pairs[0], longLastAggregateCombiner.getObject()); + + columnSelector.increment(); + longLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[1], longLastAggregateCombiner.getObject()); + + columnSelector.increment(); + longLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(pairs[1], longLastAggregateCombiner.getObject()); + + longLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(pairs[2], longLastAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java index 2a700c08bd48..b5a92bf7e67e 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java @@ -39,6 +39,9 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; @@ -951,9 +954,11 @@ private SerializablePair getIntermediateComplexValue(final ValueType valueType, { switch (valueType) { case LONG: + return new SerializablePairLongLong(123L, (long) dimValue); case DOUBLE: + return new SerializablePairLongDouble(123L, (double) dimValue); case FLOAT: - return new SerializablePair<>(123L, dimValue); + return new SerializablePairLongFloat(123L, (float) dimValue); case STRING: return new SerializablePairLongString(123L, (String) dimValue); default: diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java index f5b5111ca55b..89f7d60453e6 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java @@ -43,6 +43,9 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregator; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; @@ -388,9 +391,11 @@ private SerializablePair getIntermediateComplexValue(final ValueType valueType, { switch (valueType) { case LONG: + return new SerializablePairLongLong(123L, (long) dimValue); case DOUBLE: + return new SerializablePairLongDouble(123L, (double) dimValue); case FLOAT: - return new SerializablePair<>(123L, dimValue); + return new SerializablePairLongFloat(123L, (float) dimValue); case STRING: return new SerializablePairLongString(123L, (String) dimValue); default: diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index fd99ae96e944..825edaeccfee 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -22,7 +22,13 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; import org.apache.druid.segment.data.IncrementalIndexTest; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -61,7 +67,7 @@ public void setUp() indexSpec = IndexSpec.DEFAULT; } - private void testStringFirstLastRollup( + private void testFirstLastRollup( AggregatorFactory[] aggregatorFactories ) throws Exception { @@ -71,6 +77,9 @@ private void testStringFirstLastRollup( { put("d", "d1"); put("m", "m1"); + put("l", 123L); + put("f", 123.0F); + put("dl", 123.5D); } }, new HashMap() @@ -78,6 +87,9 @@ private void testStringFirstLastRollup( { put("d", "d1"); put("m", "m2"); + put("l", 124L); + put("f", 124.0F); + put("dl", 124.5D); } } ); @@ -102,20 +114,26 @@ private void testStringFirstLastRollup( } @Test - public void testStringFirstRollup() throws Exception + public void testFirstRollup() throws Exception { AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{ - new StringFirstAggregatorFactory("m", "m", null, 1024) + new StringFirstAggregatorFactory("m", "m", null, 1024), + new LongFirstAggregatorFactory("l", "l", null), + new FloatFirstAggregatorFactory("f", "f", null), + new DoubleFirstAggregatorFactory("dl", "dl", null), }; - testStringFirstLastRollup(aggregatorFactories); + testFirstLastRollup(aggregatorFactories); } @Test - public void testStringLastRollup() throws Exception + public void testLastRollup() throws Exception { AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{ - new StringLastAggregatorFactory("m", "m", null, 1024) + new StringLastAggregatorFactory("m", "m", null, 1024), + new LongLastAggregatorFactory("l", "l", null), + new FloatLastAggregatorFactory("f", "f", null), + new DoubleLastAggregatorFactory("dl", "dl", null), }; - testStringFirstLastRollup(aggregatorFactories); + testFirstLastRollup(aggregatorFactories); } } From e35a906fd1fcb85158bfbdb6cb73410fc2ee6bd8 Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Wed, 2 Aug 2023 12:17:48 -0700 Subject: [PATCH 02/14] Addressing comments --- .../query/aggregation/first/DoubleFirstAggregator.java | 2 +- .../aggregation/first/DoubleFirstAggregatorFactory.java | 1 - .../druid/query/aggregation/first/FloatFirstAggregator.java | 2 +- .../aggregation/first/FloatFirstAggregatorFactory.java | 1 - .../druid/query/aggregation/first/LongFirstAggregator.java | 2 +- .../query/aggregation/first/LongFirstAggregatorFactory.java | 1 - .../query/aggregation/first/NumericFirstAggregator.java | 6 +++--- .../aggregation/first/NumericFirstBufferAggregator.java | 2 +- .../druid/query/aggregation/last/DoubleLastAggregator.java | 2 +- .../query/aggregation/last/DoubleLastAggregatorFactory.java | 1 - .../druid/query/aggregation/last/FloatLastAggregator.java | 2 +- .../query/aggregation/last/FloatLastAggregatorFactory.java | 1 - .../druid/query/aggregation/last/LongLastAggregator.java | 2 +- .../query/aggregation/last/LongLastAggregatorFactory.java | 1 - .../druid/query/aggregation/last/NumericLastAggregator.java | 6 +++--- .../query/aggregation/last/NumericLastBufferAggregator.java | 2 +- 16 files changed, 14 insertions(+), 20 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java index f713a15de190..cf121f72fc98 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java @@ -34,7 +34,7 @@ public DoubleFirstAggregator(BaseLongColumnValueSelector timeSelector, ColumnVal } @Override - void setFirstValue(ColumnValueSelector valueSelector) + void setFirstValue() { firstValue = valueSelector.getDouble(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index dc042aa2ab93..be3bce7ae02e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -238,7 +238,6 @@ public byte[] getCacheKey() @Override public ColumnType getIntermediateType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde return TYPE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java index d02bfdb922cd..987937c2ba87 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java @@ -38,7 +38,7 @@ public FloatFirstAggregator( } @Override - void setFirstValue(ColumnValueSelector valueSelector) + void setFirstValue() { firstValue = valueSelector.getFloat(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 7884ea4a9280..0336aaaa294d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -235,7 +235,6 @@ public byte[] getCacheKey() @Override public ColumnType getIntermediateType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde return TYPE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java index af070dbb5e8c..d229819382e9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java @@ -34,7 +34,7 @@ public LongFirstAggregator(BaseLongColumnValueSelector timeSelector, ColumnValue } @Override - void setFirstValue(ColumnValueSelector valueSelector) + void setFirstValue() { firstValue = valueSelector.getLong(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 3ecaabf15971..27cad349ead4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -234,7 +234,6 @@ public byte[] getCacheKey() @Override public ColumnType getIntermediateType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde return TYPE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java index b7f88f963fed..b3092377b578 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java @@ -52,7 +52,7 @@ public NumericFirstAggregator(BaseLongColumnValueSelector timeSelector, ColumnVa /** * Store the current primitive typed 'first' value */ - abstract void setFirstValue(ColumnValueSelector valueSelector); + abstract void setFirstValue(); /** * Store a non-null first value @@ -71,7 +71,7 @@ public void aggregate() if (object instanceof SerializablePair) { SerializablePair inPair = (SerializablePair) object; - if (inPair != null && inPair.lhs < firstTime) { + if (inPair.lhs < firstTime) { firstTime = inPair.lhs; if (inPair.rhs == null) { rhsNull = true; @@ -88,7 +88,7 @@ public void aggregate() if (time < firstTime) { firstTime = time; if (useDefault || !valueSelector.isNull()) { - setFirstValue(valueSelector); + setFirstValue(); rhsNull = false; } else { setFirstValue(0); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java index c9911399ed11..4531ee71bcd6 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java @@ -106,7 +106,7 @@ public void aggregate(ByteBuffer buf, int position) final Object object = valueSelector.getObject(); if (object instanceof SerializablePair) { final SerializablePair inPair = (SerializablePair) object; - if (inPair != null && inPair.lhs < firstTime) { + if (inPair.lhs < firstTime) { if (inPair.rhs == null) { updateTimeWithNull(buf, position, inPair.lhs); } else { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java index e5eca666986d..009e9c82333f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java @@ -34,7 +34,7 @@ public DoubleLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValu } @Override - void setLastValue(ColumnValueSelector valueSelector) + void setLastValue() { lastValue = valueSelector.getDouble(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index de6e707310f7..55da0be236a3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -265,7 +265,6 @@ public byte[] getCacheKey() @Override public ColumnType getIntermediateType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde return TYPE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java index c55fbb997baa..63147a92db77 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java @@ -34,7 +34,7 @@ public FloatLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValue } @Override - void setLastValue(ColumnValueSelector valueSelector) + void setLastValue() { lastValue = valueSelector.getFloat(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index a42491013dbb..a97eb5f61fd3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -264,7 +264,6 @@ public byte[] getCacheKey() @Override public ColumnType getIntermediateType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde return TYPE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java index b07c3784aaeb..f5f5791da960 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java @@ -34,7 +34,7 @@ public LongLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueS } @Override - void setLastValue(ColumnValueSelector valueSelector) + void setLastValue() { lastValue = valueSelector.getLong(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 51063bdb5d94..bb407400106c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -262,7 +262,6 @@ public byte[] getCacheKey() @Override public ColumnType getIntermediateType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde return TYPE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java index 8d36d534a872..36032e71ecef 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java @@ -54,7 +54,7 @@ public NumericLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnVal /** * Store the current primitive typed 'last' value */ - abstract void setLastValue(ColumnValueSelector valueSelector); + abstract void setLastValue(); abstract void setLastValue(Number lastValue); @@ -70,7 +70,7 @@ public void aggregate() if (object instanceof SerializablePair) { final SerializablePair inPair = (SerializablePair) object; - if (inPair != null && inPair.lhs >= lastTime) { + if (inPair.lhs >= lastTime) { lastTime = inPair.lhs; if (inPair.rhs == null) { @@ -88,7 +88,7 @@ public void aggregate() if (time >= lastTime) { lastTime = time; if (useDefault || !valueSelector.isNull()) { - setLastValue(valueSelector); + setLastValue(); rhsNull = false; } else { setLastValue(0); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java index e6f4e6a893b4..9de6f9968878 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java @@ -110,7 +110,7 @@ public void aggregate(ByteBuffer buf, int position) if (object instanceof SerializablePair) { final SerializablePair inPair = (SerializablePair) object; - if (inPair != null && inPair.lhs >= lastTime) { + if (inPair.lhs >= lastTime) { if (inPair.rhs == null) { updateTimeWithNull(buf, position, inPair.lhs); } else { From 5389438490205a147a7a15054fd91f49ea20f09a Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Wed, 2 Aug 2023 15:10:52 -0700 Subject: [PATCH 03/14] static check jdk-17 fix --- .../druid/segment/IndexMergerRollupTest.java | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index 825edaeccfee..ddea2cbf62b0 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -20,6 +20,7 @@ package org.apache.druid.segment; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; @@ -72,26 +73,22 @@ private void testFirstLastRollup( ) throws Exception { List> eventsList = Arrays.asList( - new HashMap() - { - { - put("d", "d1"); - put("m", "m1"); - put("l", 123L); - put("f", 123.0F); - put("dl", 123.5D); - } - }, - new HashMap() - { - { - put("d", "d1"); - put("m", "m2"); - put("l", 124L); - put("f", 124.0F); - put("dl", 124.5D); - } - } + new HashMap<>( + ImmutableMap.of( + "d", "d1", + "m", "m2", + "l", 124L, + "f", 124.0F, + "dl", 124.5D + )), + new HashMap<>( + ImmutableMap.of( + "d", "d1", + "m", "m2", + "l", 124L, + "f", 124.0F, + "dl", 124.5D + )) ); final File tempDir = temporaryFolder.newFolder(); From 90fd6e76adc0949c052f23147f92f4f54f724f6d Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Thu, 3 Aug 2023 13:52:38 -0700 Subject: [PATCH 04/14] update index/query for tests-ex/IT cases --- .../wikipedia_merge_index_queries.json | 38 ++++++++++++++++++- .../indexer/wikipedia_merge_index_task.json | 30 +++++++++++++++ ...merge_reindex_druid_input_source_task.json | 30 +++++++++++++++ .../indexer/wikipedia_merge_reindex_task.json | 30 +++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_queries.json b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_queries.json index ab4674999b5d..25ea61496165 100644 --- a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_queries.json +++ b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_queries.json @@ -26,6 +26,36 @@ "type":"stringLast", "name":"latest_user", "fieldName":"last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longFirst", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ] }, @@ -35,7 +65,13 @@ "event" : { "continent":"Asia", "earliest_user":"masterYi", - "latest_user":"stringer" + "latest_user":"stringer", + "double_first_delta": 111.0, + "double_last_delta": -9.0, + "long_first_delta": 111, + "long_last_delta": -9, + "float_first_delta": 111.0, + "float_last_delta": -9.0 } } ] } diff --git a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_task.json b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_task.json index 36bb6a9c27ad..c787e37c8f94 100644 --- a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_task.json +++ b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_index_task.json @@ -40,6 +40,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "delta" } ], "granularitySpec": { diff --git a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json index 9daae62c8d42..348aff886455 100644 --- a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json +++ b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json @@ -56,6 +56,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ] } diff --git a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json index 4a8b60e9466f..47e657e9e7f7 100644 --- a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json +++ b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json @@ -37,6 +37,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "delta" } ], "granularitySpec": { From 8fc627686307d2e70148262dfbc9f123e613a2e1 Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Mon, 7 Aug 2023 14:13:35 -0700 Subject: [PATCH 05/14] fixing batch IT failure --- .../indexer/wikipedia_merge_reindex_task.json | 12 ++++++------ .../AbstractSerializableLongObjectPairSerde.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 2 +- .../aggregation/last/FloatLastAggregatorFactory.java | 2 +- .../aggregation/last/NumericLastAggregator.java | 1 - 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json index 47e657e9e7f7..865ab3a24641 100644 --- a/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json +++ b/integration-tests-ex/cases/src/test/resources/indexer/wikipedia_merge_reindex_task.json @@ -41,32 +41,32 @@ { "type": "doubleFirst", "name": "double_first_delta", - "fieldName": "delta" + "fieldName": "double_first_delta" }, { "type": "doubleLast", "name": "double_last_delta", - "fieldName": "delta" + "fieldName": "double_last_delta" }, { "type": "longFirst", "name": "long_first_delta", - "fieldName": "delta" + "fieldName": "long_first_delta" }, { "type": "longLast", "name": "long_last_delta", - "fieldName": "delta" + "fieldName": "long_last_delta" }, { "type": "floatFirst", "name": "float_first_delta", - "fieldName": "delta" + "fieldName": "float_first_delta" }, { "type": "floatLast", "name": "float_last_delta", - "fieldName": "delta" + "fieldName": "float_last_delta" } ], "granularitySpec": { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java index e754fa8c2f58..c2c657ceabfa 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java @@ -58,7 +58,7 @@ public Class extractedClass() @Override public Object extractValue(InputRow inputRow, String metricName) { - return inputRow.getMetric(metricName); + return inputRow.getRaw(metricName); } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 55da0be236a3..7001c3c36d14 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -57,7 +57,7 @@ import java.util.Map; import java.util.Objects; -@JsonTypeName("lastDouble") +@JsonTypeName("doubleLast") public class DoubleLastAggregatorFactory extends AggregatorFactory { public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongDoubleComplexMetricSerde.TYPE_NAME); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index a97eb5f61fd3..c13e0441410f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -57,7 +57,7 @@ import java.util.Map; import java.util.Objects; -@JsonTypeName("lastFloat") +@JsonTypeName("floatLast") public class FloatLastAggregatorFactory extends AggregatorFactory { public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongFloatComplexMetricSerde.TYPE_NAME); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java index 36032e71ecef..159939450ee8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java @@ -78,7 +78,6 @@ public void aggregate() } else { rhsNull = false; setLastValue(inPair.rhs); - } } return; From 258213b3a3354220f6637453d966e9aa47b93b9b Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Mon, 2 Oct 2023 14:18:27 -0700 Subject: [PATCH 06/14] avoid GenericIndexed for creating columns in favor of SerializablePairLong*ComplexColumn.Builder --- ...stractSerializableLongObjectPairSerde.java | 26 +- ...SerializablePairLongObjectBufferStore.java | 116 ++++++ ...erializablePairLongObjectColumnHeader.java | 121 ++++++ ...lizablePairLongObjectColumnSerializer.java | 103 ++++++ ...PairLongObjectDeltaEncodedStagedSerde.java | 107 ++++++ ...izablePairLongObjectSimpleStagedSerde.java | 88 +++++ ...SerializablePairLongDoubleBufferStore.java | 69 ++++ ...erializablePairLongDoubleColumnHeader.java | 39 ++ ...lizablePairLongDoubleColumnSerializer.java | 51 +++ ...rializablePairLongDoubleComplexColumn.java | 129 +++++++ ...zablePairLongDoubleComplexMetricSerde.java | 23 ++ ...PairLongDoubleDeltaEncodedStagedSerde.java | 60 +++ ...izablePairLongDoubleSimpleStagedSerde.java | 51 +++ .../SerializablePairLongFloatBufferStore.java | 69 ++++ ...SerializablePairLongFloatColumnHeader.java | 39 ++ ...alizablePairLongFloatColumnSerializer.java | 51 +++ ...erializablePairLongFloatComplexColumn.java | 129 +++++++ ...izablePairLongFloatComplexMetricSerde.java | 24 ++ ...ePairLongFloatDeltaEncodedStagedSerde.java | 60 +++ ...lizablePairLongFloatSimpleStagedSerde.java | 51 +++ .../SerializablePairLongLongBufferStore.java | 68 ++++ .../SerializablePairLongLongColumnHeader.java | 39 ++ ...ializablePairLongLongColumnSerializer.java | 51 +++ ...SerializablePairLongLongComplexColumn.java | 129 +++++++ ...lizablePairLongLongComplexMetricSerde.java | 23 ++ ...lePairLongLongDeltaEncodedStagedSerde.java | 60 +++ ...alizablePairLongLongSimpleStagedSerde.java | 51 +++ ...lizablePairLongStringColumnSerializer.java | 25 +- ...alizablePairLongDoubleBufferStoreTest.java | 350 ++++++++++++++++++ ...ePairLongDoubleComplexMetricSerdeTest.java | 238 ++++++++++++ ...LongDoubleDeltaEncodedStagedSerdeTest.java | 99 +++++ ...ializablePairLongFloatBufferStoreTest.java | 350 ++++++++++++++++++ ...lePairLongFloatComplexMetricSerdeTest.java | 238 ++++++++++++ ...rLongFloatDeltaEncodedStagedSerdeTest.java | 99 +++++ ...rializablePairLongLongBufferStoreTest.java | 350 ++++++++++++++++++ ...blePairLongLongComplexMetricSerdeTest.java | 238 ++++++++++++ ...irLongLongDeltaEncodedStagedSerdeTest.java | 99 +++++ ...ablePairLongLongSimpleStagedSerdeTest.java | 69 ++++ 38 files changed, 3941 insertions(+), 41 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnHeader.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnSerializer.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnHeader.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnSerializer.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnHeader.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnSerializer.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java index c2c657ceabfa..049dc4a4f6db 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java @@ -21,24 +21,17 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.data.input.InputRow; -import org.apache.druid.segment.GenericColumnSerializer; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.data.GenericIndexed; -import org.apache.druid.segment.serde.ComplexColumnPartSupplier; import org.apache.druid.segment.serde.ComplexMetricExtractor; import org.apache.druid.segment.serde.ComplexMetricSerde; -import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; -import org.apache.druid.segment.writeout.SegmentWriteOutMedium; import javax.annotation.Nullable; -import java.nio.ByteBuffer; public abstract class AbstractSerializableLongObjectPairSerde> extends ComplexMetricSerde { private final Class pairClass; - public AbstractSerializableLongObjectPairSerde(Class pairClass) + AbstractSerializableLongObjectPairSerde(Class pairClass) { this.pairClass = pairClass; } @@ -62,21 +55,4 @@ public Object extractValue(InputRow inputRow, String metricName) } }; } - - @Override - public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) - { - final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); - } - - @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) - { - return LargeColumnSupportedComplexColumnSerializer.create( - segmentWriteOutMedium, - column, - getObjectStrategy() - ); - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java new file mode 100644 index 000000000000..2c77a4b0919a --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; +import org.apache.druid.segment.serde.Serializer; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.serde.cell.CellWriter; +import org.apache.druid.segment.serde.cell.IOIterator; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.channels.WritableByteChannel; + +public abstract class AbstractSerializablePairLongObjectBufferStore> +{ + private final SerializedStorage serializedStorage; + + long minValue = Long.MAX_VALUE; + long maxValue = Long.MIN_VALUE; + + AbstractSerializablePairLongObjectBufferStore(SerializedStorage serializedStorage) + { + this.serializedStorage = serializedStorage; + } + + public void store(@Nullable T pairLongObject) throws IOException + { + if (pairLongObject != null && pairLongObject.lhs != null) { + minValue = Math.min(minValue, pairLongObject.lhs); + maxValue = Math.max(maxValue, pairLongObject.lhs); + } + + serializedStorage.store(pairLongObject); + } + + public TransferredBuffer transferToRowWriter( + ByteBufferProvider byteBufferProvider, + SegmentWriteOutMedium segmentWriteOutMedium + ) throws IOException + { + AbstractSerializablePairLongObjectColumnHeader columnHeader = createColumnHeader(); + AbstractSerializablePairLongObjectDeltaEncodedStagedSerde deltaEncodedSerde = createDeltaEncodedSerde(columnHeader); + + try (CellWriter cellWriter = new CellWriter.Builder(segmentWriteOutMedium).setByteBufferProvider(byteBufferProvider) + .build()) { + try (IOIterator bufferIterator = iterator()) { + while (bufferIterator.hasNext()) { + T pairLongObject = bufferIterator.next(); + byte[] serialized = deltaEncodedSerde.serialize(pairLongObject); + + cellWriter.write(serialized); + } + + cellWriter.close(); + + return new TransferredBuffer(cellWriter, columnHeader); + } + } + } + + public abstract AbstractSerializablePairLongObjectColumnHeader createColumnHeader(); + public abstract AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createDeltaEncodedSerde(AbstractSerializablePairLongObjectColumnHeader columnHeader); + + public IOIterator iterator() throws IOException + { + return serializedStorage.iterator(); + } + + public static class TransferredBuffer implements Serializer + { + private final CellWriter cellWriter; + private final AbstractSerializablePairLongObjectColumnHeader columnHeader; + + public TransferredBuffer( + CellWriter cellWriter, + AbstractSerializablePairLongObjectColumnHeader columnHeader + ) + { + this.cellWriter = cellWriter; + this.columnHeader = columnHeader; + } + + @Override + public long getSerializedSize() throws IOException + { + return columnHeader.getSerializedSize() + cellWriter.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + { + columnHeader.transferTo(channel); + cellWriter.writeTo(channel, smoosher); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java new file mode 100644 index 000000000000..38f0529c2cfc --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.segment.serde.cell.LongSerializer; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.Locale; + +public abstract class AbstractSerializablePairLongObjectColumnHeader> +{ + public static final int EXPECTED_VERSION = 3; + private static final int HEADER_SIZE_BYTES = 4; + private static final int USE_INTEGER_MASK = 0x80; + private static final int VERSION_INDEX = 0; + private static final int ENCODING_INDEX = 1; + + private final byte[] bytes; + final long minValue; + + AbstractSerializablePairLongObjectColumnHeader(byte[] bytes, long minValue) + { + this.bytes = bytes; + this.minValue = minValue; + } + + AbstractSerializablePairLongObjectColumnHeader(int version, boolean useIntegerDeltas, long minTimestamp) + { + this.minValue = minTimestamp; + bytes = new byte[HEADER_SIZE_BYTES]; + Preconditions.checkArgument(version <= 255, "max version 255"); + bytes[VERSION_INDEX] = (byte) version; + + if (useIntegerDeltas) { + bytes[ENCODING_INDEX] |= USE_INTEGER_MASK; + } + } + + static AbstractSerializablePairLongObjectColumnHeader fromBuffer(ByteBuffer byteBuffer, Class pairClass) + { + byte[] bytes = new byte[HEADER_SIZE_BYTES]; + byteBuffer.get(bytes); + long minTimestamp = byteBuffer.getLong(); + + if (pairClass.isAssignableFrom(SerializablePairLongLong.class)) { + return new SerializablePairLongLongColumnHeader(bytes, minTimestamp); + } + + if (pairClass.isAssignableFrom(SerializablePairLongDouble.class)) { + return new SerializablePairLongDoubleColumnHeader(bytes, minTimestamp); + } + + if (pairClass.isAssignableFrom(SerializablePairLongFloat.class)) { + return new SerializablePairLongFloatColumnHeader(bytes, minTimestamp); + } + + throw new RE(String.format(Locale.ENGLISH, "Unsupported pairClass type: %s", pairClass.getSimpleName())); + } + + public abstract AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createSerde(); + + public void transferTo(WritableByteChannel channel) throws IOException + { + LongSerializer longSerializer = new LongSerializer(); + + channel.write(ByteBuffer.wrap(bytes)); + channel.write(longSerializer.serialize(minValue)); + } + + public int getVersion() + { + return 0XFF & bytes[VERSION_INDEX]; + } + + public boolean isUseIntegerDeltas() + { + return (bytes[ENCODING_INDEX] & USE_INTEGER_MASK) != 0; + } + + public long getMinValue() + { + return minValue; + } + + public int getSerializedSize() + { + return HEADER_SIZE_BYTES + Long.BYTES; + } + + @Override + public String toString() + { + return Objects.toStringHelper(this) + .add("bytes", bytes) + .add("minValue", minValue) + .toString(); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java new file mode 100644 index 000000000000..21ba824f9e26 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.serde.cell.StagedSerde; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import java.io.IOException; +import java.nio.channels.WritableByteChannel; + +public abstract class AbstractSerializablePairLongObjectColumnSerializer> implements + GenericColumnSerializer +{ + public final StagedSerde stagedSerde; + final SegmentWriteOutMedium segmentWriteOutMedium; + private final ByteBufferProvider byteBufferProvider; + + State state = State.START; + AbstractSerializablePairLongObjectBufferStore bufferStore; + private AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer; + + AbstractSerializablePairLongObjectColumnSerializer( + StagedSerde stagedSerde, + SegmentWriteOutMedium segmentWriteOutMedium, + ByteBufferProvider byteBufferProvider + ) + { + this.stagedSerde = stagedSerde; + this.segmentWriteOutMedium = segmentWriteOutMedium; + this.byteBufferProvider = byteBufferProvider; + } + + @Override + public void serialize(ColumnValueSelector selector) throws IOException + { + Preconditions.checkState(state == State.OPEN, "serialize called in invalid state %s", state); + T pairLongObject = selector.getObject(); + bufferStore.store(pairLongObject); + } + + @Override + public long getSerializedSize() throws IOException + { + Preconditions.checkState( + state != State.START, + "getSerializedSize called in invalid state %s (must have opened at least)", + state + ); + + transferToRowWriterIfNecessary(); + + return transferredBuffer.getSerializedSize(); + } + + @Override + public void writeTo( + WritableByteChannel channel, + FileSmoosher smoosher + ) throws IOException + { + Preconditions.checkState(state != State.START, "writeTo called in invalid state %s", state); + transferToRowWriterIfNecessary(); + transferredBuffer.writeTo(channel, smoosher); + } + + private void transferToRowWriterIfNecessary() throws IOException + { + if (state == State.OPEN) { + transferredBuffer = bufferStore.transferToRowWriter(byteBufferProvider, segmentWriteOutMedium); + state = State.CLOSED; + } + } + + enum State + { + START, + OPEN, + CLOSED, + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java new file mode 100644 index 000000000000..2abbc88e104a --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.segment.serde.cell.StagedSerde; +import org.apache.druid.segment.serde.cell.StorableBuffer; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Locale; + +public abstract class AbstractSerializablePairLongObjectDeltaEncodedStagedSerde> implements + StagedSerde +{ + + final long minValue; + final boolean useIntegerDelta; + private final Class pairClass; + + AbstractSerializablePairLongObjectDeltaEncodedStagedSerde( + long minValue, + boolean useIntegerDelta, + Class pairClass + ) + { + this.minValue = minValue; + this.useIntegerDelta = useIntegerDelta; + this.pairClass = pairClass; + } + + @Override + public StorableBuffer serializeDelayed(@Nullable T value) + { + if (value == null) { + return StorableBuffer.EMPTY; + } + + Object rhsObject = value.getRhs(); + + return new StorableBuffer() + { + @Override + public void store(ByteBuffer byteBuffer) + { + Preconditions.checkNotNull(value.lhs, String.format(Locale.ENGLISH, "Long in %s must be non-null", pairClass.getSimpleName())); + + long delta = value.lhs - minValue; + + Preconditions.checkState(delta >= 0 || delta == value.lhs); + + if (useIntegerDelta) { + byteBuffer.putInt(Ints.checkedCast(delta)); + } else { + byteBuffer.putLong(delta); + } + + if (rhsObject != null) { + if (pairClass.isAssignableFrom(SerializablePairLongLong.class)) { + byteBuffer.putLong((long) rhsObject); + } else if (pairClass.isAssignableFrom(SerializablePairLongDouble.class)) { + byteBuffer.putDouble((double) rhsObject); + } else if (pairClass.isAssignableFrom(SerializablePairLongFloat.class)) { + byteBuffer.putFloat((float) rhsObject); + } + } + } + + @Override + public int getSerializedSize() + { + int rhsBytes = 0; + + if (rhsObject != null) { + if (pairClass.isAssignableFrom(SerializablePairLongLong.class)) { + rhsBytes = Long.BYTES; + } else if (pairClass.isAssignableFrom(SerializablePairLongDouble.class)) { + rhsBytes = Double.BYTES; + } else if (pairClass.isAssignableFrom(SerializablePairLongFloat.class)) { + rhsBytes = Float.BYTES; + } + } + + return (useIntegerDelta ? Integer.BYTES : Long.BYTES) + rhsBytes; + } + }; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java new file mode 100644 index 000000000000..95a1b8cc3097 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.segment.serde.cell.StagedSerde; +import org.apache.druid.segment.serde.cell.StorableBuffer; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Locale; + +public abstract class AbstractSerializablePairLongObjectSimpleStagedSerde> implements StagedSerde +{ + + private final Class pairCLass; + + AbstractSerializablePairLongObjectSimpleStagedSerde(Class pairCLass) + { + this.pairCLass = pairCLass; + } + + @Override + public StorableBuffer serializeDelayed( + @Nullable T value + ) + { + if (value == null) { + return StorableBuffer.EMPTY; + } + + Object rhsObject = value.getRhs(); + + return new StorableBuffer() + { + @Override + public void store(ByteBuffer byteBuffer) + { + Preconditions.checkNotNull(value.getLhs(), String.format(Locale.ENGLISH, "Long in %s must be non-null", pairCLass.getSimpleName())); + byteBuffer.putLong(value.getLhs()); + if (rhsObject != null) { + if (pairCLass.isAssignableFrom(SerializablePairLongLong.class)) { + byteBuffer.putLong((long) rhsObject); + } else if (pairCLass.isAssignableFrom(SerializablePairLongDouble.class)) { + byteBuffer.putDouble((double) rhsObject); + } else if (pairCLass.isAssignableFrom(SerializablePairLongFloat.class)) { + byteBuffer.putFloat((float) rhsObject); + } + } + } + + @Override + public int getSerializedSize() + { + int rhsBytes = 0; + + if (rhsObject != null) { + if (pairCLass.isAssignableFrom(SerializablePairLongLong.class)) { + rhsBytes = Long.BYTES; + } else if (pairCLass.isAssignableFrom(SerializablePairLongDouble.class)) { + rhsBytes = Double.BYTES; + } else if (pairCLass.isAssignableFrom(SerializablePairLongFloat.class)) { + rhsBytes = Float.BYTES; + } + } + return Long.BYTES + rhsBytes; + } + }; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java new file mode 100644 index 000000000000..b5c22c159dfe --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nonnull; + +public class SerializablePairLongDoubleBufferStore extends AbstractSerializablePairLongObjectBufferStore +{ + public SerializablePairLongDoubleBufferStore(SerializedStorage serializedStorage) + { + super(serializedStorage); + } + + + @Override + @Nonnull + public AbstractSerializablePairLongObjectColumnHeader createColumnHeader() + { + long maxDelta = maxValue - minValue; + SerializablePairLongDoubleColumnHeader columnHeader; + + if (minValue < maxValue && maxDelta < 0 || minValue > maxValue) { + // true iff + // 1. we have overflow in our range || 2. we have only seen null values + // in this case, effectively disable delta encoding by using longs and a min value of 0 + maxDelta = Long.MAX_VALUE; + minValue = 0; + } + + if (maxDelta <= Integer.MAX_VALUE) { + columnHeader = new SerializablePairLongDoubleColumnHeader( + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + true, + minValue + ); + } else { + columnHeader = new SerializablePairLongDoubleColumnHeader( + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + false, + minValue + ); + } + + return columnHeader; + } + + @Override + public AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createDeltaEncodedSerde(AbstractSerializablePairLongObjectColumnHeader columnHeader) + { + return new SerializablePairLongDoubleDeltaEncodedStagedSerde(minValue, columnHeader.isUseIntegerDeltas()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnHeader.java new file mode 100644 index 000000000000..7a09114c0e77 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnHeader.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +public class SerializablePairLongDoubleColumnHeader extends AbstractSerializablePairLongObjectColumnHeader +{ + SerializablePairLongDoubleColumnHeader(byte[] bytes, long minValue) + { + super(bytes, minValue); + } + + SerializablePairLongDoubleColumnHeader(int version, boolean useIntegerDeltas, long minTimestamp) + { + super(version, useIntegerDeltas, minTimestamp); + } + + @Override + public SerializablePairLongDoubleDeltaEncodedStagedSerde createSerde() + { + return new SerializablePairLongDoubleDeltaEncodedStagedSerde(minValue, isUseIntegerDeltas()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnSerializer.java new file mode 100644 index 000000000000..84b2b706c8f6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleColumnSerializer.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import java.io.IOException; + +public class SerializablePairLongDoubleColumnSerializer extends AbstractSerializablePairLongObjectColumnSerializer +{ + + public SerializablePairLongDoubleColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, + ByteBufferProvider byteBufferProvider + ) + { + super(new SerializablePairLongDoubleSimpleStagedSerde(), segmentWriteOutMedium, byteBufferProvider); + } + + @Override + public void open() throws IOException + { + Preconditions.checkState(state == State.START || state == State.OPEN, "open called in invalid state %s", state); + + if (state == State.START) { + bufferStore = new SerializablePairLongDoubleBufferStore( + new SerializedStorage<>(segmentWriteOutMedium.makeWriteOutBytes(), stagedSerde) + ); + state = State.OPEN; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java new file mode 100644 index 000000000000..3bdb4bfa7b78 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexColumn.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.serde.cell.CellReader; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongDoubleComplexColumn implements ComplexColumn +{ + private final Closer closer; + private final int serializedSize; + private final CellReader cellReader; + private final AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde; + + public SerializablePairLongDoubleComplexColumn( + CellReader cellReader, + AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde, + Closer closer, + int serializedSize + ) + { + this.cellReader = cellReader; + this.serde = serde; + this.closer = closer; + this.serializedSize = serializedSize; + } + + @Override + public Class getClazz() + { + return SerializablePairLongDouble.class; + } + + @Override + public String getTypeName() + { + return SerializablePairLongDoubleComplexMetricSerde.TYPE_NAME; + } + + @Override + public Object getRowValue(int rowNum) + { + return serde.deserialize(cellReader.getCell(rowNum)); + } + + @Override + public int getLength() + { + return serializedSize; + } + + @Override + public void close() + { + try { + closer.close(); + } + catch (IOException e) { + throw new RE(e, "error closing " + getClass().getName()); + } + } + + public static class Builder + { + private final int serializedSize; + private final AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde; + private final CellReader.Builder cellReaderBuilder; + + public Builder(ByteBuffer buffer) + { + ByteBuffer masterByteBuffer = buffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + + serializedSize = masterByteBuffer.remaining(); + + AbstractSerializablePairLongObjectColumnHeader columnHeader = + AbstractSerializablePairLongObjectColumnHeader.fromBuffer(masterByteBuffer, SerializablePairLongDouble.class); + + Preconditions.checkArgument( + columnHeader.getVersion() == AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + "version %s expected, got %s", + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + columnHeader.getVersion() + ); + + serde = columnHeader.createSerde(); + cellReaderBuilder = new CellReader.Builder(masterByteBuffer); + } + + public Builder setByteBufferProvier(ByteBufferProvider byteBufferProvider) + { + cellReaderBuilder.setByteBufferProvider(byteBufferProvider); + return this; + } + + public SerializablePairLongDoubleComplexColumn build() + { + Closer closer = Closer.create(); + CellReader cellReader = cellReaderBuilder.build(); + closer.register(cellReader); + + return new SerializablePairLongDoubleComplexColumn(cellReader, serde, closer, serializedSize); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java index 8ebaa2322939..9506524f9d49 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java @@ -21,7 +21,11 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.cell.NativeClearedByteBufferProvider; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -47,6 +51,25 @@ public String getTypeName() return TYPE_NAME; } + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return new SerializablePairLongDoubleColumnSerializer( + segmentWriteOutMedium, + NativeClearedByteBufferProvider.INSTANCE + ); + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + SerializablePairLongDoubleComplexColumn.Builder builder = + new SerializablePairLongDoubleComplexColumn.Builder(buffer) + .setByteBufferProvier(NativeClearedByteBufferProvider.INSTANCE); + + columnBuilder.setComplexColumnSupplier(builder::build); + } + @Override public ObjectStrategy getObjectStrategy() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java new file mode 100644 index 000000000000..5278ef691175 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongDoubleDeltaEncodedStagedSerde extends AbstractSerializablePairLongObjectDeltaEncodedStagedSerde +{ + + public SerializablePairLongDoubleDeltaEncodedStagedSerde(long minValue, boolean useIntegerDelta) + { + super(minValue, useIntegerDelta, SerializablePairLongDouble.class); + } + + @Nullable + @Override + public SerializablePairLongDouble deserialize(ByteBuffer byteBuffer) + { + if (byteBuffer.remaining() == 0) { + return null; + } + + ByteBuffer readOnlyBuffer = byteBuffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + long lhs; + + if (useIntegerDelta) { + lhs = readOnlyBuffer.getInt(); + } else { + lhs = readOnlyBuffer.getLong(); + } + + lhs += minValue; + + Double rhs = null; + if (readOnlyBuffer.hasRemaining()) { + rhs = readOnlyBuffer.getDouble(); + } + + return new SerializablePairLongDouble(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java new file mode 100644 index 000000000000..5f62a7ed3a65 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongDoubleSimpleStagedSerde extends AbstractSerializablePairLongObjectSimpleStagedSerde +{ + public SerializablePairLongDoubleSimpleStagedSerde() + { + super(SerializablePairLongDouble.class); + } + + @Nullable + @Override + public SerializablePairLongDouble deserialize(ByteBuffer byteBuffer) + { + if (byteBuffer.remaining() == 0) { + return null; + } + + ByteBuffer readOnlyBuffer = byteBuffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + long lhs = readOnlyBuffer.getLong(); + + Double rhs = null; + if (readOnlyBuffer.hasRemaining()) { + rhs = readOnlyBuffer.getDouble(); + } + + return new SerializablePairLongDouble(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java new file mode 100644 index 000000000000..9fe268da2023 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nonnull; + +public class SerializablePairLongFloatBufferStore extends AbstractSerializablePairLongObjectBufferStore +{ + public SerializablePairLongFloatBufferStore(SerializedStorage serializedStorage) + { + super(serializedStorage); + } + + + @Override + @Nonnull + public AbstractSerializablePairLongObjectColumnHeader createColumnHeader() + { + long maxDelta = maxValue - minValue; + SerializablePairLongFloatColumnHeader columnHeader; + + if (minValue < maxValue && maxDelta < 0 || minValue > maxValue) { + // true iff + // 1. we have overflow in our range || 2. we have only seen null values + // in this case, effectively disable delta encoding by using longs and a min value of 0 + maxDelta = Long.MAX_VALUE; + minValue = 0; + } + + if (maxDelta <= Integer.MAX_VALUE) { + columnHeader = new SerializablePairLongFloatColumnHeader( + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + true, + minValue + ); + } else { + columnHeader = new SerializablePairLongFloatColumnHeader( + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + false, + minValue + ); + } + + return columnHeader; + } + + @Override + public AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createDeltaEncodedSerde(AbstractSerializablePairLongObjectColumnHeader columnHeader) + { + return new SerializablePairLongFloatDeltaEncodedStagedSerde(minValue, columnHeader.isUseIntegerDeltas()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnHeader.java new file mode 100644 index 000000000000..4f2ac021a9dd --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnHeader.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +public class SerializablePairLongFloatColumnHeader extends AbstractSerializablePairLongObjectColumnHeader +{ + SerializablePairLongFloatColumnHeader(byte[] bytes, long minValue) + { + super(bytes, minValue); + } + + SerializablePairLongFloatColumnHeader(int version, boolean useIntegerDeltas, long minTimestamp) + { + super(version, useIntegerDeltas, minTimestamp); + } + + @Override + public SerializablePairLongFloatDeltaEncodedStagedSerde createSerde() + { + return new SerializablePairLongFloatDeltaEncodedStagedSerde(minValue, isUseIntegerDeltas()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnSerializer.java new file mode 100644 index 000000000000..6d96b99f1597 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatColumnSerializer.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import java.io.IOException; + +public class SerializablePairLongFloatColumnSerializer extends AbstractSerializablePairLongObjectColumnSerializer +{ + + public SerializablePairLongFloatColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, + ByteBufferProvider byteBufferProvider + ) + { + super(new SerializablePairLongFloatSimpleStagedSerde(), segmentWriteOutMedium, byteBufferProvider); + } + + @Override + public void open() throws IOException + { + Preconditions.checkState(state == State.START || state == State.OPEN, "open called in invalid state %s", state); + + if (state == State.START) { + bufferStore = new SerializablePairLongFloatBufferStore( + new SerializedStorage<>(segmentWriteOutMedium.makeWriteOutBytes(), stagedSerde) + ); + state = State.OPEN; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java new file mode 100644 index 000000000000..14b7f268c2d5 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.serde.cell.CellReader; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongFloatComplexColumn implements ComplexColumn +{ + private final Closer closer; + private final int serializedSize; + private final CellReader cellReader; + private final AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde; + + public SerializablePairLongFloatComplexColumn( + CellReader cellReader, + AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde, + Closer closer, + int serializedSize + ) + { + this.cellReader = cellReader; + this.serde = serde; + this.closer = closer; + this.serializedSize = serializedSize; + } + + @Override + public Class getClazz() + { + return SerializablePairLongFloat.class; + } + + @Override + public String getTypeName() + { + return SerializablePairLongFloatComplexMetricSerde.TYPE_NAME; + } + + @Override + public Object getRowValue(int rowNum) + { + return serde.deserialize(cellReader.getCell(rowNum)); + } + + @Override + public int getLength() + { + return serializedSize; + } + + @Override + public void close() + { + try { + closer.close(); + } + catch (IOException e) { + throw new RE(e, "error closing " + getClass().getName()); + } + } + + public static class Builder + { + private final int serializedSize; + private final AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde; + private final CellReader.Builder cellReaderBuilder; + + public Builder(ByteBuffer buffer) + { + ByteBuffer masterByteBuffer = buffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + + serializedSize = masterByteBuffer.remaining(); + + AbstractSerializablePairLongObjectColumnHeader columnHeader = + AbstractSerializablePairLongObjectColumnHeader.fromBuffer(masterByteBuffer, SerializablePairLongFloat.class); + + Preconditions.checkArgument( + columnHeader.getVersion() == AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + "version %s expected, got %s", + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + columnHeader.getVersion() + ); + + serde = columnHeader.createSerde(); + cellReaderBuilder = new CellReader.Builder(masterByteBuffer); + } + + public Builder setByteBufferProvier(ByteBufferProvider byteBufferProvider) + { + cellReaderBuilder.setByteBufferProvider(byteBufferProvider); + return this; + } + + public SerializablePairLongFloatComplexColumn build() + { + Closer closer = Closer.create(); + CellReader cellReader = cellReaderBuilder.build(); + closer.register(cellReader); + + return new SerializablePairLongFloatComplexColumn(cellReader, serde, closer, serializedSize); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java index 7578c3d7c52d..137e8e58268f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java @@ -21,7 +21,11 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.cell.NativeClearedByteBufferProvider; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -47,6 +51,26 @@ public String getTypeName() return TYPE_NAME; } + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return new SerializablePairLongFloatColumnSerializer( + segmentWriteOutMedium, + NativeClearedByteBufferProvider.INSTANCE + ); + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + SerializablePairLongFloatComplexColumn.Builder builder = + new SerializablePairLongFloatComplexColumn.Builder(buffer) + .setByteBufferProvier(NativeClearedByteBufferProvider.INSTANCE); + + columnBuilder.setComplexColumnSupplier(builder::build); + } + + @Override public ObjectStrategy getObjectStrategy() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java new file mode 100644 index 000000000000..059fa4bd0b6d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongFloatDeltaEncodedStagedSerde extends AbstractSerializablePairLongObjectDeltaEncodedStagedSerde +{ + + public SerializablePairLongFloatDeltaEncodedStagedSerde(long minValue, boolean useIntegerDelta) + { + super(minValue, useIntegerDelta, SerializablePairLongFloat.class); + } + + @Nullable + @Override + public SerializablePairLongFloat deserialize(ByteBuffer byteBuffer) + { + if (byteBuffer.remaining() == 0) { + return null; + } + + ByteBuffer readOnlyBuffer = byteBuffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + long lhs; + + if (useIntegerDelta) { + lhs = readOnlyBuffer.getInt(); + } else { + lhs = readOnlyBuffer.getLong(); + } + + lhs += minValue; + + Float rhs = null; + if (readOnlyBuffer.hasRemaining()) { + rhs = readOnlyBuffer.getFloat(); + } + + return new SerializablePairLongFloat(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java new file mode 100644 index 000000000000..d8d6f9b5ef47 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongFloatSimpleStagedSerde extends AbstractSerializablePairLongObjectSimpleStagedSerde +{ + public SerializablePairLongFloatSimpleStagedSerde() + { + super(SerializablePairLongFloat.class); + } + + @Nullable + @Override + public SerializablePairLongFloat deserialize(ByteBuffer byteBuffer) + { + if (byteBuffer.remaining() == 0) { + return null; + } + + ByteBuffer readOnlyBuffer = byteBuffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + long lhs = readOnlyBuffer.getLong(); + + Float rhs = null; + if (readOnlyBuffer.hasRemaining()) { + rhs = readOnlyBuffer.getFloat(); + } + + return new SerializablePairLongFloat(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java new file mode 100644 index 000000000000..1dd1465dde4c --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nonnull; + +public class SerializablePairLongLongBufferStore extends AbstractSerializablePairLongObjectBufferStore +{ + public SerializablePairLongLongBufferStore(SerializedStorage serializedStorage) + { + super(serializedStorage); + } + + @Override + @Nonnull + public AbstractSerializablePairLongObjectColumnHeader createColumnHeader() + { + long maxDelta = maxValue - minValue; + SerializablePairLongLongColumnHeader columnHeader; + + if (minValue < maxValue && maxDelta < 0 || minValue > maxValue) { + // true iff + // 1. we have overflow in our range || 2. we have only seen null values + // in this case, effectively disable delta encoding by using longs and a min value of 0 + maxDelta = Long.MAX_VALUE; + minValue = 0; + } + + if (maxDelta <= Integer.MAX_VALUE) { + columnHeader = new SerializablePairLongLongColumnHeader( + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + true, + minValue + ); + } else { + columnHeader = new SerializablePairLongLongColumnHeader( + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + false, + minValue + ); + } + + return columnHeader; + } + + @Override + public AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createDeltaEncodedSerde(AbstractSerializablePairLongObjectColumnHeader columnHeader) + { + return new SerializablePairLongLongDeltaEncodedStagedSerde(minValue, columnHeader.isUseIntegerDeltas()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnHeader.java new file mode 100644 index 000000000000..6280883c5229 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnHeader.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +public class SerializablePairLongLongColumnHeader extends AbstractSerializablePairLongObjectColumnHeader +{ + SerializablePairLongLongColumnHeader(byte[] bytes, long minValue) + { + super(bytes, minValue); + } + + SerializablePairLongLongColumnHeader(int version, boolean useIntegerDeltas, long minTimestamp) + { + super(version, useIntegerDeltas, minTimestamp); + } + + @Override + public SerializablePairLongLongDeltaEncodedStagedSerde createSerde() + { + return new SerializablePairLongLongDeltaEncodedStagedSerde(minValue, isUseIntegerDeltas()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnSerializer.java new file mode 100644 index 000000000000..00018628c398 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongColumnSerializer.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import java.io.IOException; + +public class SerializablePairLongLongColumnSerializer extends AbstractSerializablePairLongObjectColumnSerializer +{ + + public SerializablePairLongLongColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, + ByteBufferProvider byteBufferProvider + ) + { + super(new SerializablePairLongLongSimpleStagedSerde(), segmentWriteOutMedium, byteBufferProvider); + } + + @Override + public void open() throws IOException + { + Preconditions.checkState(state == State.START || state == State.OPEN, "open called in invalid state %s", state); + + if (state == State.START) { + bufferStore = new SerializablePairLongLongBufferStore( + new SerializedStorage<>(segmentWriteOutMedium.makeWriteOutBytes(), stagedSerde) + ); + state = State.OPEN; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java new file mode 100644 index 000000000000..1a80541f2966 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.serde.cell.ByteBufferProvider; +import org.apache.druid.segment.serde.cell.CellReader; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongLongComplexColumn implements ComplexColumn +{ + private final Closer closer; + private final int serializedSize; + private final CellReader cellReader; + private final AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde; + + public SerializablePairLongLongComplexColumn( + CellReader cellReader, + AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde, + Closer closer, + int serializedSize + ) + { + this.cellReader = cellReader; + this.serde = serde; + this.closer = closer; + this.serializedSize = serializedSize; + } + + @Override + public Class getClazz() + { + return SerializablePairLongLong.class; + } + + @Override + public String getTypeName() + { + return SerializablePairLongLongComplexMetricSerde.TYPE_NAME; + } + + @Override + public Object getRowValue(int rowNum) + { + return serde.deserialize(cellReader.getCell(rowNum)); + } + + @Override + public int getLength() + { + return serializedSize; + } + + @Override + public void close() + { + try { + closer.close(); + } + catch (IOException e) { + throw new RE(e, "error closing " + getClass().getName()); + } + } + + public static class Builder + { + private final int serializedSize; + private final AbstractSerializablePairLongObjectDeltaEncodedStagedSerde serde; + private final CellReader.Builder cellReaderBuilder; + + public Builder(ByteBuffer buffer) + { + ByteBuffer masterByteBuffer = buffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + + serializedSize = masterByteBuffer.remaining(); + + AbstractSerializablePairLongObjectColumnHeader columnHeader = + AbstractSerializablePairLongObjectColumnHeader.fromBuffer(masterByteBuffer, SerializablePairLongLong.class); + + Preconditions.checkArgument( + columnHeader.getVersion() == AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + "version %s expected, got %s", + AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + columnHeader.getVersion() + ); + + serde = columnHeader.createSerde(); + cellReaderBuilder = new CellReader.Builder(masterByteBuffer); + } + + public Builder setByteBufferProvier(ByteBufferProvider byteBufferProvider) + { + cellReaderBuilder.setByteBufferProvider(byteBufferProvider); + return this; + } + + public SerializablePairLongLongComplexColumn build() + { + Closer closer = Closer.create(); + CellReader cellReader = cellReaderBuilder.build(); + closer.register(cellReader); + + return new SerializablePairLongLongComplexColumn(cellReader, serde, closer, serializedSize); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java index 37fe5eef183f..fececb873101 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java @@ -21,7 +21,11 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.cell.NativeClearedByteBufferProvider; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -47,6 +51,25 @@ public String getTypeName() return TYPE_NAME; } + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return new SerializablePairLongLongColumnSerializer( + segmentWriteOutMedium, + NativeClearedByteBufferProvider.INSTANCE + ); + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + SerializablePairLongLongComplexColumn.Builder builder = + new SerializablePairLongLongComplexColumn.Builder(buffer) + .setByteBufferProvier(NativeClearedByteBufferProvider.INSTANCE); + + columnBuilder.setComplexColumnSupplier(builder::build); + } + @Override public ObjectStrategy getObjectStrategy() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java new file mode 100644 index 000000000000..bf0414031a88 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongLongDeltaEncodedStagedSerde extends AbstractSerializablePairLongObjectDeltaEncodedStagedSerde +{ + + public SerializablePairLongLongDeltaEncodedStagedSerde(long minValue, boolean useIntegerDelta) + { + super(minValue, useIntegerDelta, SerializablePairLongLong.class); + } + + @Nullable + @Override + public SerializablePairLongLong deserialize(ByteBuffer byteBuffer) + { + if (byteBuffer.remaining() == 0) { + return null; + } + + ByteBuffer readOnlyBuffer = byteBuffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + long lhs; + + if (useIntegerDelta) { + lhs = readOnlyBuffer.getInt(); + } else { + lhs = readOnlyBuffer.getLong(); + } + + lhs += minValue; + + Long rhs = null; + if (readOnlyBuffer.hasRemaining()) { + rhs = readOnlyBuffer.getLong(); + } + + return new SerializablePairLongLong(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java new file mode 100644 index 000000000000..c94e842b5289 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class SerializablePairLongLongSimpleStagedSerde extends AbstractSerializablePairLongObjectSimpleStagedSerde +{ + public SerializablePairLongLongSimpleStagedSerde() + { + super(SerializablePairLongLong.class); + } + + @Nullable + @Override + public SerializablePairLongLong deserialize(ByteBuffer byteBuffer) + { + if (byteBuffer.remaining() == 0) { + return null; + } + + ByteBuffer readOnlyBuffer = byteBuffer.asReadOnlyBuffer().order(ByteOrder.nativeOrder()); + long lhs = readOnlyBuffer.getLong(); + + Long rhs = null; + if (readOnlyBuffer.hasRemaining()) { + rhs = readOnlyBuffer.getLong(); + } + + return new SerializablePairLongLong(lhs, rhs); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java index 3930ee6060ba..5962aec928e5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java @@ -48,7 +48,7 @@ public class SerializablePairLongStringColumnSerializer implements GenericColumn private final SegmentWriteOutMedium segmentWriteOutMedium; private final ByteBufferProvider byteBufferProvider; - private State state = State.START; + private AbstractSerializablePairLongObjectColumnSerializer.State state = AbstractSerializablePairLongObjectColumnSerializer.State.START; private SerializablePairLongStringBufferStore bufferStore; private SerializablePairLongStringBufferStore.TransferredBuffer transferredBuffer; @@ -64,20 +64,20 @@ public SerializablePairLongStringColumnSerializer( @Override public void open() throws IOException { - Preconditions.checkState(state == State.START || state == State.OPEN, "open called in invalid state %s", state); + Preconditions.checkState(state == AbstractSerializablePairLongObjectColumnSerializer.State.START || state == AbstractSerializablePairLongObjectColumnSerializer.State.OPEN, "open called in invalid state %s", state); - if (state == State.START) { + if (state == AbstractSerializablePairLongObjectColumnSerializer.State.START) { bufferStore = new SerializablePairLongStringBufferStore( new SerializedStorage<>(segmentWriteOutMedium.makeWriteOutBytes(), STAGED_SERDE) ); - state = State.OPEN; + state = AbstractSerializablePairLongObjectColumnSerializer.State.OPEN; } } @Override public void serialize(ColumnValueSelector selector) throws IOException { - Preconditions.checkState(state == State.OPEN, "serialize called in invalid state %s", state); + Preconditions.checkState(state == AbstractSerializablePairLongObjectColumnSerializer.State.OPEN, "serialize called in invalid state %s", state); SerializablePairLongString pairLongString = selector.getObject(); @@ -88,7 +88,7 @@ public void serialize(ColumnValueSelector public long getSerializedSize() throws IOException { Preconditions.checkState( - state != State.START, + state != AbstractSerializablePairLongObjectColumnSerializer.State.START, "getSerializedSize called in invalid state %s (must have opened at least)", state ); @@ -101,23 +101,16 @@ public long getSerializedSize() throws IOException @Override public void writeTo(WritableByteChannel channel, @Nullable FileSmoosher smoosher) throws IOException { - Preconditions.checkState(state != State.START, "writeTo called in invalid state %s", state); + Preconditions.checkState(state != AbstractSerializablePairLongObjectColumnSerializer.State.START, "writeTo called in invalid state %s", state); transferToRowWriterIfNecessary(); transferredBuffer.writeTo(channel, smoosher); } private void transferToRowWriterIfNecessary() throws IOException { - if (state == State.OPEN) { + if (state == AbstractSerializablePairLongObjectColumnSerializer.State.OPEN) { transferredBuffer = bufferStore.transferToRowWriter(byteBufferProvider, segmentWriteOutMedium); - state = State.CLOSED; + state = AbstractSerializablePairLongObjectColumnSerializer.State.CLOSED; } } - - private enum State - { - START, - OPEN, - CLOSED, - } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java new file mode 100644 index 000000000000..69ee858cbd78 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Ints; +import org.apache.druid.collections.ResourceHolder; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.serde.cell.IOIterator; +import org.apache.druid.segment.serde.cell.NativeClearedByteBufferProvider; +import org.apache.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Random; + +public class SerializablePairLongDoubleBufferStoreTest +{ + private final Random random = new Random(0); + private static final int MIN_INTEGER = 100; + private static final long MIN_LONG = 0L; + private final SerializablePairLongDouble[] integerRangeArr = new SerializablePairLongDouble[]{ + new SerializablePairLongDouble((long) MIN_INTEGER, 10D), + new SerializablePairLongDouble(101L, 20D), + new SerializablePairLongDouble(102L, 30D), + }; + + private final SerializablePairLongDouble[] longRangeArr = new SerializablePairLongDouble[]{ + new SerializablePairLongDouble((long) MIN_LONG, 10D), + new SerializablePairLongDouble(101L, 20D), + new SerializablePairLongDouble(102L, 30D), + new SerializablePairLongDouble((long) Integer.MAX_VALUE, 40D), + new SerializablePairLongDouble(Long.MAX_VALUE, 50D), + }; + + private final SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + private SerializablePairLongDoubleBufferStore bufferStore; + + @Before + public void setup() throws Exception + { + bufferStore = new SerializablePairLongDoubleBufferStore( + new SerializedStorage<>( + writeOutMedium.makeWriteOutBytes(), + new SerializablePairLongDoubleSimpleStagedSerde() + ) + ); + } + + @Test + public void testIteratorSimple() throws Exception + { + for (SerializablePairLongDouble value : integerRangeArr) { + bufferStore.store(value); + } + + IOIterator iterator = bufferStore.iterator(); + + int i = 0; + while (iterator.hasNext()) { + Assert.assertEquals(integerRangeArr[i], iterator.next()); + i++; + } + } + + @Test + public void testIteratorNull() throws Exception + { + bufferStore.store(null); + IOIterator iterator = bufferStore.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertNull(iterator.next()); + } + + @Test + public void testIteratorIdempotentHasNext() throws Exception + { + bufferStore.store(integerRangeArr[0]); + + IOIterator iterator = bufferStore.iterator(); + + Assert.assertTrue(iterator.hasNext()); + // expect hasNext() to not modify state + Assert.assertTrue(iterator.hasNext()); + } + + @Test(expected = NoSuchElementException.class) + public void testIteratorEmptyThrows() throws Exception + { + IOIterator iterator = bufferStore.iterator(); + iterator.next(); + } + + @Test + public void testIteratorEmptyHasNext() throws Exception + { + IOIterator iterator = bufferStore.iterator(); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testMinValueUsesInteger() throws Exception + { + for (SerializablePairLongDouble value : integerRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongDoubleColumnHeader columnHeader = (SerializablePairLongDoubleColumnHeader) bufferStore.createColumnHeader(); + Assert.assertEquals(integerRangeArr[0].lhs.longValue(), columnHeader.getMinValue()); + Assert.assertTrue(columnHeader.isUseIntegerDeltas()); + } + + @Test + public void testMinValueUsesLong() throws Exception + { + for (SerializablePairLongDouble value : longRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongDoubleColumnHeader columnHeader = (SerializablePairLongDoubleColumnHeader) bufferStore.createColumnHeader(); + Assert.assertEquals(MIN_LONG, columnHeader.getMinValue()); + Assert.assertFalse(columnHeader.isUseIntegerDeltas()); + } + + @Test + public void testMinValueUsesIntegerSerialization() throws Exception + { + for (SerializablePairLongDouble value : integerRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongDoubleColumnHeader columnHeader = (SerializablePairLongDoubleColumnHeader) bufferStore.createColumnHeader(); + + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { + columnHeader.transferTo(channel); + + ByteBuffer byteBuffer = resourceHolder.get(); + channel.writeTo(byteBuffer); + byteBuffer.flip(); + + SerializablePairLongDoubleColumnHeader deserializedColumnhHeader = + (SerializablePairLongDoubleColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongDouble.class); + Assert.assertEquals(MIN_INTEGER, deserializedColumnhHeader.getMinValue()); + Assert.assertTrue(deserializedColumnhHeader.isUseIntegerDeltas()); + } + } + + @Test + public void testMinValueSerialization() throws Exception + + { + for (SerializablePairLongDouble value : longRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongDoubleColumnHeader columnHeader = (SerializablePairLongDoubleColumnHeader) bufferStore.createColumnHeader(); + + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { + columnHeader.transferTo(channel); + + ByteBuffer byteBuffer = resourceHolder.get(); + + channel.writeTo(byteBuffer); + byteBuffer.flip(); + + SerializablePairLongDoubleColumnHeader deserializedColumnhHeader = + (SerializablePairLongDoubleColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongDouble.class); + Assert.assertEquals(MIN_LONG, deserializedColumnhHeader.getMinValue()); + Assert.assertFalse(deserializedColumnhHeader.isUseIntegerDeltas()); + } + } + + @Test + public void testLargeBuffer() throws Exception + { + SerializablePairLongDouble value = + new SerializablePairLongDouble(Long.MAX_VALUE, Double.POSITIVE_INFINITY); + + bufferStore.store(value); + + IOIterator iterator = bufferStore.iterator(); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(value, iterator.next()); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testLargeValueCount() throws Exception + { + List valueList = new ArrayList<>(); + + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongDouble(Integer.MAX_VALUE + (long) i, random.nextDouble())); + } + + assertBufferedValuesEqual(valueList); + } + + @Test + public void testOverflowTransfer() throws Exception + { + bufferStore.store(new SerializablePairLongDouble(Long.MIN_VALUE, 10D)); + bufferStore.store(new SerializablePairLongDouble(Long.MAX_VALUE, 10D)); + + SerializablePairLongDoubleColumnHeader columnHeader = (SerializablePairLongDoubleColumnHeader) bufferStore.createColumnHeader(); + + Assert.assertEquals(0, columnHeader.getMinValue()); + + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + NativeClearedByteBufferProvider.INSTANCE, + writeOutMedium + ); + + Assert.assertEquals(93, transferredBuffer.getSerializedSize()); + } + + @Test + public void testNullOnlyTransfer() throws Exception + { + bufferStore.store(null); + + bufferStore.store(null); + + bufferStore.store(null); + + SerializablePairLongDoubleColumnHeader columnHeader = (SerializablePairLongDoubleColumnHeader) bufferStore.createColumnHeader(); + + Assert.assertEquals(0, columnHeader.getMinValue()); + + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + NativeClearedByteBufferProvider.INSTANCE, + writeOutMedium + ); + + Assert.assertEquals(59, transferredBuffer.getSerializedSize()); + } + + @Test + public void testTransferIntegerRange() throws Exception + { + for (SerializablePairLongDouble value : integerRangeArr) { + bufferStore.store(value); + } + + Assert.assertTrue(bufferStore.createColumnHeader().isUseIntegerDeltas()); + + assertTransferredValuesEqual(integerRangeArr); + } + + @Test + public void testTransferLongRange() throws Exception + { + for (SerializablePairLongDouble value : longRangeArr) { + bufferStore.store(value); + } + + Assert.assertFalse(bufferStore.createColumnHeader().isUseIntegerDeltas()); + + assertTransferredValuesEqual(longRangeArr); + } + + private void assertBufferedValuesEqual(List input) throws IOException + { + for (SerializablePairLongDouble pairLongLong : input) { + bufferStore.store(pairLongLong); + } + + IOIterator iterator = bufferStore.iterator(); + int i = 0; + + while (iterator.hasNext()) { + Assert.assertEquals(input.get(i), iterator.next()); + i++; + } + + Assert.assertEquals( + StringUtils.format("element count mismatch: expected %s, got %s", input.size(), i), + input.size(), + i + ); + } + + private void assertTransferredValuesEqual(SerializablePairLongDouble[] input) throws IOException + { + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = + bufferStore.transferToRowWriter(NativeClearedByteBufferProvider.INSTANCE, writeOutMedium); + HeapByteBufferWriteOutBytes resultChannel = new HeapByteBufferWriteOutBytes(); + + transferredBuffer.writeTo(resultChannel, null); + + try (SerializablePairLongDoubleComplexColumn column = createComplexColumn(transferredBuffer, resultChannel)) { + for (int i = 0; i < input.length; i++) { + Assert.assertEquals(input[i], column.getRowValue(i)); + } + } + } + + private static SerializablePairLongDoubleComplexColumn createComplexColumn( + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, + HeapByteBufferWriteOutBytes resultChannel + ) throws IOException + { + ByteBuffer byteBuffer = ByteBuffer.allocate(Ints.checkedCast(transferredBuffer.getSerializedSize())); + + resultChannel.readFully(0, byteBuffer); + byteBuffer.flip(); + + SerializablePairLongDoubleComplexMetricSerde complexMetricSerde = new SerializablePairLongDoubleComplexMetricSerde(); + ColumnBuilder builder = new ColumnBuilder(); + + complexMetricSerde.deserializeColumn(byteBuffer, builder); + builder.setType(ValueType.COMPLEX); + + ColumnHolder columnHolder = builder.build(); + SerializablePairLongDoubleComplexColumn column = (SerializablePairLongDoubleComplexColumn) columnHolder.getColumn(); + + return column; + } + +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java new file mode 100644 index 000000000000..178aab7d1cc0 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; + +public class SerializablePairLongDoubleComplexMetricSerdeTest +{ + static { + NullHandling.initializeForTests(); + } + + private static final SerializablePairLongDoubleComplexMetricSerde COMPRESSED_SERDE = + new SerializablePairLongDoubleComplexMetricSerde(); + + // want deterministic test input + private final Random random = new Random(0); + + @Test + public void testSingle() throws Exception + { + assertExpected(ImmutableList.of(new SerializablePairLongDouble(100L, 10D)), 78); + } + + @Test + public void testLargeRHS() throws Exception + { + // single entry spans more than one block in underlying storage + assertExpected(ImmutableList.of(new SerializablePairLongDouble( + 100L, + random.nextDouble() + )), 78); + } + + @Test + public void testCompressable() throws Exception + { + int numLongs = 10; + List valueList = new ArrayList<>(); + List doubleList = new ArrayList<>(); + + for (int i = 0; i < numLongs; i++) { + doubleList.add(random.nextDouble()); + } + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongDouble(Integer.MAX_VALUE + (long) i, doubleList.get(i % numLongs))); + } + + assertExpected(valueList, 80258); + } + + @Test + public void testHighlyCompressable() throws Exception + { + List valueList = new ArrayList<>(); + + Double doubleValue = random.nextDouble(); + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongDouble(Integer.MAX_VALUE + (long) i, doubleValue)); + } + + assertExpected(valueList, 80024); + } + + @Test + public void testRandom() throws Exception + { + List valueList = new ArrayList<>(); + + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongDouble(random.nextLong(), random.nextDouble())); + } + + assertExpected(valueList, 200612); + } + + @Test + public void testNullRHS() throws Exception + { + assertExpected(ImmutableList.of(new SerializablePairLongDouble(100L, null)), 70); + } + + @Test + public void testEmpty() throws Exception + { + // minimum size for empty data + assertExpected(Collections.emptyList(), 57); + } + + @Test + public void testSingleNull() throws Exception + { + assertExpected(Arrays.asList(new SerializablePairLongDouble[]{null}), 58); + } + + @Test + public void testMultipleNull() throws Exception + { + assertExpected(Arrays.asList(null, null, null, null), 59); + } + + private ByteBuffer assertExpected( + List expected, + int expectedCompressedSize + ) throws IOException + { + SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer( + expected, + COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"), + expectedCompressedSize + ).asReadOnlyBuffer(); + + try (ComplexColumn compressedCol = createComplexColumn(compressedBuffer) + ) { + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals(expected.get(i), compressedCol.getRowValue(i)); + } + } + return compressedBuffer; + } + + private ComplexColumn createComplexColumn(ByteBuffer byteBuffer) + { + ColumnBuilder builder = new ColumnBuilder(); + int serializedSize = byteBuffer.remaining(); + + COMPRESSED_SERDE.deserializeColumn(byteBuffer, builder); + builder.setType(ValueType.COMPLEX); + + + ColumnHolder columnHolder = builder.build(); + + final ComplexColumn col = (ComplexColumn) columnHolder.getColumn(); + if (col instanceof SerializablePairLongDoubleComplexColumn) { + Assert.assertEquals(serializedSize, col.getLength()); + } + Assert.assertEquals("serializablePairLongDouble", col.getTypeName()); + Assert.assertEquals(SerializablePairLongDouble.class, col.getClazz()); + + return col; + } + + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static ByteBuffer serializeAllValuesToByteBuffer( + List values, + GenericColumnSerializer serializer, + int expectedSize + ) throws IOException + { + serializer.open(); + + final AtomicReference reference = new AtomicReference<>(null); + ColumnValueSelector valueSelector = + new SingleObjectColumnValueSelector( + SerializablePairLongDouble.class + ) + { + @Nullable + @Override + public SerializablePairLongDouble getObject() + { + return reference.get(); + } + }; + + for (SerializablePairLongDouble selector : values) { + reference.set(selector); + serializer.serialize(valueSelector); + } + + return serializeToByteBuffer(serializer, expectedSize); + } + + private static ByteBuffer serializeToByteBuffer( + GenericColumnSerializer serializer, + int expectedSize + ) throws IOException + { + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + + serializer.writeTo(channel, null); + + ByteBuffer byteBuffer = ByteBuffer.allocate((int) channel.size()).order(ByteOrder.nativeOrder()); + + channel.readFully(0, byteBuffer); + byteBuffer.flip(); + + if (expectedSize > -1) { + Assert.assertEquals(expectedSize, serializer.getSerializedSize()); + } + + Assert.assertEquals(serializer.getSerializedSize(), byteBuffer.limit()); + + return byteBuffer; + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java new file mode 100644 index 000000000000..85e17b14b8fa --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Random; + +public class SerializablePairLongDoubleDeltaEncodedStagedSerdeTest +{ + private static final SerializablePairLongDoubleDeltaEncodedStagedSerde INTEGER_SERDE = + new SerializablePairLongDoubleDeltaEncodedStagedSerde(0L, true); + + private static final SerializablePairLongDoubleDeltaEncodedStagedSerde LONG_SERDE = + new SerializablePairLongDoubleDeltaEncodedStagedSerde(0L, false); + + private final Random random = new Random(0); + + @Test + public void testNull() + { + assertValueEquals(null, 0, INTEGER_SERDE); + } + + @Test + public void testSimpleInteger() + { + assertValueEquals(new SerializablePairLongDouble(100L, 1000000000000.12312312312D), 12, INTEGER_SERDE); + } + + @Test + public void testNullRHSInteger() + { + assertValueEquals(new SerializablePairLongDouble(100L, null), 4, INTEGER_SERDE); + } + + @Test + public void testLargeRHSInteger() + { + assertValueEquals( + new SerializablePairLongDouble(100L, random.nextDouble()), + 12, + INTEGER_SERDE + ); + } + + @Test + public void testSimpleLong() + { + assertValueEquals(new SerializablePairLongDouble(100L, 1000000000000.12312312312D), 16, LONG_SERDE); + } + + @Test + public void testNullRHSLong() + { + assertValueEquals(new SerializablePairLongDouble(100L, null), 8, LONG_SERDE); + } + + @Test + public void testLargeRHSLong() + { + assertValueEquals( + new SerializablePairLongDouble(100L, random.nextDouble()), + 16, + LONG_SERDE + ); + } + + private static void assertValueEquals( + @Nullable SerializablePairLongDouble value, + int size, + SerializablePairLongDoubleDeltaEncodedStagedSerde serde + ) + { + byte[] bytes = serde.serialize(value); + Assert.assertEquals(size, bytes.length); + SerializablePairLongDouble deserialized = serde.deserialize(bytes); + Assert.assertEquals(value, deserialized); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java new file mode 100644 index 000000000000..cb71b49b44b2 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Ints; +import org.apache.druid.collections.ResourceHolder; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.serde.cell.IOIterator; +import org.apache.druid.segment.serde.cell.NativeClearedByteBufferProvider; +import org.apache.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Random; + +public class SerializablePairLongFloatBufferStoreTest +{ + private final Random random = new Random(0); + private static final int MIN_INTEGER = 100; + private static final long MIN_LONG = 0L; + private final SerializablePairLongFloat[] integerRangeArr = new SerializablePairLongFloat[]{ + new SerializablePairLongFloat((long) MIN_INTEGER, 10F), + new SerializablePairLongFloat(101L, 20F), + new SerializablePairLongFloat(102L, 30F), + }; + + private final SerializablePairLongFloat[] longRangeArr = new SerializablePairLongFloat[]{ + new SerializablePairLongFloat((long) MIN_LONG, 10F), + new SerializablePairLongFloat(101L, 20F), + new SerializablePairLongFloat(102L, 30F), + new SerializablePairLongFloat((long) Integer.MAX_VALUE, 40F), + new SerializablePairLongFloat(Long.MAX_VALUE, 50F), + }; + + private final SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + private SerializablePairLongFloatBufferStore bufferStore; + + @Before + public void setup() throws Exception + { + bufferStore = new SerializablePairLongFloatBufferStore( + new SerializedStorage<>( + writeOutMedium.makeWriteOutBytes(), + new SerializablePairLongFloatSimpleStagedSerde() + ) + ); + } + + @Test + public void testIteratorSimple() throws Exception + { + for (SerializablePairLongFloat value : integerRangeArr) { + bufferStore.store(value); + } + + IOIterator iterator = bufferStore.iterator(); + + int i = 0; + while (iterator.hasNext()) { + Assert.assertEquals(integerRangeArr[i], iterator.next()); + i++; + } + } + + @Test + public void testIteratorNull() throws Exception + { + bufferStore.store(null); + IOIterator iterator = bufferStore.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertNull(iterator.next()); + } + + @Test + public void testIteratorIdempotentHasNext() throws Exception + { + bufferStore.store(integerRangeArr[0]); + + IOIterator iterator = bufferStore.iterator(); + + Assert.assertTrue(iterator.hasNext()); + // expect hasNext() to not modify state + Assert.assertTrue(iterator.hasNext()); + } + + @Test(expected = NoSuchElementException.class) + public void testIteratorEmptyThrows() throws Exception + { + IOIterator iterator = bufferStore.iterator(); + iterator.next(); + } + + @Test + public void testIteratorEmptyHasNext() throws Exception + { + IOIterator iterator = bufferStore.iterator(); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testMinValueUsesInteger() throws Exception + { + for (SerializablePairLongFloat value : integerRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongFloatColumnHeader columnHeader = (SerializablePairLongFloatColumnHeader) bufferStore.createColumnHeader(); + Assert.assertEquals(integerRangeArr[0].lhs.longValue(), columnHeader.getMinValue()); + Assert.assertTrue(columnHeader.isUseIntegerDeltas()); + } + + @Test + public void testMinValueUsesLong() throws Exception + { + for (SerializablePairLongFloat value : longRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongFloatColumnHeader columnHeader = (SerializablePairLongFloatColumnHeader) bufferStore.createColumnHeader(); + Assert.assertEquals(MIN_LONG, columnHeader.getMinValue()); + Assert.assertFalse(columnHeader.isUseIntegerDeltas()); + } + + @Test + public void testMinValueUsesIntegerSerialization() throws Exception + { + for (SerializablePairLongFloat value : integerRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongFloatColumnHeader columnHeader = (SerializablePairLongFloatColumnHeader) bufferStore.createColumnHeader(); + + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { + columnHeader.transferTo(channel); + + ByteBuffer byteBuffer = resourceHolder.get(); + channel.writeTo(byteBuffer); + byteBuffer.flip(); + + SerializablePairLongFloatColumnHeader deserializedColumnhHeader = + (SerializablePairLongFloatColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongFloat.class); + Assert.assertEquals(MIN_INTEGER, deserializedColumnhHeader.getMinValue()); + Assert.assertTrue(deserializedColumnhHeader.isUseIntegerDeltas()); + } + } + + @Test + public void testMinValueSerialization() throws Exception + + { + for (SerializablePairLongFloat value : longRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongFloatColumnHeader columnHeader = (SerializablePairLongFloatColumnHeader) bufferStore.createColumnHeader(); + + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { + columnHeader.transferTo(channel); + + ByteBuffer byteBuffer = resourceHolder.get(); + + channel.writeTo(byteBuffer); + byteBuffer.flip(); + + SerializablePairLongFloatColumnHeader deserializedColumnhHeader = + (SerializablePairLongFloatColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongFloat.class); + Assert.assertEquals(MIN_LONG, deserializedColumnhHeader.getMinValue()); + Assert.assertFalse(deserializedColumnhHeader.isUseIntegerDeltas()); + } + } + + @Test + public void testLargeBuffer() throws Exception + { + SerializablePairLongFloat value = + new SerializablePairLongFloat(Long.MAX_VALUE, Float.POSITIVE_INFINITY); + + bufferStore.store(value); + + IOIterator iterator = bufferStore.iterator(); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(value, iterator.next()); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testLargeValueCount() throws Exception + { + List valueList = new ArrayList<>(); + + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongFloat(Integer.MAX_VALUE + (long) i, random.nextFloat())); + } + + assertBufferedValuesEqual(valueList); + } + + @Test + public void testOverflowTransfer() throws Exception + { + bufferStore.store(new SerializablePairLongFloat(Long.MIN_VALUE, 10F)); + bufferStore.store(new SerializablePairLongFloat(Long.MAX_VALUE, 10F)); + + SerializablePairLongFloatColumnHeader columnHeader = (SerializablePairLongFloatColumnHeader) bufferStore.createColumnHeader(); + + Assert.assertEquals(0, columnHeader.getMinValue()); + + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + NativeClearedByteBufferProvider.INSTANCE, + writeOutMedium + ); + + Assert.assertEquals(92, transferredBuffer.getSerializedSize()); + } + + @Test + public void testNullOnlyTransfer() throws Exception + { + bufferStore.store(null); + + bufferStore.store(null); + + bufferStore.store(null); + + SerializablePairLongFloatColumnHeader columnHeader = (SerializablePairLongFloatColumnHeader) bufferStore.createColumnHeader(); + + Assert.assertEquals(0, columnHeader.getMinValue()); + + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + NativeClearedByteBufferProvider.INSTANCE, + writeOutMedium + ); + + Assert.assertEquals(59, transferredBuffer.getSerializedSize()); + } + + @Test + public void testTransferIntegerRange() throws Exception + { + for (SerializablePairLongFloat value : integerRangeArr) { + bufferStore.store(value); + } + + Assert.assertTrue(bufferStore.createColumnHeader().isUseIntegerDeltas()); + + assertTransferredValuesEqual(integerRangeArr); + } + + @Test + public void testTransferLongRange() throws Exception + { + for (SerializablePairLongFloat value : longRangeArr) { + bufferStore.store(value); + } + + Assert.assertFalse(bufferStore.createColumnHeader().isUseIntegerDeltas()); + + assertTransferredValuesEqual(longRangeArr); + } + + private void assertBufferedValuesEqual(List input) throws IOException + { + for (SerializablePairLongFloat pairLongLong : input) { + bufferStore.store(pairLongLong); + } + + IOIterator iterator = bufferStore.iterator(); + int i = 0; + + while (iterator.hasNext()) { + Assert.assertEquals(input.get(i), iterator.next()); + i++; + } + + Assert.assertEquals( + StringUtils.format("element count mismatch: expected %s, got %s", input.size(), i), + input.size(), + i + ); + } + + private void assertTransferredValuesEqual(SerializablePairLongFloat[] input) throws IOException + { + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = + bufferStore.transferToRowWriter(NativeClearedByteBufferProvider.INSTANCE, writeOutMedium); + HeapByteBufferWriteOutBytes resultChannel = new HeapByteBufferWriteOutBytes(); + + transferredBuffer.writeTo(resultChannel, null); + + try (SerializablePairLongFloatComplexColumn column = createComplexColumn(transferredBuffer, resultChannel)) { + for (int i = 0; i < input.length; i++) { + Assert.assertEquals(input[i], column.getRowValue(i)); + } + } + } + + private static SerializablePairLongFloatComplexColumn createComplexColumn( + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, + HeapByteBufferWriteOutBytes resultChannel + ) throws IOException + { + ByteBuffer byteBuffer = ByteBuffer.allocate(Ints.checkedCast(transferredBuffer.getSerializedSize())); + + resultChannel.readFully(0, byteBuffer); + byteBuffer.flip(); + + SerializablePairLongFloatComplexMetricSerde complexMetricSerde = new SerializablePairLongFloatComplexMetricSerde(); + ColumnBuilder builder = new ColumnBuilder(); + + complexMetricSerde.deserializeColumn(byteBuffer, builder); + builder.setType(ValueType.COMPLEX); + + ColumnHolder columnHolder = builder.build(); + SerializablePairLongFloatComplexColumn column = (SerializablePairLongFloatComplexColumn) columnHolder.getColumn(); + + return column; + } + +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java new file mode 100644 index 000000000000..97d6493110c0 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; + +public class SerializablePairLongFloatComplexMetricSerdeTest +{ + static { + NullHandling.initializeForTests(); + } + + private static final SerializablePairLongFloatComplexMetricSerde COMPRESSED_SERDE = + new SerializablePairLongFloatComplexMetricSerde(); + + // want deterministic test input + private final Random random = new Random(0); + + @Test + public void testSingle() throws Exception + { + assertExpected(ImmutableList.of(new SerializablePairLongFloat(100L, 10F)), 74); + } + + @Test + public void testLargeRHS() throws Exception + { + // single entry spans more than one block in underlying storage + assertExpected(ImmutableList.of(new SerializablePairLongFloat( + 100L, + random.nextFloat() + )), 74); + } + + @Test + public void testCompressable() throws Exception + { + int numLongs = 10; + List valueList = new ArrayList<>(); + List floatList = new ArrayList<>(); + + for (int i = 0; i < numLongs; i++) { + floatList.add(random.nextFloat()); + } + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongFloat(Integer.MAX_VALUE + (long) i, floatList.get(i % numLongs))); + } + + assertExpected(valueList, 80124); + } + + @Test + public void testHighlyCompressable() throws Exception + { + List valueList = new ArrayList<>(); + + Float floatValue = random.nextFloat(); + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongFloat(Integer.MAX_VALUE + (long) i, floatValue)); + } + + assertExpected(valueList, 79970); + } + + @Test + public void testRandom() throws Exception + { + List valueList = new ArrayList<>(); + + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongFloat(random.nextLong(), random.nextFloat())); + } + + assertExpected(valueList, 160464); + } + + @Test + public void testNullRHS() throws Exception + { + assertExpected(ImmutableList.of(new SerializablePairLongFloat(100L, null)), 70); + } + + @Test + public void testEmpty() throws Exception + { + // minimum size for empty data + assertExpected(Collections.emptyList(), 57); + } + + @Test + public void testSingleNull() throws Exception + { + assertExpected(Arrays.asList(new SerializablePairLongFloat[]{null}), 58); + } + + @Test + public void testMultipleNull() throws Exception + { + assertExpected(Arrays.asList(null, null, null, null), 59); + } + + private ByteBuffer assertExpected( + List expected, + int expectedCompressedSize + ) throws IOException + { + SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer( + expected, + COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"), + expectedCompressedSize + ).asReadOnlyBuffer(); + + try (ComplexColumn compressedCol = createComplexColumn(compressedBuffer) + ) { + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals(expected.get(i), compressedCol.getRowValue(i)); + } + } + return compressedBuffer; + } + + private ComplexColumn createComplexColumn(ByteBuffer byteBuffer) + { + ColumnBuilder builder = new ColumnBuilder(); + int serializedSize = byteBuffer.remaining(); + + COMPRESSED_SERDE.deserializeColumn(byteBuffer, builder); + builder.setType(ValueType.COMPLEX); + + + ColumnHolder columnHolder = builder.build(); + + final ComplexColumn col = (ComplexColumn) columnHolder.getColumn(); + if (col instanceof SerializablePairLongFloatComplexColumn) { + Assert.assertEquals(serializedSize, col.getLength()); + } + Assert.assertEquals("serializablePairLongFloat", col.getTypeName()); + Assert.assertEquals(SerializablePairLongFloat.class, col.getClazz()); + + return col; + } + + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static ByteBuffer serializeAllValuesToByteBuffer( + List values, + GenericColumnSerializer serializer, + int expectedSize + ) throws IOException + { + serializer.open(); + + final AtomicReference reference = new AtomicReference<>(null); + ColumnValueSelector valueSelector = + new SingleObjectColumnValueSelector( + SerializablePairLongFloat.class + ) + { + @Nullable + @Override + public SerializablePairLongFloat getObject() + { + return reference.get(); + } + }; + + for (SerializablePairLongFloat selector : values) { + reference.set(selector); + serializer.serialize(valueSelector); + } + + return serializeToByteBuffer(serializer, expectedSize); + } + + private static ByteBuffer serializeToByteBuffer( + GenericColumnSerializer serializer, + int expectedSize + ) throws IOException + { + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + + serializer.writeTo(channel, null); + + ByteBuffer byteBuffer = ByteBuffer.allocate((int) channel.size()).order(ByteOrder.nativeOrder()); + + channel.readFully(0, byteBuffer); + byteBuffer.flip(); + + if (expectedSize > -1) { + Assert.assertEquals(expectedSize, serializer.getSerializedSize()); + } + + Assert.assertEquals(serializer.getSerializedSize(), byteBuffer.limit()); + + return byteBuffer; + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java new file mode 100644 index 000000000000..d76d4d7772f6 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Random; + +public class SerializablePairLongFloatDeltaEncodedStagedSerdeTest +{ + private static final SerializablePairLongFloatDeltaEncodedStagedSerde INTEGER_SERDE = + new SerializablePairLongFloatDeltaEncodedStagedSerde(0L, true); + + private static final SerializablePairLongFloatDeltaEncodedStagedSerde LONG_SERDE = + new SerializablePairLongFloatDeltaEncodedStagedSerde(0L, false); + + private final Random random = new Random(0); + + @Test + public void testNull() + { + assertValueEquals(null, 0, INTEGER_SERDE); + } + + @Test + public void testSimpleInteger() + { + assertValueEquals(new SerializablePairLongFloat(100L, 10F), 8, INTEGER_SERDE); + } + + @Test + public void testNullRHSInteger() + { + assertValueEquals(new SerializablePairLongFloat(100L, null), 4, INTEGER_SERDE); + } + + @Test + public void testLargeRHSInteger() + { + assertValueEquals( + new SerializablePairLongFloat(100L, random.nextFloat()), + 8, + INTEGER_SERDE + ); + } + + @Test + public void testSimpleLong() + { + assertValueEquals(new SerializablePairLongFloat(100L, 10F), 12, LONG_SERDE); + } + + @Test + public void testNullRHSLong() + { + assertValueEquals(new SerializablePairLongFloat(100L, null), 8, LONG_SERDE); + } + + @Test + public void testLargeRHSLong() + { + assertValueEquals( + new SerializablePairLongFloat(100L, random.nextFloat()), + 12, + LONG_SERDE + ); + } + + private static void assertValueEquals( + @Nullable SerializablePairLongFloat value, + int size, + SerializablePairLongFloatDeltaEncodedStagedSerde serde + ) + { + byte[] bytes = serde.serialize(value); + Assert.assertEquals(size, bytes.length); + SerializablePairLongFloat deserialized = serde.deserialize(bytes); + Assert.assertEquals(value, deserialized); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java new file mode 100644 index 000000000000..20c1437297b3 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Ints; +import org.apache.druid.collections.ResourceHolder; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.serde.cell.IOIterator; +import org.apache.druid.segment.serde.cell.NativeClearedByteBufferProvider; +import org.apache.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Random; + +public class SerializablePairLongLongBufferStoreTest +{ + private final Random random = new Random(0); + private static final int MIN_INTEGER = 100; + private static final long MIN_LONG = 0L; + private final SerializablePairLongLong[] integerRangeArr = new SerializablePairLongLong[]{ + new SerializablePairLongLong((long) MIN_INTEGER, 10L), + new SerializablePairLongLong(101L, 20L), + new SerializablePairLongLong(102L, 30L), + }; + + private final SerializablePairLongLong[] longRangeArr = new SerializablePairLongLong[]{ + new SerializablePairLongLong((long) MIN_LONG, 10L), + new SerializablePairLongLong(101L, 20L), + new SerializablePairLongLong(102L, 30L), + new SerializablePairLongLong((long) Integer.MAX_VALUE, 40L), + new SerializablePairLongLong(Long.MAX_VALUE, 50L), + }; + + private final SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + private SerializablePairLongLongBufferStore bufferStore; + + @Before + public void setup() throws Exception + { + bufferStore = new SerializablePairLongLongBufferStore( + new SerializedStorage<>( + writeOutMedium.makeWriteOutBytes(), + new SerializablePairLongLongSimpleStagedSerde() + ) + ); + } + + @Test + public void testIteratorSimple() throws Exception + { + for (SerializablePairLongLong value : integerRangeArr) { + bufferStore.store(value); + } + + IOIterator iterator = bufferStore.iterator(); + + int i = 0; + while (iterator.hasNext()) { + Assert.assertEquals(integerRangeArr[i], iterator.next()); + i++; + } + } + + @Test + public void testIteratorNull() throws Exception + { + bufferStore.store(null); + IOIterator iterator = bufferStore.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertNull(iterator.next()); + } + + @Test + public void testIteratorIdempotentHasNext() throws Exception + { + bufferStore.store(integerRangeArr[0]); + + IOIterator iterator = bufferStore.iterator(); + + Assert.assertTrue(iterator.hasNext()); + // expect hasNext() to not modify state + Assert.assertTrue(iterator.hasNext()); + } + + @Test(expected = NoSuchElementException.class) + public void testIteratorEmptyThrows() throws Exception + { + IOIterator iterator = bufferStore.iterator(); + iterator.next(); + } + + @Test + public void testIteratorEmptyHasNext() throws Exception + { + IOIterator iterator = bufferStore.iterator(); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testMinValueUsesInteger() throws Exception + { + for (SerializablePairLongLong value : integerRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongLongColumnHeader columnHeader = (SerializablePairLongLongColumnHeader) bufferStore.createColumnHeader(); + Assert.assertEquals(integerRangeArr[0].lhs.longValue(), columnHeader.getMinValue()); + Assert.assertTrue(columnHeader.isUseIntegerDeltas()); + } + + @Test + public void testMinValueUsesLong() throws Exception + { + for (SerializablePairLongLong value : longRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongLongColumnHeader columnHeader = (SerializablePairLongLongColumnHeader) bufferStore.createColumnHeader(); + Assert.assertEquals(MIN_LONG, columnHeader.getMinValue()); + Assert.assertFalse(columnHeader.isUseIntegerDeltas()); + } + + @Test + public void testMinValueUsesIntegerSerialization() throws Exception + { + for (SerializablePairLongLong value : integerRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongLongColumnHeader columnHeader = (SerializablePairLongLongColumnHeader) bufferStore.createColumnHeader(); + + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { + columnHeader.transferTo(channel); + + ByteBuffer byteBuffer = resourceHolder.get(); + channel.writeTo(byteBuffer); + byteBuffer.flip(); + + SerializablePairLongLongColumnHeader deserializedColumnhHeader = + (SerializablePairLongLongColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongLong.class); + Assert.assertEquals(MIN_INTEGER, deserializedColumnhHeader.getMinValue()); + Assert.assertTrue(deserializedColumnhHeader.isUseIntegerDeltas()); + } + } + + @Test + public void testMinValueSerialization() throws Exception + + { + for (SerializablePairLongLong value : longRangeArr) { + bufferStore.store(value); + } + + SerializablePairLongLongColumnHeader columnHeader = (SerializablePairLongLongColumnHeader) bufferStore.createColumnHeader(); + + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { + columnHeader.transferTo(channel); + + ByteBuffer byteBuffer = resourceHolder.get(); + + channel.writeTo(byteBuffer); + byteBuffer.flip(); + + SerializablePairLongLongColumnHeader deserializedColumnhHeader = + (SerializablePairLongLongColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongLong.class); + Assert.assertEquals(MIN_LONG, deserializedColumnhHeader.getMinValue()); + Assert.assertFalse(deserializedColumnhHeader.isUseIntegerDeltas()); + } + } + + @Test + public void testLargeBuffer() throws Exception + { + SerializablePairLongLong value = + new SerializablePairLongLong(Long.MAX_VALUE, Long.MAX_VALUE); + + bufferStore.store(value); + + IOIterator iterator = bufferStore.iterator(); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(value, iterator.next()); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testLargeValueCount() throws Exception + { + List valueList = new ArrayList<>(); + + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongLong(Integer.MAX_VALUE + (long) i, random.nextLong())); + } + + assertBufferedValuesEqual(valueList); + } + + @Test + public void testOverflowTransfer() throws Exception + { + bufferStore.store(new SerializablePairLongLong(Long.MIN_VALUE, 10L)); + bufferStore.store(new SerializablePairLongLong(Long.MAX_VALUE, 10L)); + + SerializablePairLongLongColumnHeader columnHeader = (SerializablePairLongLongColumnHeader) bufferStore.createColumnHeader(); + + Assert.assertEquals(0, columnHeader.getMinValue()); + + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + NativeClearedByteBufferProvider.INSTANCE, + writeOutMedium + ); + + Assert.assertEquals(92, transferredBuffer.getSerializedSize()); + } + + @Test + public void testNullOnlyTransfer() throws Exception + { + bufferStore.store(null); + + bufferStore.store(null); + + bufferStore.store(null); + + SerializablePairLongLongColumnHeader columnHeader = (SerializablePairLongLongColumnHeader) bufferStore.createColumnHeader(); + + Assert.assertEquals(0, columnHeader.getMinValue()); + + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + NativeClearedByteBufferProvider.INSTANCE, + writeOutMedium + ); + + Assert.assertEquals(59, transferredBuffer.getSerializedSize()); + } + + @Test + public void testTransferIntegerRange() throws Exception + { + for (SerializablePairLongLong value : integerRangeArr) { + bufferStore.store(value); + } + + Assert.assertTrue(bufferStore.createColumnHeader().isUseIntegerDeltas()); + + assertTransferredValuesEqual(integerRangeArr); + } + + @Test + public void testTransferLongRange() throws Exception + { + for (SerializablePairLongLong value : longRangeArr) { + bufferStore.store(value); + } + + Assert.assertFalse(bufferStore.createColumnHeader().isUseIntegerDeltas()); + + assertTransferredValuesEqual(longRangeArr); + } + + private void assertBufferedValuesEqual(List input) throws IOException + { + for (SerializablePairLongLong pairLongLong : input) { + bufferStore.store(pairLongLong); + } + + IOIterator iterator = bufferStore.iterator(); + int i = 0; + + while (iterator.hasNext()) { + Assert.assertEquals(input.get(i), iterator.next()); + i++; + } + + Assert.assertEquals( + StringUtils.format("element count mismatch: expected %s, got %s", input.size(), i), + input.size(), + i + ); + } + + private void assertTransferredValuesEqual(SerializablePairLongLong[] input) throws IOException + { + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = + bufferStore.transferToRowWriter(NativeClearedByteBufferProvider.INSTANCE, writeOutMedium); + HeapByteBufferWriteOutBytes resultChannel = new HeapByteBufferWriteOutBytes(); + + transferredBuffer.writeTo(resultChannel, null); + + try (SerializablePairLongLongComplexColumn column = createComplexColumn(transferredBuffer, resultChannel)) { + for (int i = 0; i < input.length; i++) { + Assert.assertEquals(input[i], column.getRowValue(i)); + } + } + } + + private static SerializablePairLongLongComplexColumn createComplexColumn( + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, + HeapByteBufferWriteOutBytes resultChannel + ) throws IOException + { + ByteBuffer byteBuffer = ByteBuffer.allocate(Ints.checkedCast(transferredBuffer.getSerializedSize())); + + resultChannel.readFully(0, byteBuffer); + byteBuffer.flip(); + + SerializablePairLongLongComplexMetricSerde complexMetricSerde = new SerializablePairLongLongComplexMetricSerde(); + ColumnBuilder builder = new ColumnBuilder(); + + complexMetricSerde.deserializeColumn(byteBuffer, builder); + builder.setType(ValueType.COMPLEX); + + ColumnHolder columnHolder = builder.build(); + SerializablePairLongLongComplexColumn column = (SerializablePairLongLongComplexColumn) columnHolder.getColumn(); + + return column; + } + +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java new file mode 100644 index 000000000000..3365cee5b4d5 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; + +public class SerializablePairLongLongComplexMetricSerdeTest +{ + static { + NullHandling.initializeForTests(); + } + + private static final SerializablePairLongLongComplexMetricSerde COMPRESSED_SERDE = + new SerializablePairLongLongComplexMetricSerde(); + + // want deterministic test input + private final Random random = new Random(0); + + @Test + public void testSingle() throws Exception + { + assertExpected(ImmutableList.of(new SerializablePairLongLong(100L, 10L)), 78); + } + + @Test + public void testLargeRHS() throws Exception + { + // single entry spans more than one block in underlying storage + assertExpected(ImmutableList.of(new SerializablePairLongLong( + 100L, + random.nextLong() + )), 78); + } + + @Test + public void testCompressable() throws Exception + { + int numLongs = 10; + List valueList = new ArrayList<>(); + List longList = new ArrayList<>(); + + for (int i = 0; i < numLongs; i++) { + longList.add(random.nextLong()); + } + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongLong(Integer.MAX_VALUE + (long) i, longList.get(i % numLongs))); + } + + assertExpected(valueList, 80258); + } + + @Test + public void testHighlyCompressable() throws Exception + { + List valueList = new ArrayList<>(); + + Long longValue = random.nextLong(); + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongLong(Integer.MAX_VALUE + (long) i, longValue)); + } + + assertExpected(valueList, 80023); + } + + @Test + public void testRandom() throws Exception + { + List valueList = new ArrayList<>(); + + for (int i = 0; i < 10000; i++) { + valueList.add(new SerializablePairLongLong(random.nextLong(), random.nextLong())); + } + + assertExpected(valueList, 200618); + } + + @Test + public void testNullRHS() throws Exception + { + assertExpected(ImmutableList.of(new SerializablePairLongLong(100L, null)), 70); + } + + @Test + public void testEmpty() throws Exception + { + // minimum size for empty data + assertExpected(Collections.emptyList(), 57); + } + + @Test + public void testSingleNull() throws Exception + { + assertExpected(Arrays.asList(new SerializablePairLongLong[]{null}), 58); + } + + @Test + public void testMultipleNull() throws Exception + { + assertExpected(Arrays.asList(null, null, null, null), 59); + } + + private ByteBuffer assertExpected( + List expected, + int expectedCompressedSize + ) throws IOException + { + SegmentWriteOutMedium writeOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + ByteBuffer compressedBuffer = serializeAllValuesToByteBuffer( + expected, + COMPRESSED_SERDE.getSerializer(writeOutMedium, "not-used"), + expectedCompressedSize + ).asReadOnlyBuffer(); + + try (ComplexColumn compressedCol = createComplexColumn(compressedBuffer) + ) { + for (int i = 0; i < expected.size(); i++) { + Assert.assertEquals(expected.get(i), compressedCol.getRowValue(i)); + } + } + return compressedBuffer; + } + + private ComplexColumn createComplexColumn(ByteBuffer byteBuffer) + { + ColumnBuilder builder = new ColumnBuilder(); + int serializedSize = byteBuffer.remaining(); + + COMPRESSED_SERDE.deserializeColumn(byteBuffer, builder); + builder.setType(ValueType.COMPLEX); + + + ColumnHolder columnHolder = builder.build(); + + final ComplexColumn col = (ComplexColumn) columnHolder.getColumn(); + if (col instanceof SerializablePairLongLongComplexColumn) { + Assert.assertEquals(serializedSize, col.getLength()); + } + Assert.assertEquals("serializablePairLongLong", col.getTypeName()); + Assert.assertEquals(SerializablePairLongLong.class, col.getClazz()); + + return col; + } + + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static ByteBuffer serializeAllValuesToByteBuffer( + List values, + GenericColumnSerializer serializer, + int expectedSize + ) throws IOException + { + serializer.open(); + + final AtomicReference reference = new AtomicReference<>(null); + ColumnValueSelector valueSelector = + new SingleObjectColumnValueSelector( + SerializablePairLongLong.class + ) + { + @Nullable + @Override + public SerializablePairLongLong getObject() + { + return reference.get(); + } + }; + + for (SerializablePairLongLong selector : values) { + reference.set(selector); + serializer.serialize(valueSelector); + } + + return serializeToByteBuffer(serializer, expectedSize); + } + + private static ByteBuffer serializeToByteBuffer( + GenericColumnSerializer serializer, + int expectedSize + ) throws IOException + { + HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); + + serializer.writeTo(channel, null); + + ByteBuffer byteBuffer = ByteBuffer.allocate((int) channel.size()).order(ByteOrder.nativeOrder()); + + channel.readFully(0, byteBuffer); + byteBuffer.flip(); + + if (expectedSize > -1) { + Assert.assertEquals(expectedSize, serializer.getSerializedSize()); + } + + Assert.assertEquals(serializer.getSerializedSize(), byteBuffer.limit()); + + return byteBuffer; + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java new file mode 100644 index 000000000000..6edfb022d811 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Random; + +public class SerializablePairLongLongDeltaEncodedStagedSerdeTest +{ + private static final SerializablePairLongLongDeltaEncodedStagedSerde INTEGER_SERDE = + new SerializablePairLongLongDeltaEncodedStagedSerde(0L, true); + + private static final SerializablePairLongLongDeltaEncodedStagedSerde LONG_SERDE = + new SerializablePairLongLongDeltaEncodedStagedSerde(0L, false); + + private final Random random = new Random(0); + + @Test + public void testNull() + { + assertValueEquals(null, 0, INTEGER_SERDE); + } + + @Test + public void testSimpleInteger() + { + assertValueEquals(new SerializablePairLongLong(100L, 10L), 12, INTEGER_SERDE); + } + + @Test + public void testNullRHSInteger() + { + assertValueEquals(new SerializablePairLongLong(100L, null), 4, INTEGER_SERDE); + } + + @Test + public void testLargeRHSInteger() + { + assertValueEquals( + new SerializablePairLongLong(100L, random.nextLong()), + 12, + INTEGER_SERDE + ); + } + + @Test + public void testSimpleLong() + { + assertValueEquals(new SerializablePairLongLong(100L, 10L), 16, LONG_SERDE); + } + + @Test + public void testNullRHSLong() + { + assertValueEquals(new SerializablePairLongLong(100L, null), 8, LONG_SERDE); + } + + @Test + public void testLargeRHSLong() + { + assertValueEquals( + new SerializablePairLongLong(100L, random.nextLong()), + 16, + LONG_SERDE + ); + } + + private static void assertValueEquals( + @Nullable SerializablePairLongLong value, + int size, + SerializablePairLongLongDeltaEncodedStagedSerde serde + ) + { + byte[] bytes = serde.serialize(value); + Assert.assertEquals(size, bytes.length); + SerializablePairLongLong deserialized = serde.deserialize(bytes); + Assert.assertEquals(value, deserialized); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java new file mode 100644 index 000000000000..e903087105f0 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Random; + +public class SerializablePairLongLongSimpleStagedSerdeTest +{ + private static final SerializablePairLongLongSimpleStagedSerde SERDE = + new SerializablePairLongLongSimpleStagedSerde(); + + private final Random random = new Random(0); + + @Test + public void testSimple() + { + assertValueEquals(new SerializablePairLongLong(Long.MAX_VALUE, 10L), 16); + } + + @Test + public void testNull() + { + assertValueEquals(null, 0); + } + + @Test + public void testNullString() + { + assertValueEquals(new SerializablePairLongLong(Long.MAX_VALUE, null), 8); + } + + @Test + public void testLargeRHS() + { + assertValueEquals( + new SerializablePairLongLong(Long.MAX_VALUE, random.nextLong()), + 16 + ); + } + + private static void assertValueEquals(@Nullable SerializablePairLongLong value, int size) + { + byte[] bytes = SERDE.serialize(value); + Assert.assertEquals(size, bytes.length); + SerializablePairLongLong deserialized = SERDE.deserialize(bytes); + Assert.assertEquals(value, deserialized); + } +} From ea8fea51d2584759645070ab19cb43a27843444f Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Mon, 2 Oct 2023 16:13:39 -0700 Subject: [PATCH 07/14] post merge fix --- ...AbstractSerializablePairLongObjectColumnHeader.java | 10 +++++----- .../first/DoubleFirstAggregatorFactory.java | 1 + .../aggregation/first/FloatFirstAggregatorFactory.java | 1 + .../aggregation/first/LongFirstAggregatorFactory.java | 1 + .../aggregation/last/DoubleLastAggregatorFactory.java | 1 - .../aggregation/last/FloatLastAggregatorFactory.java | 1 - 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java index 38f0529c2cfc..16c4dfe375b5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.RE; @@ -113,9 +113,9 @@ public int getSerializedSize() @Override public String toString() { - return Objects.toStringHelper(this) - .add("bytes", bytes) - .add("minValue", minValue) - .toString(); + return MoreObjects.toStringHelper(this) + .add("bytes", bytes) + .add("minValue", minValue) + .toString(); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 9fb0cc5151c6..8c9f13a6ced0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -34,6 +34,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 705407048fde..56556ab13745 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -34,6 +34,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 11ea7fa7c454..9be911d35a25 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -34,6 +34,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 475e56bf7aa3..63c1385f2f5f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -35,7 +35,6 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index b563c4b445e4..7977b4ff601e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -35,7 +35,6 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; From 97d1dace85ed7a3b1fb0208396dbc1d510983ba5 Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Mon, 2 Oct 2023 21:04:58 -0700 Subject: [PATCH 08/14] failure (firstVectorAggregator) fix --- .../AbstractSerializablePairLongObjectBufferStore.java | 2 +- .../query/aggregation/first/DoubleFirstVectorAggregator.java | 4 ++-- .../query/aggregation/first/FloatFirstVectorAggregator.java | 4 ++-- .../query/aggregation/first/LongFirstVectorAggregator.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java index 2c77a4b0919a..e5808b444044 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java @@ -101,7 +101,7 @@ public TransferredBuffer( } @Override - public long getSerializedSize() throws IOException + public long getSerializedSize() { return columnHeader.getSerializedSize() + cellWriter.getSerializedSize(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java index 562f14547a6a..5221bc74b3e4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -56,6 +56,6 @@ void putValue(ByteBuffer buf, int position, int index) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java index 7c8c4e76f6b4..a1671d9c7a24 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -56,6 +56,6 @@ void putValue(ByteBuffer buf, int position, int index) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java index 769b148b5e49..0a40e5ad870c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -55,6 +55,6 @@ void putValue(ByteBuffer buf, int position, int index) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); } } From f5a1c50c7fd832f7dbff8f0cf9196dc9d3881ad3 Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Wed, 11 Oct 2023 22:44:13 -0700 Subject: [PATCH 09/14] Change *VectorAggregator to support pair serdes --- ...PairLongObjectDeltaEncodedStagedSerde.java | 6 +- ...izablePairLongObjectSimpleStagedSerde.java | 6 +- ...zablePairLongDoubleComplexMetricSerde.java | 30 +-- ...PairLongDoubleDeltaEncodedStagedSerde.java | 4 +- ...izablePairLongDoubleSimpleStagedSerde.java | 4 +- ...izablePairLongFloatComplexMetricSerde.java | 30 +-- ...ePairLongFloatDeltaEncodedStagedSerde.java | 4 +- ...lizablePairLongFloatSimpleStagedSerde.java | 4 +- ...lizablePairLongLongComplexMetricSerde.java | 30 +-- ...lePairLongLongDeltaEncodedStagedSerde.java | 4 +- ...alizablePairLongLongSimpleStagedSerde.java | 4 +- .../aggregation/any/NilVectorAggregator.java | 10 +- .../first/DoubleFirstAggregatorFactory.java | 24 +- .../first/DoubleFirstVectorAggregator.java | 9 +- .../aggregation/first/FirstLastUtils.java | 83 +++++++ .../first/FloatFirstAggregatorFactory.java | 23 +- .../first/FloatFirstVectorAggregator.java | 10 +- .../first/LongFirstAggregatorFactory.java | 25 +- .../first/LongFirstVectorAggregator.java | 11 +- .../first/NumericFirstVectorAggregator.java | 98 ++++++-- .../first/StringFirstAggregatorFactory.java | 4 +- .../first/StringFirstLastUtils.java | 41 ---- .../first/StringFirstVectorAggregator.java | 4 +- .../last/DoubleLastAggregatorFactory.java | 28 ++- .../last/DoubleLastVectorAggregator.java | 9 +- .../last/FloatLastAggregatorFactory.java | 28 ++- .../last/FloatLastVectorAggregator.java | 11 +- .../last/LongLastAggregatorFactory.java | 29 ++- .../last/LongLastVectorAggregator.java | 10 +- .../last/NumericLastVectorAggregator.java | 98 ++++++-- .../last/StringLastAggregatorFactory.java | 5 +- .../last/StringLastVectorAggregator.java | 5 +- ...alizablePairLongDoubleBufferStoreTest.java | 2 +- ...ePairLongDoubleComplexMetricSerdeTest.java | 10 +- ...LongDoubleDeltaEncodedStagedSerdeTest.java | 12 +- ...ializablePairLongFloatBufferStoreTest.java | 2 +- ...lePairLongFloatComplexMetricSerdeTest.java | 12 +- ...rLongFloatDeltaEncodedStagedSerdeTest.java | 12 +- ...rializablePairLongLongBufferStoreTest.java | 2 +- ...blePairLongLongComplexMetricSerdeTest.java | 8 +- ...irLongLongDeltaEncodedStagedSerdeTest.java | 12 +- ...ablePairLongLongSimpleStagedSerdeTest.java | 6 +- .../DoubleFirstVectorAggregationTest.java | 134 ++++++---- .../FloatFirstVectorAggregationTest.java | 113 ++++++--- .../first/LongFirstVectorAggregationTest.java | 120 ++++++--- .../last/DoubleLastVectorAggregatorTest.java | 224 ++++++++++++++--- .../last/FloatLastVectorAggregatorTest.java | 230 ++++++++++++++--- .../last/LongLastVectorAggregatorTest.java | 232 +++++++++++++++--- 48 files changed, 1352 insertions(+), 470 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java index 2abbc88e104a..62b0a52642e7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; import org.apache.druid.collections.SerializablePair; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.segment.serde.cell.StagedSerde; import org.apache.druid.segment.serde.cell.StorableBuffer; @@ -75,6 +76,7 @@ public void store(ByteBuffer byteBuffer) } if (rhsObject != null) { + byteBuffer.put(NullHandling.IS_NOT_NULL_BYTE); if (pairClass.isAssignableFrom(SerializablePairLongLong.class)) { byteBuffer.putLong((long) rhsObject); } else if (pairClass.isAssignableFrom(SerializablePairLongDouble.class)) { @@ -82,6 +84,8 @@ public void store(ByteBuffer byteBuffer) } else if (pairClass.isAssignableFrom(SerializablePairLongFloat.class)) { byteBuffer.putFloat((float) rhsObject); } + } else { + byteBuffer.put(NullHandling.IS_NULL_BYTE); } } @@ -100,7 +104,7 @@ public int getSerializedSize() } } - return (useIntegerDelta ? Integer.BYTES : Long.BYTES) + rhsBytes; + return (useIntegerDelta ? Integer.BYTES : Long.BYTES) + Byte.BYTES + rhsBytes; } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java index 95a1b8cc3097..cf43cbc34c5c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.segment.serde.cell.StagedSerde; import org.apache.druid.segment.serde.cell.StorableBuffer; @@ -57,6 +58,7 @@ public void store(ByteBuffer byteBuffer) Preconditions.checkNotNull(value.getLhs(), String.format(Locale.ENGLISH, "Long in %s must be non-null", pairCLass.getSimpleName())); byteBuffer.putLong(value.getLhs()); if (rhsObject != null) { + byteBuffer.put(NullHandling.IS_NOT_NULL_BYTE); if (pairCLass.isAssignableFrom(SerializablePairLongLong.class)) { byteBuffer.putLong((long) rhsObject); } else if (pairCLass.isAssignableFrom(SerializablePairLongDouble.class)) { @@ -64,6 +66,8 @@ public void store(ByteBuffer byteBuffer) } else if (pairCLass.isAssignableFrom(SerializablePairLongFloat.class)) { byteBuffer.putFloat((float) rhsObject); } + } else { + byteBuffer.put(NullHandling.IS_NULL_BYTE); } } @@ -81,7 +85,7 @@ public int getSerializedSize() rhsBytes = Float.BYTES; } } - return Long.BYTES + rhsBytes; + return Long.BYTES + Byte.BYTES + rhsBytes; } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java index 9506524f9d49..2fd854c1da97 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java @@ -20,7 +20,6 @@ package org.apache.druid.query.aggregation; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.segment.GenericColumnSerializer; import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.ObjectStrategy; @@ -35,6 +34,8 @@ public class SerializablePairLongDoubleComplexMetricSerde extends AbstractSerial { public static final String TYPE_NAME = "serializablePairLongDouble"; + private static final SerializablePairLongDoubleSimpleStagedSerde SERDE = new SerializablePairLongDoubleSimpleStagedSerde(); + private static final Comparator> COMPARATOR = SerializablePair.createNullHandlingComparator( Double::compare, true @@ -90,32 +91,17 @@ public Class getClazz() @Override public SerializablePairLongDouble fromByteBuffer(ByteBuffer buffer, int numBytes) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); - } else { - return new SerializablePairLongDouble(lhs, null); - } + ByteBuffer readOnlyByteBuffer = buffer.asReadOnlyBuffer().order(buffer.order()); + + readOnlyByteBuffer.limit(buffer.position() + numBytes); + + return SERDE.deserialize(readOnlyByteBuffer); } @Override public byte[] toBytes(@Nullable SerializablePairLongDouble inPair) { - if (inPair == null) { - return new byte[]{}; - } - - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); - bbuf.putLong(inPair.lhs); - if (inPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putDouble(inPair.rhs); - } - return bbuf.array(); + return SERDE.serialize(inPair); } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java index 5278ef691175..ce087ec623e0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -51,7 +53,7 @@ public SerializablePairLongDouble deserialize(ByteBuffer byteBuffer) lhs += minValue; Double rhs = null; - if (readOnlyBuffer.hasRemaining()) { + if (readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE) { rhs = readOnlyBuffer.getDouble(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java index 5f62a7ed3a65..bf5c60e0c5b5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSimpleStagedSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -42,7 +44,7 @@ public SerializablePairLongDouble deserialize(ByteBuffer byteBuffer) long lhs = readOnlyBuffer.getLong(); Double rhs = null; - if (readOnlyBuffer.hasRemaining()) { + if (readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE) { rhs = readOnlyBuffer.getDouble(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java index 137e8e58268f..2ffcb635b82e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java @@ -20,7 +20,6 @@ package org.apache.druid.query.aggregation; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.segment.GenericColumnSerializer; import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.ObjectStrategy; @@ -35,6 +34,8 @@ public class SerializablePairLongFloatComplexMetricSerde extends AbstractSeriali { public static final String TYPE_NAME = "serializablePairLongFloat"; + private static final SerializablePairLongFloatSimpleStagedSerde SERDE = new SerializablePairLongFloatSimpleStagedSerde(); + private static final Comparator> COMPARATOR = SerializablePair.createNullHandlingComparator( Float::compare, true @@ -91,32 +92,17 @@ public Class getClazz() @Override public SerializablePairLongFloat fromByteBuffer(ByteBuffer buffer, int numBytes) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); - } else { - return new SerializablePairLongFloat(lhs, null); - } + ByteBuffer readOnlyByteBuffer = buffer.asReadOnlyBuffer().order(buffer.order()); + + readOnlyByteBuffer.limit(buffer.position() + numBytes); + + return SERDE.deserialize(readOnlyByteBuffer); } @Override public byte[] toBytes(@Nullable SerializablePairLongFloat inPair) { - if (inPair == null) { - return new byte[]{}; - } - - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); - bbuf.putLong(inPair.lhs); - if (inPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putFloat(inPair.rhs); - } - return bbuf.array(); + return SERDE.serialize(inPair); } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java index 059fa4bd0b6d..0f5e4f7f4a93 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -51,7 +53,7 @@ public SerializablePairLongFloat deserialize(ByteBuffer byteBuffer) lhs += minValue; Float rhs = null; - if (readOnlyBuffer.hasRemaining()) { + if (readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE) { rhs = readOnlyBuffer.getFloat(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java index d8d6f9b5ef47..d8390974a95d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSimpleStagedSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -42,7 +44,7 @@ public SerializablePairLongFloat deserialize(ByteBuffer byteBuffer) long lhs = readOnlyBuffer.getLong(); Float rhs = null; - if (readOnlyBuffer.hasRemaining()) { + if (readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE) { rhs = readOnlyBuffer.getFloat(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java index fececb873101..a5f79f007279 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java @@ -20,7 +20,6 @@ package org.apache.druid.query.aggregation; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.segment.GenericColumnSerializer; import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.ObjectStrategy; @@ -35,6 +34,8 @@ public class SerializablePairLongLongComplexMetricSerde extends AbstractSerializ { public static final String TYPE_NAME = "serializablePairLongLong"; + private static final SerializablePairLongLongSimpleStagedSerde SERDE = new SerializablePairLongLongSimpleStagedSerde(); + private static final Comparator> COMPARATOR = SerializablePair.createNullHandlingComparator( Long::compare, true @@ -90,32 +91,17 @@ public Class getClazz() @Override public SerializablePairLongLong fromByteBuffer(ByteBuffer buffer, int numBytes) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); - } else { - return new SerializablePairLongLong(lhs, null); - } + ByteBuffer readOnlyByteBuffer = buffer.asReadOnlyBuffer().order(buffer.order()); + + readOnlyByteBuffer.limit(buffer.position() + numBytes); + + return SERDE.deserialize(readOnlyByteBuffer); } @Override public byte[] toBytes(@Nullable SerializablePairLongLong inPair) { - if (inPair == null) { - return new byte[]{}; - } - - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); - bbuf.putLong(inPair.lhs); - if (inPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putLong(inPair.rhs); - } - return bbuf.array(); + return SERDE.serialize(inPair); } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java index bf0414031a88..dad3711c3c73 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -51,7 +53,7 @@ public SerializablePairLongLong deserialize(ByteBuffer byteBuffer) lhs += minValue; Long rhs = null; - if (readOnlyBuffer.hasRemaining()) { + if (readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE) { rhs = readOnlyBuffer.getLong(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java index c94e842b5289..587ce18a0b4c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -42,7 +44,7 @@ public SerializablePairLongLong deserialize(ByteBuffer byteBuffer) long lhs = readOnlyBuffer.getLong(); Long rhs = null; - if (readOnlyBuffer.hasRemaining()) { + if (readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE) { rhs = readOnlyBuffer.getLong(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java index ac6c5c7a75e4..80a1e0f7e12a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java @@ -19,8 +19,10 @@ package org.apache.druid.query.aggregation.any; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; import javax.annotation.Nullable; @@ -43,9 +45,9 @@ public class NilVectorAggregator implements VectorAggregator NullHandling.defaultLongValue() ); - public static final SerializablePair DOUBLE_NIL_PAIR = new SerializablePair<>(0L, NullHandling.defaultDoubleValue()); - public static final SerializablePair LONG_NIL_PAIR = new SerializablePair<>(0L, NullHandling.defaultLongValue()); - public static final SerializablePair FLOAT_NIL_PAIR = new SerializablePair<>(0L, NullHandling.defaultFloatValue()); + public static final SerializablePairLongDouble DOUBLE_NIL_PAIR = new SerializablePairLongDouble(0L, NullHandling.defaultDoubleValue()); + public static final SerializablePairLongLong LONG_NIL_PAIR = new SerializablePairLongLong(0L, NullHandling.defaultLongValue()); + public static final SerializablePairLongFloat FLOAT_NIL_PAIR = new SerializablePairLongFloat(0L, NullHandling.defaultFloatValue()); /** * @return A vectorized aggregator that returns the default double value. diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 8c9f13a6ced0..20251410f38f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -43,7 +43,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -125,7 +127,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new DoubleFirstAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongDouble.class @@ -144,7 +146,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) return new DoubleFirstBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongDouble.class @@ -158,12 +160,24 @@ public VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory ) { + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector( - timeColumn); - return new DoubleFirstVectorAggregator(timeSelector, valueSelector); + VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( + columnSelectorFactory.getReadableVectorInspector(), + fieldName, + valueSelector, + capabilities.toColumnType(), + ColumnType.DOUBLE + ); + return new DoubleFirstVectorAggregator(timeSelector, objectSelector); + } + + VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new DoubleFirstVectorAggregator(timeSelector, vSelector); } return NilVectorAggregator.of(NilVectorAggregator.DOUBLE_NIL_PAIR); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java index 5221bc74b3e4..d58882e8439d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -28,9 +29,9 @@ public class DoubleFirstVectorAggregator extends NumericFirstVectorAggregator { - public DoubleFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + public DoubleFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, SerializablePairLongDouble.class); } @Override @@ -41,9 +42,9 @@ public void initValue(ByteBuffer buf, int position) @Override - void putValue(ByteBuffer buf, int position, int index) + void putValue(ByteBuffer buf, int position, Number number) { - double firstValue = valueSelector.getDoubleVector()[index]; + double firstValue = number.doubleValue(); buf.putDouble(position, firstValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java new file mode 100644 index 000000000000..7120d3a4605f --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import org.apache.druid.segment.BaseObjectColumnValueSelector; +import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ValueType; + +import javax.annotation.Nullable; + +public class FirstLastUtils +{ + + /** + * Returns whether a given value selector *might* contain SerializablePairLongString objects. + */ + public static boolean selectorNeedsFoldCheck( + final BaseObjectColumnValueSelector valueSelector, + @Nullable final ColumnCapabilities valueSelectorCapabilities, + Class pairClass + ) + { + if (valueSelectorCapabilities != null && !valueSelectorCapabilities.is(ValueType.COMPLEX)) { + // Known, non-complex type. + return false; + } + + if (valueSelector instanceof NilColumnValueSelector) { + // Nil column, definitely no SerializablePairLongStrings. + return false; + } + + // Check if the selector class could possibly be a SerializablePairLongString (either a superclass or subclass). + final Class clazz = valueSelector.classOfObject(); + return clazz.isAssignableFrom(pairClass) + || pairClass.isAssignableFrom(clazz); + } + + /** + * Returns whether an object *might* contain SerializablePairLongString objects. + */ + public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) + { + if (obj == null) { + return false; + } + final Class clazz = obj.getClass(); + return clazz.isAssignableFrom(pairClass) + || pairClass.isAssignableFrom(clazz); + } + + + public static boolean[] getNullVector(Object[] objectVector) + { + boolean containsNonNullValues = false; + boolean[] nullValueVector = new boolean[objectVector.length]; + for (int i = 0; i < objectVector.length; i++) { + if (objectVector[i] != null) { + containsNonNullValues = true; + nullValueVector[i] = false; + } + } + return containsNonNullValues ? null : nullValueVector; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 56556ab13745..51eb0345d77c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -43,7 +43,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -116,7 +118,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new FloatFirstAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongFloat.class @@ -135,7 +137,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) return new FloatFirstBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongFloat.class @@ -147,12 +149,25 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) @Override public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); if (Types.isNumeric(capabilities)) { VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - return new FloatFirstVectorAggregator(timeSelector, valueSelector); + VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( + columnSelectorFactory.getReadableVectorInspector(), + fieldName, + valueSelector, + capabilities.toColumnType(), + ColumnType.FLOAT + ); + return new FloatFirstVectorAggregator(timeSelector, objectSelector); } + + VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new FloatFirstVectorAggregator(timeSelector, vSelector); + } + return NilVectorAggregator.of(NilVectorAggregator.FLOAT_NIL_PAIR); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java index a1671d9c7a24..44ddf24c0319 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -28,9 +29,9 @@ public class FloatFirstVectorAggregator extends NumericFirstVectorAggregator { - public FloatFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + public FloatFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, SerializablePairLongFloat.class); } @Override @@ -39,11 +40,10 @@ public void initValue(ByteBuffer buf, int position) buf.putFloat(position, 0); } - @Override - void putValue(ByteBuffer buf, int position, int index) + void putValue(ByteBuffer buf, int position, Number number) { - float firstValue = valueSelector.getFloatVector()[index]; + float firstValue = number.floatValue(); buf.putFloat(position, firstValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 9be911d35a25..39afef69c300 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -43,7 +44,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -56,6 +59,7 @@ public class LongFirstAggregatorFactory extends AggregatorFactory { public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongLongComplexMetricSerde.TYPE_NAME); + private static final Logger log = new Logger(LongFirstAggregatorFactory.class); private static final Aggregator NIL_AGGREGATOR = new LongFirstAggregator( NilColumnValueSelector.instance(), @@ -115,7 +119,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new LongFirstAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongLong.class @@ -134,7 +138,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) return new LongFirstBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongLong.class @@ -146,12 +150,23 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) @Override public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); if (Types.isNumeric(capabilities)) { VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector( - timeColumn); - return new LongFirstVectorAggregator(timeSelector, valueSelector); + VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( + columnSelectorFactory.getReadableVectorInspector(), + fieldName, + valueSelector, + capabilities.toColumnType(), + ColumnType.LONG + ); + return new LongFirstVectorAggregator(timeSelector, objectSelector); + } + + VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new LongFirstVectorAggregator(timeSelector, vSelector); } return NilVectorAggregator.of(NilVectorAggregator.LONG_NIL_PAIR); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java index 0a40e5ad870c..961f124c5df3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.query.aggregation.SerializablePairLongLong; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -27,9 +28,9 @@ public class LongFirstVectorAggregator extends NumericFirstVectorAggregator { - public LongFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + public LongFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, SerializablePairLongLong.class); } @Override @@ -38,15 +39,13 @@ public void initValue(ByteBuffer buf, int position) buf.putLong(position, 0); } - @Override - void putValue(ByteBuffer buf, int position, int index) + void putValue(ByteBuffer buf, int position, Number number) { - long firstValue = valueSelector.getLongVector()[index]; + long firstValue = number.longValue(); buf.putLong(position, firstValue); } - /** * @return The object as a pair with the position and the value stored at the position in the buffer. */ diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java index 7fcd10352da9..46d311ed45fa 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java @@ -19,8 +19,10 @@ package org.apache.druid.query.aggregation.first; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -33,15 +35,17 @@ public abstract class NumericFirstVectorAggregator implements VectorAggregator { static final int NULL_OFFSET = Long.BYTES; static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; - final VectorValueSelector valueSelector; + final VectorObjectSelector valueSelector; + private final Class pairClass; private final boolean useDefault = NullHandling.replaceWithDefault(); private final VectorValueSelector timeSelector; private long firstTime; - public NumericFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + NumericFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector, Class pairClass) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.pairClass = pairClass; firstTime = Long.MAX_VALUE; } @@ -58,7 +62,8 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) { final long[] timeVector = timeSelector.getLongVector(); final boolean[] nullTimeVector = timeSelector.getNullVector(); - final boolean[] nullValueVector = valueSelector.getNullVector(); + final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); + final boolean[] nullValueVector = FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); firstTime = buf.getLong(position); // the time vector is already sorted @@ -68,22 +73,42 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) // A possible optimization here is to have 2 paths one for earliest where // we can take advantage of the sorted nature of time // and the earliest_by where we have to go over all elements. - int index; + int index; for (int i = startRow; i < endRow; i++) { - index = i; - if (nullTimeVector != null && nullTimeVector[index]) { + if (nullTimeVector != null && nullTimeVector[i]) { continue; } - final long earliestTime = timeVector[index]; - if (earliestTime >= firstTime) { + + if (timeVector[i] >= firstTime) { continue; } - firstTime = earliestTime; - if (useDefault || nullValueVector == null || !nullValueVector[index]) { - updateTimeWithValue(buf, position, firstTime, index); + index = i; + + final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeNumeric[index], pairClass); + if (foldNeeded) { + final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; + + if (inPair.lhs < firstTime) { + firstTime = inPair.lhs; + if (useDefault || inPair.rhs != null) { + updateTimeWithValue(buf, position, firstTime, inPair.getRhs()); + } else { + updateTimeWithNull(buf, position, firstTime); + } + } } else { - updateTimeWithNull(buf, position, firstTime); + final long earliestTime = timeVector[index]; + + if (earliestTime < firstTime) { + firstTime = earliestTime; + + if (useDefault || nullValueVector == null || !nullValueVector[index]) { + updateTimeWithValue(buf, position, earliestTime, (Number) objectsWhichMightBeNumeric[index]); + } else { + updateTimeWithNull(buf, position, earliestTime); + } + } } } } @@ -110,20 +135,45 @@ public void aggregate( int positionOffset ) { - boolean[] nulls = useDefault ? null : valueSelector.getNullVector(); - long[] timeVector = timeSelector.getLongVector(); + final long[] timeVector = timeSelector.getLongVector(); + final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); + final boolean[] nullValueVector = FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); + boolean[] nulls = useDefault ? null : nullValueVector; + + // iterate once over the object vector to find first non null element and + // determine if the type is Pair or not + boolean foldNeeded = false; + for (Object obj : objectsWhichMightBeNumeric) { + if (obj != null) { + foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, pairClass); + break; + } + } for (int i = 0; i < numRows; i++) { int position = positions[i] + positionOffset; int row = rows == null ? i : rows[i]; - long firstTime = buf.getLong(position); - if (timeVector[row] < firstTime) { - if (useDefault || nulls == null || !nulls[row]) { - updateTimeWithValue(buf, position, timeVector[row], row); + firstTime = buf.getLong(position); + + if (foldNeeded) { + final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; + if (useDefault || inPair != null) { + if (inPair.lhs < firstTime) { + updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + } } else { - updateTimeWithNull(buf, position, timeVector[row]); + updateTimeWithNull(buf, position, inPair.lhs); + } + } else { + if (timeVector[row] < firstTime) { + if (useDefault || nulls == null || objectsWhichMightBeNumeric[row] != null) { + updateTimeWithValue(buf, position, timeVector[row], (Number) objectsWhichMightBeNumeric[row]); + } else { + updateTimeWithNull(buf, position, timeVector[row]); + } } } + } } @@ -132,14 +182,14 @@ public void aggregate( * * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the current aggregate value is stored - * @param time the time to be updated in the buffer as the last time - * @param index the index of the vectorized vector which is the last value + * @param time the time to be updated in the buffer as the first time + * @param number number which is the first value */ - void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) + void updateTimeWithValue(ByteBuffer buf, int position, long time, Number number) { buf.putLong(position, time); buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, index); + putValue(buf, position + VALUE_OFFSET, number); } /** @@ -164,7 +214,7 @@ void updateTimeWithNull(ByteBuffer buf, int position, long time) * Abstract function which needs to be overridden by subclasses to set the * latest value in the buffer depending on the datatype */ - abstract void putValue(ByteBuffer buf, int position, int index); + abstract void putValue(ByteBuffer buf, int position, Number number); @Override public void close() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index c1eaf2abd1e1..8dbe12d4ddc5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -165,7 +165,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) + FirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } @@ -181,7 +181,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) + FirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java index 4e5809f3cc8f..ff1113c91132 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java @@ -24,9 +24,6 @@ import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.DimensionHandlerUtils; -import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; -import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -37,44 +34,6 @@ public class StringFirstLastUtils { private static final int NULL_VALUE = -1; - /** - * Returns whether a given value selector *might* contain SerializablePairLongString objects. - */ - public static boolean selectorNeedsFoldCheck( - final BaseObjectColumnValueSelector valueSelector, - @Nullable final ColumnCapabilities valueSelectorCapabilities, - Class pairClass - ) - { - if (valueSelectorCapabilities != null && !valueSelectorCapabilities.is(ValueType.COMPLEX)) { - // Known, non-complex type. - return false; - } - - if (valueSelector instanceof NilColumnValueSelector) { - // Nil column, definitely no SerializablePairLongStrings. - return false; - } - - // Check if the selector class could possibly be a SerializablePairLongString (either a superclass or subclass). - final Class clazz = valueSelector.classOfObject(); - return clazz.isAssignableFrom(pairClass) - || pairClass.isAssignableFrom(clazz); - } - - /** - * Returns whether an object *might* contain SerializablePairLongString objects. - */ - public static boolean objectNeedsFoldCheck(Object obj) - { - if (obj == null) { - return false; - } - final Class clazz = obj.getClass(); - return clazz.isAssignableFrom(SerializablePairLongString.class) - || SerializablePairLongString.class.isAssignableFrom(clazz); - } - /** * Return the object at a particular index from the vector selectors. * index of bounds issues is the responsibility of the caller diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java index 3e31300bad8b..fd2260b8d665 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java @@ -76,7 +76,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) continue; } index = i; - final boolean foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index]); + final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index], SerializablePairLongString.class); if (foldNeeded) { final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( timeSelector, @@ -125,7 +125,7 @@ public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable in if (obj == null) { continue; } else { - foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(obj); + foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, SerializablePairLongString.class); break; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 63c1385f2f5f..bac1109e570c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -35,7 +34,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstLastUtils; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -46,7 +45,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -116,7 +117,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new DoubleLastAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongDouble.class @@ -136,13 +137,26 @@ public VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory ) { + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - return new DoubleLastVectorAggregator(timeSelector, valueSelector); + VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( + columnSelectorFactory.getReadableVectorInspector(), + fieldName, + valueSelector, + capabilities.toColumnType(), + ColumnType.DOUBLE + ); + return new DoubleLastVectorAggregator(timeSelector, objectSelector); + } + + VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new DoubleLastVectorAggregator(timeSelector, vSelector); } else { - return NilVectorAggregator.of(new SerializablePair<>(0L, NullHandling.defaultDoubleValue())); + return NilVectorAggregator.of(NilVectorAggregator.DOUBLE_NIL_PAIR); } } @@ -156,7 +170,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) return new DoubleLastBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongDouble.class diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java index a5e9cf9d324e..ab02d8e86d22 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -32,16 +33,16 @@ public class DoubleLastVectorAggregator extends NumericLastVectorAggregator { double lastValue; - public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, SerializablePairLongDouble.class); lastValue = 0; } @Override - void putValue(ByteBuffer buf, int position, int index) + void putValue(ByteBuffer buf, int position, Number number) { - lastValue = valueSelector.getDoubleVector()[index]; + lastValue = number.doubleValue(); buf.putDouble(position, lastValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 7977b4ff601e..d8d71765a25c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -35,7 +34,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstLastUtils; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -46,7 +45,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -114,7 +115,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new FloatLastAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongFloat.class @@ -133,7 +134,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) return new FloatLastBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongFloat.class @@ -154,12 +155,25 @@ public VectorAggregator factorizeVector( ) { final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); + if (Types.isNumeric(capabilities)) { VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - return new FloatLastVectorAggregator(timeSelector, valueSelector); + VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( + columnSelectorFactory.getReadableVectorInspector(), + fieldName, + valueSelector, + capabilities.toColumnType(), + ColumnType.FLOAT + ); + return new FloatLastVectorAggregator(timeSelector, objectSelector); + } + + VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new FloatLastVectorAggregator(timeSelector, vSelector); } else { - return NilVectorAggregator.of(new SerializablePair<>(0L, NullHandling.defaultFloatValue())); + return NilVectorAggregator.of(NilVectorAggregator.FLOAT_NIL_PAIR); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java index b1814b5dc07e..c9bde26266c9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -32,20 +33,20 @@ public class FloatLastVectorAggregator extends NumericLastVectorAggregator { float lastValue; - public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, SerializablePairLongFloat.class); lastValue = 0; } - @Override - void putValue(ByteBuffer buf, int position, int index) + void putValue(ByteBuffer buf, int position, Number number) { - lastValue = valueSelector.getFloatVector()[index]; + lastValue = number.floatValue(); buf.putFloat(position, lastValue); } + @Override public void initValue(ByteBuffer buf, int position) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 2637d1a26c7b..751de9fbb7ef 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -35,7 +34,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstLastUtils; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -46,7 +45,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -115,7 +116,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) return new LongLastAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongLong.class @@ -134,7 +135,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) return new LongLastBufferAggregator( metricFactory.makeColumnValueSelector(timeColumn), valueSelector, - StringFirstLastUtils.selectorNeedsFoldCheck( + FirstLastUtils.selectorNeedsFoldCheck( valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongLong.class @@ -155,12 +156,26 @@ public VectorAggregator factorizeVector( ) { final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); + if (Types.isNumeric(capabilities)) { VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - return new LongLastVectorAggregator(timeSelector, valueSelector); + VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( + columnSelectorFactory.getReadableVectorInspector(), + fieldName, + valueSelector, + capabilities.toColumnType(), + ColumnType.LONG + ); + + return new LongLastVectorAggregator(timeSelector, objectSelector); + } + + VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); + if (capabilities != null) { + return new LongLastVectorAggregator(timeSelector, vSelector); } else { - return NilVectorAggregator.of(new SerializablePair<>(0L, NullHandling.defaultLongValue())); + return NilVectorAggregator.of(NilVectorAggregator.LONG_NIL_PAIR); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java index ea91430c80f1..c0446e9e8e32 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongLong; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -32,9 +33,9 @@ public class LongLastVectorAggregator extends NumericLastVectorAggregator { long lastValue; - public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, SerializablePairLongLong.class); lastValue = 0; } @@ -44,11 +45,10 @@ public void initValue(ByteBuffer buf, int position) buf.putLong(position, 0); } - @Override - void putValue(ByteBuffer buf, int position, int index) + void putValue(ByteBuffer buf, int position, Number number) { - lastValue = valueSelector.getLongVector()[index]; + lastValue = number.longValue(); buf.putLong(position, lastValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java index 717470e8921d..0cda99dfba56 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java @@ -19,8 +19,11 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.first.FirstLastUtils; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -33,15 +36,18 @@ public abstract class NumericLastVectorAggregator implements VectorAggregator { static final int NULL_OFFSET = Long.BYTES; static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; - final VectorValueSelector valueSelector; + final VectorObjectSelector valueSelector; + private final Class pairClass; private final boolean useDefault = NullHandling.replaceWithDefault(); private final VectorValueSelector timeSelector; private long lastTime; - public NumericLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + + NumericLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector, Class pairClass) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.pairClass = pairClass; lastTime = Long.MIN_VALUE; } @@ -56,13 +62,17 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) { + if (timeSelector == null) { + return; + } + final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullValueVector = valueSelector.getNullVector(); + final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); + final boolean[] nullValueVector = FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); + boolean nullAbsent = false; lastTime = buf.getLong(position); - //check if nullVector is found or not - // the nullVector is null if no null values are found - // set the nullAbsent flag accordingly + if (nullValueVector == null) { nullAbsent = true; } @@ -79,14 +89,26 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) } } - //find the first non-null value - final long latestTime = timeVector[index]; - if (latestTime >= lastTime) { - lastTime = latestTime; - if (useDefault || nullValueVector == null || !nullValueVector[index]) { - updateTimeWithValue(buf, position, lastTime, index); - } else { - updateTimeWithNull(buf, position, lastTime); + final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeNumeric[index], pairClass); + if (foldNeeded) { + final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; + if (inPair.lhs >= lastTime) { + lastTime = inPair.lhs; + if (useDefault || inPair.rhs != null) { + updateTimeWithValue(buf, position, lastTime, inPair.getRhs()); + } else { + updateTimeWithNull(buf, position, lastTime); + } + } + } else { + final long latestTime = timeVector[index]; + if (latestTime >= lastTime) { + lastTime = latestTime; + if (useDefault || nullValueVector == null || !nullValueVector[index]) { + updateTimeWithValue(buf, position, lastTime, (Number) objectsWhichMightBeNumeric[index]); + } else { + updateTimeWithNull(buf, position, lastTime); + } } } } @@ -113,21 +135,47 @@ public void aggregate( int positionOffset ) { + if (timeSelector == null) { + return; + } + + final long[] timeVector = timeSelector.getLongVector(); - boolean[] nulls = useDefault ? null : valueSelector.getNullVector(); - long[] timeVector = timeSelector.getLongVector(); + final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); + boolean[] nulls = useDefault ? null : FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); + + boolean foldNeeded = false; + for (Object obj : objectsWhichMightBeNumeric) { + if (obj != null) { + foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, pairClass); + break; + } + } for (int i = 0; i < numRows; i++) { int position = positions[i] + positionOffset; int row = rows == null ? i : rows[i]; long lastTime = buf.getLong(position); - if (timeVector[row] >= lastTime) { - if (useDefault || nulls == null || !nulls[row]) { - updateTimeWithValue(buf, position, timeVector[row], row); - } else { - updateTimeWithNull(buf, position, timeVector[row]); + + if (foldNeeded) { + final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; + if (inPair.lhs >= lastTime) { + if (useDefault || inPair.rhs != null) { + updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + } else { + updateTimeWithNull(buf, position, inPair.lhs); + } + } + } else { + if (timeVector[row] >= lastTime) { + if (useDefault || nulls == null || !nulls[row]) { + updateTimeWithValue(buf, position, timeVector[row], (Number) objectsWhichMightBeNumeric[row]); + } else { + updateTimeWithNull(buf, position, timeVector[row]); + } } } + } } @@ -136,13 +184,13 @@ public void aggregate( * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the current aggregate value is stored * @param time the time to be updated in the buffer as the last time - * @param index the index of the vectorized vector which is the last value + * @param number number which is the last value */ - void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) + void updateTimeWithValue(ByteBuffer buf, int position, long time, Number number) { buf.putLong(position, time); buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, index); + putValue(buf, position + VALUE_OFFSET, number); } /** @@ -166,7 +214,7 @@ void updateTimeWithNull(ByteBuffer buf, int position, long time) *Abstract function which needs to be overridden by subclasses to set the * latest value in the buffer depending on the datatype */ - abstract void putValue(ByteBuffer buf, int position, int index); + abstract void putValue(ByteBuffer buf, int position, Number number); @Override public void close() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index 78cba0f3e034..c8282529091c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -32,6 +32,7 @@ import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; @@ -130,7 +131,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) + FirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } @@ -146,7 +147,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(timeColumn), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) + FirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java index 00e70c78098e..09ef10572b59 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java @@ -22,6 +22,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorObjectSelector; @@ -81,7 +82,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) continue; } index = i; - final boolean foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index]); + final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index], SerializablePairLongString.class); if (foldNeeded) { // Less efficient code path when folding is a possibility (we must read the value selector first just in case // it's a foldable object). @@ -140,7 +141,7 @@ public void aggregate( boolean foldNeeded = false; for (Object obj : objectsWhichMightBeStrings) { if (obj != null) { - foldNeeded = StringFirstLastUtils.objectNeedsFoldCheck(obj); + foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, SerializablePairLongString.class); break; } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java index 69ee858cbd78..5e6344dd3368 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java @@ -241,7 +241,7 @@ public void testOverflowTransfer() throws Exception writeOutMedium ); - Assert.assertEquals(93, transferredBuffer.getSerializedSize()); + Assert.assertEquals(94, transferredBuffer.getSerializedSize()); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java index 178aab7d1cc0..1f5900ce1c99 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerdeTest.java @@ -59,7 +59,7 @@ public class SerializablePairLongDoubleComplexMetricSerdeTest @Test public void testSingle() throws Exception { - assertExpected(ImmutableList.of(new SerializablePairLongDouble(100L, 10D)), 78); + assertExpected(ImmutableList.of(new SerializablePairLongDouble(100L, 10D)), 75); } @Test @@ -86,7 +86,7 @@ public void testCompressable() throws Exception valueList.add(new SerializablePairLongDouble(Integer.MAX_VALUE + (long) i, doubleList.get(i % numLongs))); } - assertExpected(valueList, 80258); + assertExpected(valueList, 80509); } @Test @@ -99,7 +99,7 @@ public void testHighlyCompressable() throws Exception valueList.add(new SerializablePairLongDouble(Integer.MAX_VALUE + (long) i, doubleValue)); } - assertExpected(valueList, 80024); + assertExpected(valueList, 80274); } @Test @@ -111,13 +111,13 @@ public void testRandom() throws Exception valueList.add(new SerializablePairLongDouble(random.nextLong(), random.nextDouble())); } - assertExpected(valueList, 200612); + assertExpected(valueList, 210958); } @Test public void testNullRHS() throws Exception { - assertExpected(ImmutableList.of(new SerializablePairLongDouble(100L, null)), 70); + assertExpected(ImmutableList.of(new SerializablePairLongDouble(100L, null)), 71); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java index 85e17b14b8fa..9618404419af 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleDeltaEncodedStagedSerdeTest.java @@ -44,13 +44,13 @@ public void testNull() @Test public void testSimpleInteger() { - assertValueEquals(new SerializablePairLongDouble(100L, 1000000000000.12312312312D), 12, INTEGER_SERDE); + assertValueEquals(new SerializablePairLongDouble(100L, 1000000000000.12312312312D), 13, INTEGER_SERDE); } @Test public void testNullRHSInteger() { - assertValueEquals(new SerializablePairLongDouble(100L, null), 4, INTEGER_SERDE); + assertValueEquals(new SerializablePairLongDouble(100L, null), 5, INTEGER_SERDE); } @Test @@ -58,7 +58,7 @@ public void testLargeRHSInteger() { assertValueEquals( new SerializablePairLongDouble(100L, random.nextDouble()), - 12, + 13, INTEGER_SERDE ); } @@ -66,13 +66,13 @@ public void testLargeRHSInteger() @Test public void testSimpleLong() { - assertValueEquals(new SerializablePairLongDouble(100L, 1000000000000.12312312312D), 16, LONG_SERDE); + assertValueEquals(new SerializablePairLongDouble(100L, 1000000000000.12312312312D), 17, LONG_SERDE); } @Test public void testNullRHSLong() { - assertValueEquals(new SerializablePairLongDouble(100L, null), 8, LONG_SERDE); + assertValueEquals(new SerializablePairLongDouble(100L, null), 9, LONG_SERDE); } @Test @@ -80,7 +80,7 @@ public void testLargeRHSLong() { assertValueEquals( new SerializablePairLongDouble(100L, random.nextDouble()), - 16, + 17, LONG_SERDE ); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java index cb71b49b44b2..7cf12af184dc 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java @@ -241,7 +241,7 @@ public void testOverflowTransfer() throws Exception writeOutMedium ); - Assert.assertEquals(92, transferredBuffer.getSerializedSize()); + Assert.assertEquals(90, transferredBuffer.getSerializedSize()); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java index 97d6493110c0..7fc270dae1fc 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerdeTest.java @@ -59,7 +59,7 @@ public class SerializablePairLongFloatComplexMetricSerdeTest @Test public void testSingle() throws Exception { - assertExpected(ImmutableList.of(new SerializablePairLongFloat(100L, 10F)), 74); + assertExpected(ImmutableList.of(new SerializablePairLongFloat(100L, 10F)), 75); } @Test @@ -69,7 +69,7 @@ public void testLargeRHS() throws Exception assertExpected(ImmutableList.of(new SerializablePairLongFloat( 100L, random.nextFloat() - )), 74); + )), 75); } @Test @@ -86,7 +86,7 @@ public void testCompressable() throws Exception valueList.add(new SerializablePairLongFloat(Integer.MAX_VALUE + (long) i, floatList.get(i % numLongs))); } - assertExpected(valueList, 80124); + assertExpected(valueList, 80418); } @Test @@ -99,7 +99,7 @@ public void testHighlyCompressable() throws Exception valueList.add(new SerializablePairLongFloat(Integer.MAX_VALUE + (long) i, floatValue)); } - assertExpected(valueList, 79970); + assertExpected(valueList, 80260); } @Test @@ -111,13 +111,13 @@ public void testRandom() throws Exception valueList.add(new SerializablePairLongFloat(random.nextLong(), random.nextFloat())); } - assertExpected(valueList, 160464); + assertExpected(valueList, 170749); } @Test public void testNullRHS() throws Exception { - assertExpected(ImmutableList.of(new SerializablePairLongFloat(100L, null)), 70); + assertExpected(ImmutableList.of(new SerializablePairLongFloat(100L, null)), 71); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java index d76d4d7772f6..7bf898f5b142 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatDeltaEncodedStagedSerdeTest.java @@ -44,13 +44,13 @@ public void testNull() @Test public void testSimpleInteger() { - assertValueEquals(new SerializablePairLongFloat(100L, 10F), 8, INTEGER_SERDE); + assertValueEquals(new SerializablePairLongFloat(100L, 10F), 9, INTEGER_SERDE); } @Test public void testNullRHSInteger() { - assertValueEquals(new SerializablePairLongFloat(100L, null), 4, INTEGER_SERDE); + assertValueEquals(new SerializablePairLongFloat(100L, null), 5, INTEGER_SERDE); } @Test @@ -58,7 +58,7 @@ public void testLargeRHSInteger() { assertValueEquals( new SerializablePairLongFloat(100L, random.nextFloat()), - 8, + 9, INTEGER_SERDE ); } @@ -66,13 +66,13 @@ public void testLargeRHSInteger() @Test public void testSimpleLong() { - assertValueEquals(new SerializablePairLongFloat(100L, 10F), 12, LONG_SERDE); + assertValueEquals(new SerializablePairLongFloat(100L, 10F), 13, LONG_SERDE); } @Test public void testNullRHSLong() { - assertValueEquals(new SerializablePairLongFloat(100L, null), 8, LONG_SERDE); + assertValueEquals(new SerializablePairLongFloat(100L, null), 9, LONG_SERDE); } @Test @@ -80,7 +80,7 @@ public void testLargeRHSLong() { assertValueEquals( new SerializablePairLongFloat(100L, random.nextFloat()), - 12, + 13, LONG_SERDE ); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java index 20c1437297b3..61d4391c45d8 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java @@ -241,7 +241,7 @@ public void testOverflowTransfer() throws Exception writeOutMedium ); - Assert.assertEquals(92, transferredBuffer.getSerializedSize()); + Assert.assertEquals(94, transferredBuffer.getSerializedSize()); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java index 3365cee5b4d5..68429ad99f4b 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerdeTest.java @@ -86,7 +86,7 @@ public void testCompressable() throws Exception valueList.add(new SerializablePairLongLong(Integer.MAX_VALUE + (long) i, longList.get(i % numLongs))); } - assertExpected(valueList, 80258); + assertExpected(valueList, 80509); } @Test @@ -99,7 +99,7 @@ public void testHighlyCompressable() throws Exception valueList.add(new SerializablePairLongLong(Integer.MAX_VALUE + (long) i, longValue)); } - assertExpected(valueList, 80023); + assertExpected(valueList, 80274); } @Test @@ -111,13 +111,13 @@ public void testRandom() throws Exception valueList.add(new SerializablePairLongLong(random.nextLong(), random.nextLong())); } - assertExpected(valueList, 200618); + assertExpected(valueList, 210967); } @Test public void testNullRHS() throws Exception { - assertExpected(ImmutableList.of(new SerializablePairLongLong(100L, null)), 70); + assertExpected(ImmutableList.of(new SerializablePairLongLong(100L, null)), 71); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java index 6edfb022d811..6b6c0c9b037f 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongDeltaEncodedStagedSerdeTest.java @@ -44,13 +44,13 @@ public void testNull() @Test public void testSimpleInteger() { - assertValueEquals(new SerializablePairLongLong(100L, 10L), 12, INTEGER_SERDE); + assertValueEquals(new SerializablePairLongLong(100L, 10L), 13, INTEGER_SERDE); } @Test public void testNullRHSInteger() { - assertValueEquals(new SerializablePairLongLong(100L, null), 4, INTEGER_SERDE); + assertValueEquals(new SerializablePairLongLong(100L, null), 5, INTEGER_SERDE); } @Test @@ -58,7 +58,7 @@ public void testLargeRHSInteger() { assertValueEquals( new SerializablePairLongLong(100L, random.nextLong()), - 12, + 13, INTEGER_SERDE ); } @@ -66,13 +66,13 @@ public void testLargeRHSInteger() @Test public void testSimpleLong() { - assertValueEquals(new SerializablePairLongLong(100L, 10L), 16, LONG_SERDE); + assertValueEquals(new SerializablePairLongLong(100L, 10L), 17, LONG_SERDE); } @Test public void testNullRHSLong() { - assertValueEquals(new SerializablePairLongLong(100L, null), 8, LONG_SERDE); + assertValueEquals(new SerializablePairLongLong(100L, null), 9, LONG_SERDE); } @Test @@ -80,7 +80,7 @@ public void testLargeRHSLong() { assertValueEquals( new SerializablePairLongLong(100L, random.nextLong()), - 16, + 17, LONG_SERDE ); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java index e903087105f0..2e0f3e8b0ef7 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongSimpleStagedSerdeTest.java @@ -35,7 +35,7 @@ public class SerializablePairLongLongSimpleStagedSerdeTest @Test public void testSimple() { - assertValueEquals(new SerializablePairLongLong(Long.MAX_VALUE, 10L), 16); + assertValueEquals(new SerializablePairLongLong(Long.MAX_VALUE, 10L), 17); } @Test @@ -47,7 +47,7 @@ public void testNull() @Test public void testNullString() { - assertValueEquals(new SerializablePairLongLong(Long.MAX_VALUE, null), 8); + assertValueEquals(new SerializablePairLongLong(Long.MAX_VALUE, null), 9); } @Test @@ -55,7 +55,7 @@ public void testLargeRHS() { assertValueEquals( new SerializablePairLongLong(Long.MAX_VALUE, random.nextLong()), - 16 + 17 ); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java index 49c15fa22a51..e055930a2169 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java @@ -19,14 +19,13 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.vector.BaseDoubleVectorValueSelector; import org.apache.druid.segment.vector.BaseLongVectorValueSelector; import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; import org.apache.druid.segment.vector.NoFilterVectorOffset; @@ -48,23 +47,29 @@ public class DoubleFirstVectorAggregationTest extends InitializedNullHandlingTes { private static final double EPSILON = 1e-5; private static final double[] VALUES = new double[]{7.8d, 11, 23.67, 60}; - private static final boolean[] NULLS = new boolean[]{false, false, true, false}; - private long[] times = {2436, 6879, 7888, 8224}; - + private static final long[] LONG_VALUES = new long[]{1L, 2L, 3L, 4L}; + private static final float[] FLOAT_VALUES = new float[]{1.0f, 2.0f, 3.0f, 4.0f}; + private static final double[] DOUBLE_VALUES = new double[]{1.0, 2.0, 3.0, 4.0}; private static final String NAME = "NAME"; private static final String FIELD_NAME = "FIELD_NAME"; + private static final String FIELD_NAME_LONG = "LONG_NAME"; private static final String TIME_COL = "__time"; + private final long[] times = {2345001L, 2345100L, 2345200L, 2345300L}; + private final SerializablePairLongDouble[] pairs = { + new SerializablePairLongDouble(2345001L, 1D), + new SerializablePairLongDouble(2345100L, 2D), + new SerializablePairLongDouble(2345200L, 3D), + new SerializablePairLongDouble(2345300L, 4D) + }; - private VectorValueSelector selector; - + private VectorObjectSelector selector; private BaseLongVectorValueSelector timeSelector; private ByteBuffer buf; - private DoubleFirstVectorAggregator target; private DoubleFirstAggregatorFactory doubleFirstAggregatorFactory; - private VectorColumnSelectorFactory selectorFactory; + private VectorValueSelector nonLongValueSelector; @Before public void setup() @@ -86,39 +91,80 @@ public long[] getLongVector() @Override public boolean[] getNullVector() { - return NULLS; + return null; } }; - selector = new BaseDoubleVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length) + selector = new VectorObjectSelector() { + @Override + public Object[] getObjectVector() + { + return pairs; + } - }) + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 0; + } + }; + + nonLongValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + LONG_VALUES.length, + 0, + LONG_VALUES.length + )) { + @Override + public long[] getLongVector() + { + return LONG_VALUES; + } + + @Override + public float[] getFloatVector() + { + return FLOAT_VALUES; + } + @Override public double[] getDoubleVector() { - return VALUES; + return DOUBLE_VALUES; } @Nullable @Override public boolean[] getNullVector() { - if (!NullHandling.replaceWithDefault()) { - return NULLS; - } return null; } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 4; + } }; - target = new DoubleFirstVectorAggregator(timeSelector, selector); - clearBufferForPositions(0, 0); selectorFactory = new VectorColumnSelectorFactory() { @Override public ReadableVectorInspector getReadableVectorInspector() { - return null; + return new NoFilterVectorOffset(VALUES.length, 0, VALUES.length); } @Override @@ -138,17 +184,21 @@ public VectorValueSelector makeValueSelector(String column) { if (TIME_COL.equals(column)) { return timeSelector; - } else if (FIELD_NAME.equals(column)) { - return selector; - } else { - return null; + } else if (FIELD_NAME_LONG.equals(column)) { + return nonLongValueSelector; } + return null; } + @Override public VectorObjectSelector makeObjectSelector(String column) { - return null; + if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } } @Nullable @@ -157,11 +207,16 @@ public ColumnCapabilities getColumnCapabilities(String column) { if (FIELD_NAME.equals(column)) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE); + } else if (FIELD_NAME_LONG.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); } return null; } }; + target = new DoubleFirstVectorAggregator(timeSelector, selector); + clearBufferForPositions(0, 0); + doubleFirstAggregatorFactory = new DoubleFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL); } @@ -185,19 +240,19 @@ public void initValueShouldInitZero() @Test public void aggregate() { - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[0], result.lhs.longValue()); - Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs, EPSILON); } @Test public void aggregateWithNulls() { - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[0], result.lhs.longValue()); - Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs, EPSILON); } @Test @@ -209,12 +264,8 @@ public void aggregateBatchWithoutRows() target.aggregate(buf, 3, positions, null, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[i], result.lhs.longValue()); - if (!NullHandling.replaceWithDefault() && NULLS[i]) { - Assert.assertNull(result.rhs); - } else { - Assert.assertEquals(VALUES[i], result.rhs, EPSILON); - } + Assert.assertEquals(pairs[i].getLhs().longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[i].rhs, result.rhs, EPSILON); } } @@ -222,18 +273,15 @@ public void aggregateBatchWithoutRows() public void aggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; - int[] rows = new int[]{3, 2, 0}; + int[] rows = new int[]{3, 0, 2}; int positionOffset = 2; clearBufferForPositions(positionOffset, positions); target.aggregate(buf, 3, positions, rows, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[rows[i]], result.lhs.longValue()); - if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { - Assert.assertNull(result.rhs); - } else { - Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); - } + Assert.assertEquals(pairs[rows[i]].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[rows[i]].rhs, result.rhs, EPSILON); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java index 6b02037824a9..c7e8210d6c6d 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java @@ -21,12 +21,12 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.vector.BaseFloatVectorValueSelector; import org.apache.druid.segment.vector.BaseLongVectorValueSelector; import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; import org.apache.druid.segment.vector.NoFilterVectorOffset; @@ -48,24 +48,32 @@ public class FloatFirstVectorAggregationTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final float[] VALUES = new float[]{7.2f, 15.6f, 2.1f, 150.0f}; - private static final boolean[] NULLS = new boolean[]{false, false, true, false}; - private long[] times = {2436, 6879, 7888, 8224}; - + private static final long[] LONG_VALUES = new long[]{1L, 2L, 3L, 4L}; + private static final float[] FLOAT_VALUES = new float[]{1.0f, 2.0f, 3.0f, 4.0f}; + private static final double[] DOUBLE_VALUES = new double[]{1.0, 2.0, 3.0, 4.0}; + private static final boolean[] NULLS = new boolean[]{false, false, false, false}; private static final String NAME = "NAME"; private static final String FIELD_NAME = "FIELD_NAME"; + private static final String FIELD_NAME_LONG = "LONG_NAME"; private static final String TIME_COL = "__time"; + private final long[] times = {2345001L, 2345100L, 2345200L, 2345300L}; + private final SerializablePairLongFloat[] pairs = { + new SerializablePairLongFloat(2345001L, 1.2F), + new SerializablePairLongFloat(2345100L, 2.2F), + new SerializablePairLongFloat(2345200L, 3.2F), + new SerializablePairLongFloat(2345300L, 4.2F) + }; - private VectorValueSelector selector; + private VectorObjectSelector selector; private BaseLongVectorValueSelector timeSelector; private ByteBuffer buf; - private FloatFirstVectorAggregator target; private FloatFirstAggregatorFactory floatFirstAggregatorFactory; - private VectorColumnSelectorFactory selectorFactory; + private VectorValueSelector nonFloatValueSelector; @Before public void setup() @@ -87,41 +95,80 @@ public long[] getLongVector() @Override public boolean[] getNullVector() { - return NULLS; + return null; } }; - selector = new BaseFloatVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length) + selector = new VectorObjectSelector() { + @Override + public Object[] getObjectVector() + { + return pairs; + } - }) + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 0; + } + }; + + nonFloatValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + LONG_VALUES.length, + 0, + LONG_VALUES.length + )) { + @Override + public long[] getLongVector() + { + return LONG_VALUES; + } @Override public float[] getFloatVector() { - return VALUES; + return FLOAT_VALUES; + } + + @Override + public double[] getDoubleVector() + { + return DOUBLE_VALUES; } @Nullable @Override public boolean[] getNullVector() { - if (!NullHandling.replaceWithDefault()) { - return NULLS; - } - return null; + return NULLS; } - }; - target = new FloatFirstVectorAggregator(timeSelector, selector); - clearBufferForPositions(0, 0); + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 4; + } + }; selectorFactory = new VectorColumnSelectorFactory() { @Override public ReadableVectorInspector getReadableVectorInspector() { - return null; + return new NoFilterVectorOffset(VALUES.length, 0, VALUES.length); } @Override @@ -142,7 +189,7 @@ public VectorValueSelector makeValueSelector(String column) if (TIME_COL.equals(column)) { return timeSelector; } else if (FIELD_NAME.equals(column)) { - return selector; + return nonFloatValueSelector; } else { return null; } @@ -151,7 +198,11 @@ public VectorValueSelector makeValueSelector(String column) @Override public VectorObjectSelector makeObjectSelector(String column) { - return null; + if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } } @Nullable @@ -160,10 +211,16 @@ public ColumnCapabilities getColumnCapabilities(String column) { if (FIELD_NAME.equals(column)) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.FLOAT); + } else if (FIELD_NAME_LONG.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); } return null; } }; + + target = new FloatFirstVectorAggregator(timeSelector, selector); + clearBufferForPositions(0, 0); + floatFirstAggregatorFactory = new FloatFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL); } @@ -191,8 +248,8 @@ public void aggregate() target.init(buf, 0); target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[0], result.lhs.longValue()); - Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs, EPSILON); } @Test @@ -200,8 +257,8 @@ public void aggregateWithNulls() { target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[0], result.lhs.longValue()); - Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs, EPSILON); } @Test @@ -213,11 +270,11 @@ public void aggregateBatchWithoutRows() target.aggregate(buf, 3, positions, null, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[i], result.lhs.longValue()); + Assert.assertEquals(pairs[i].getLhs().longValue(), result.lhs.longValue()); if (!NullHandling.replaceWithDefault() && NULLS[i]) { Assert.assertNull(result.rhs); } else { - Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + Assert.assertEquals(pairs[i].rhs, result.rhs, EPSILON); } } } @@ -236,7 +293,7 @@ public void aggregateBatchWithRows() if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { Assert.assertNull(result.rhs); } else { - Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + Assert.assertEquals(pairs[rows[i]].rhs, result.rhs, EPSILON); } } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java index ec4017600628..a45e0f25563d 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java @@ -21,6 +21,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; @@ -48,18 +49,30 @@ public class LongFirstVectorAggregationTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final long[] VALUES = new long[]{7, 15, 2, 150}; - private static final boolean[] NULLS = new boolean[]{false, false, true, false}; + private static final long[] LONG_VALUES = new long[]{1L, 2L, 3L, 4L}; + private static final float[] FLOAT_VALUES = new float[]{1.0f, 2.0f, 3.0f, 4.0f}; + private static final double[] DOUBLE_VALUES = new double[]{1.0, 2.0, 3.0, 4.0}; + private static final boolean[] NULLS = new boolean[]{false, false, false, false}; private static final String NAME = "NAME"; private static final String FIELD_NAME = "FIELD_NAME"; + private static final String FIELD_NAME_LONG = "LONG_NAME"; private static final String TIME_COL = "__time"; - private long[] times = {2436, 6879, 7888, 8224}; - private VectorValueSelector selector; + private final long[] times = {2345001L, 2345100L, 2345200L, 2345300L}; + private final SerializablePairLongLong[] pairs = { + new SerializablePairLongLong(2345001L, 1L), + new SerializablePairLongLong(2345100L, 2L), + new SerializablePairLongLong(2345200L, 3L), + new SerializablePairLongLong(2345300L, 4L) + }; + + private VectorObjectSelector selector; private BaseLongVectorValueSelector timeSelector; private ByteBuffer buf; private LongFirstVectorAggregator target; private LongFirstAggregatorFactory longFirstAggregatorFactory; private VectorColumnSelectorFactory selectorFactory; + private VectorValueSelector nonLongValueSelector; @Before public void setup() @@ -81,40 +94,81 @@ public long[] getLongVector() @Override public boolean[] getNullVector() { - return NULLS; + return null; } }; - selector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length) + + selector = new VectorObjectSelector() { + @Override + public Object[] getObjectVector() + { + return pairs; + } - }) + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 0; + } + }; + + nonLongValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + LONG_VALUES.length, + 0, + LONG_VALUES.length + )) { @Override public long[] getLongVector() { - return VALUES; + return LONG_VALUES; + } + + @Override + public float[] getFloatVector() + { + return FLOAT_VALUES; + } + + @Override + public double[] getDoubleVector() + { + return DOUBLE_VALUES; } @Nullable @Override public boolean[] getNullVector() { - if (!NullHandling.replaceWithDefault()) { - return NULLS; - } return null; } - }; - target = new LongFirstVectorAggregator(timeSelector, selector); - clearBufferForPositions(0, 0); + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 4; + } + }; selectorFactory = new VectorColumnSelectorFactory() { @Override public ReadableVectorInspector getReadableVectorInspector() { - return null; + return new NoFilterVectorOffset(VALUES.length, 0, VALUES.length); } @Override @@ -134,17 +188,21 @@ public VectorValueSelector makeValueSelector(String column) { if (TIME_COL.equals(column)) { return timeSelector; - } else if (FIELD_NAME.equals(column)) { - return selector; - } else { - return null; + } else if (FIELD_NAME_LONG.equals(column)) { + return nonLongValueSelector; } + return null; } + @Override public VectorObjectSelector makeObjectSelector(String column) { - return null; + if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } } @Nullable @@ -153,10 +211,16 @@ public ColumnCapabilities getColumnCapabilities(String column) { if (FIELD_NAME.equals(column)) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); + } else if (FIELD_NAME_LONG.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE); } return null; } }; + + target = new LongFirstVectorAggregator(timeSelector, selector); + clearBufferForPositions(0, 0); + longFirstAggregatorFactory = new LongFirstAggregatorFactory(NAME, FIELD_NAME, TIME_COL); } @@ -180,19 +244,19 @@ public void initValueShouldInitZero() @Test public void aggregate() { - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[0], result.lhs.longValue()); - Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs, EPSILON); } @Test public void aggregateWithNulls() { - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[0], result.lhs.longValue()); - Assert.assertEquals(VALUES[0], result.rhs, EPSILON); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[0].rhs, result.rhs, EPSILON); } @Test @@ -204,11 +268,11 @@ public void aggregateBatchWithoutRows() target.aggregate(buf, 3, positions, null, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[i], result.lhs.longValue()); + Assert.assertEquals(pairs[i].getLhs().longValue(), result.lhs.longValue()); if (!NullHandling.replaceWithDefault() && NULLS[i]) { Assert.assertNull(result.rhs); } else { - Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + Assert.assertEquals(pairs[i].rhs, result.rhs, EPSILON); } } } @@ -227,7 +291,7 @@ public void aggregateBatchWithRows() if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { Assert.assertNull(result.rhs); } else { - Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + Assert.assertEquals(pairs[rows[i]].rhs, result.rhs, EPSILON); } } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java index 391aa60866bc..d3125f190e1c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java @@ -19,47 +19,215 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; +import org.apache.druid.segment.vector.NoFilterVectorOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; -@RunWith(MockitoJUnitRunner.class) public class DoubleLastVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final double[] VALUES = new double[]{7.8d, 11, 23.67, 60}; + private static final long[] LONG_VALUES = new long[]{1L, 2L, 3L, 4L}; + private static final float[] FLOAT_VALUES = new float[]{1.0f, 2.0f, 3.0f, 4.0f}; + private static final double[] DOUBLE_VALUES = new double[]{1.0, 2.0, 3.0, 4.0}; private static final boolean[] NULLS = new boolean[]{false, false, true, false}; - private long[] times = {2436, 6879, 7888, 8224}; + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String FIELD_NAME_LONG = "LONG_NAME"; + private static final String TIME_COL = "__time"; + private final long[] times = {2345001L, 2345100L, 2345200L, 2345300L}; + private final SerializablePairLongDouble[] pairs = { + new SerializablePairLongDouble(2345001L, 1D), + new SerializablePairLongDouble(2345100L, 2D), + new SerializablePairLongDouble(2345200L, 3D), + new SerializablePairLongDouble(2345300L, 4D) + }; - @Mock - private VectorValueSelector selector; - @Mock - private VectorValueSelector timeSelector; + private VectorObjectSelector selector; + private BaseLongVectorValueSelector timeSelector; private ByteBuffer buf; - private DoubleLastVectorAggregator target; + private DoubleLastAggregatorFactory doubleLastAggregatorFactory; + private VectorColumnSelectorFactory selectorFactory; + private VectorValueSelector nonLongValueSelector; + @Before public void setup() { byte[] randomBytes = new byte[1024]; ThreadLocalRandom.current().nextBytes(randomBytes); buf = ByteBuffer.wrap(randomBytes); - Mockito.doReturn(VALUES).when(selector).getDoubleVector(); - Mockito.doReturn(times).when(timeSelector).getLongVector(); + timeSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length) + { + }) + { + @Override + public long[] getLongVector() + { + return times; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + }; + selector = new VectorObjectSelector() + { + @Override + public Object[] getObjectVector() + { + return pairs; + } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 0; + } + }; + + nonLongValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + LONG_VALUES.length, + 0, + LONG_VALUES.length + )) + { + @Override + public long[] getLongVector() + { + return LONG_VALUES; + } + + @Override + public float[] getFloatVector() + { + return FLOAT_VALUES; + } + + @Override + public double[] getDoubleVector() + { + return DOUBLE_VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 4; + } + }; + + selectorFactory = new VectorColumnSelectorFactory() + { + @Override + public ReadableVectorInspector getReadableVectorInspector() + { + return new NoFilterVectorOffset(VALUES.length, 0, VALUES.length); + } + + @Override + public SingleValueDimensionVectorSelector makeSingleValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public MultiValueDimensionVectorSelector makeMultiValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public VectorValueSelector makeValueSelector(String column) + { + if (TIME_COL.equals(column)) { + return timeSelector; + } else if (FIELD_NAME_LONG.equals(column)) { + return nonLongValueSelector; + } + return null; + } + + + @Override + public VectorObjectSelector makeObjectSelector(String column) + { + if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + if (FIELD_NAME.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE); + } else if (FIELD_NAME_LONG.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); + } + return null; + } + }; + target = new DoubleLastVectorAggregator(timeSelector, selector); clearBufferForPositions(0, 0); + + doubleLastAggregatorFactory = new DoubleLastAggregatorFactory(NAME, FIELD_NAME, TIME_COL); + } + + @Test + public void testFactory() + { + Assert.assertTrue(doubleLastAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = doubleLastAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(DoubleLastVectorAggregator.class, vectorAggregator.getClass()); } @Test @@ -73,20 +241,19 @@ public void initValueShouldInitZero() @Test public void aggregate() { - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[3], result.lhs.longValue()); - Assert.assertEquals(VALUES[3], result.rhs, EPSILON); + Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[3].rhs, result.rhs, EPSILON); } @Test public void aggregateWithNulls() { - mockNullsVector(); - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[3], result.lhs.longValue()); - Assert.assertEquals(VALUES[3], result.rhs, EPSILON); + Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[3].rhs, result.rhs, EPSILON); } @Test @@ -98,8 +265,8 @@ public void aggregateBatchWithoutRows() target.aggregate(buf, 3, positions, null, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[i], result.lhs.longValue()); - Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + Assert.assertEquals(pairs[i].getLhs().longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[i].rhs, result.rhs, EPSILON); } } @@ -113,8 +280,8 @@ public void aggregateBatchWithRows() target.aggregate(buf, 3, positions, rows, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[rows[i]], result.lhs.longValue()); - Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + Assert.assertEquals(pairs[rows[i]].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[rows[i]].rhs, result.rhs, EPSILON); } } @@ -124,11 +291,4 @@ private void clearBufferForPositions(int offset, int... positions) target.init(buf, offset + position); } } - - private void mockNullsVector() - { - if (!NullHandling.replaceWithDefault()) { - Mockito.doReturn(NULLS).when(selector).getNullVector(); - } - } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java index 82615bcd7fe2..957d585e5fd9 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java @@ -21,45 +21,217 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; +import org.apache.druid.segment.vector.NoFilterVectorOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; -@RunWith(MockitoJUnitRunner.class) public class FloatLastVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final float[] VALUES = new float[]{7.2f, 15.6f, 2.1f, 150.0f}; - private static final boolean[] NULLS = new boolean[]{false, false, true, false}; - private long[] times = {2436, 6879, 7888, 8224}; + private static final long[] LONG_VALUES = new long[]{1L, 2L, 3L, 4L}; + private static final float[] FLOAT_VALUES = new float[]{1.0f, 2.0f, 3.0f, 4.0f}; + private static final double[] DOUBLE_VALUES = new double[]{1.0, 2.0, 3.0, 4.0}; + private static final boolean[] NULLS = new boolean[]{false, false, false, false}; + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String FIELD_NAME_LONG = "LONG_NAME"; + private static final String TIME_COL = "__time"; + private final long[] times = {2345001L, 2345100L, 2345200L, 2345300L}; + private final SerializablePairLongFloat[] pairs = { + new SerializablePairLongFloat(2345001L, 1.2F), + new SerializablePairLongFloat(2345100L, 2.2F), + new SerializablePairLongFloat(2345200L, 3.2F), + new SerializablePairLongFloat(2345300L, 4.2F) + }; + - @Mock - private VectorValueSelector selector; - @Mock - private VectorValueSelector timeSelector; - private ByteBuffer buf; + private VectorObjectSelector selector; + private BaseLongVectorValueSelector timeSelector; + private ByteBuffer buf; private FloatLastVectorAggregator target; + private FloatLastAggregatorFactory floatLastAggregatorFactory; + private VectorColumnSelectorFactory selectorFactory; + private VectorValueSelector nonFloatValueSelector; + @Before public void setup() { byte[] randomBytes = new byte[1024]; ThreadLocalRandom.current().nextBytes(randomBytes); buf = ByteBuffer.wrap(randomBytes); - Mockito.doReturn(VALUES).when(selector).getFloatVector(); - Mockito.doReturn(times).when(timeSelector).getLongVector(); + timeSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length) + { + }) + { + @Override + public long[] getLongVector() + { + return times; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + }; + selector = new VectorObjectSelector() + { + @Override + public Object[] getObjectVector() + { + return pairs; + } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 0; + } + }; + + nonFloatValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + LONG_VALUES.length, + 0, + LONG_VALUES.length + )) + { + @Override + public long[] getLongVector() + { + return LONG_VALUES; + } + + @Override + public float[] getFloatVector() + { + return FLOAT_VALUES; + } + + @Override + public double[] getDoubleVector() + { + return DOUBLE_VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return NULLS; + } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 4; + } + }; + + selectorFactory = new VectorColumnSelectorFactory() + { + @Override + public ReadableVectorInspector getReadableVectorInspector() + { + return new NoFilterVectorOffset(VALUES.length, 0, VALUES.length); + } + + @Override + public SingleValueDimensionVectorSelector makeSingleValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public MultiValueDimensionVectorSelector makeMultiValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public VectorValueSelector makeValueSelector(String column) + { + if (TIME_COL.equals(column)) { + return timeSelector; + } else if (FIELD_NAME.equals(column)) { + return nonFloatValueSelector; + } else { + return null; + } + } + + @Override + public VectorObjectSelector makeObjectSelector(String column) + { + if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + if (FIELD_NAME.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.FLOAT); + } else if (FIELD_NAME_LONG.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); + } + return null; + } + }; + target = new FloatLastVectorAggregator(timeSelector, selector); clearBufferForPositions(0, 0); + + floatLastAggregatorFactory = new FloatLastAggregatorFactory(NAME, FIELD_NAME, TIME_COL); + + } + + @Test + public void testFactory() + { + Assert.assertTrue(floatLastAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = floatLastAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(FloatLastVectorAggregator.class, vectorAggregator.getClass()); } @Test @@ -76,18 +248,17 @@ public void aggregate() target.init(buf, 0); target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[3], result.lhs.longValue()); - Assert.assertEquals(VALUES[3], result.rhs, EPSILON); + Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[3].rhs, result.rhs, EPSILON); } @Test public void aggregateWithNulls() { - mockNullsVector(); target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[3], result.lhs.longValue()); - Assert.assertEquals(VALUES[3], result.rhs, EPSILON); + Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[3].rhs, result.rhs, EPSILON); } @Test @@ -99,8 +270,12 @@ public void aggregateBatchWithoutRows() target.aggregate(buf, 3, positions, null, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[i], result.lhs.longValue()); - Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + Assert.assertEquals(pairs[i].getLhs().longValue(), result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[i]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(pairs[i].rhs, result.rhs, EPSILON); + } } } @@ -115,7 +290,11 @@ public void aggregateBatchWithRows() for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); Assert.assertEquals(times[rows[i]], result.lhs.longValue()); - Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(pairs[rows[i]].rhs, result.rhs, EPSILON); + } } } @@ -125,12 +304,5 @@ private void clearBufferForPositions(int offset, int... positions) target.init(buf, offset + position); } } - - private void mockNullsVector() - { - if (!NullHandling.replaceWithDefault()) { - Mockito.doReturn(NULLS).when(selector).getNullVector(); - } - } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java index acbc1a8f2480..6cc8bfa9f5a2 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java @@ -21,45 +21,215 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; +import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseLongVectorValueSelector; +import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; +import org.apache.druid.segment.vector.NoFilterVectorOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; -@RunWith(MockitoJUnitRunner.class) public class LongLastVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final long[] VALUES = new long[]{7, 15, 2, 150}; - private static final boolean[] NULLS = new boolean[]{false, false, true, false}; - private long[] times = {2436, 6879, 7888, 8224}; + private static final long[] LONG_VALUES = new long[]{1L, 2L, 3L, 4L}; + private static final float[] FLOAT_VALUES = new float[]{1.0f, 2.0f, 3.0f, 4.0f}; + private static final double[] DOUBLE_VALUES = new double[]{1.0, 2.0, 3.0, 4.0}; + private static final boolean[] NULLS = new boolean[]{false, false, false, false}; + private static final String NAME = "NAME"; + private static final String FIELD_NAME = "FIELD_NAME"; + private static final String FIELD_NAME_LONG = "LONG_NAME"; + private static final String TIME_COL = "__time"; + private final long[] times = {2345001L, 2345100L, 2345200L, 2345300L}; + private final SerializablePairLongLong[] pairs = { + new SerializablePairLongLong(2345001L, 1L), + new SerializablePairLongLong(2345100L, 2L), + new SerializablePairLongLong(2345200L, 3L), + new SerializablePairLongLong(2345300L, 4L) + }; - @Mock - private VectorValueSelector selector; - @Mock - private VectorValueSelector timeSelector; + private VectorObjectSelector selector; + private BaseLongVectorValueSelector timeSelector; private ByteBuffer buf; - private LongLastVectorAggregator target; + private LongLastAggregatorFactory longLastAggregatorFactory; + private VectorColumnSelectorFactory selectorFactory; + private VectorValueSelector nonLongValueSelector; + @Before public void setup() { byte[] randomBytes = new byte[1024]; ThreadLocalRandom.current().nextBytes(randomBytes); buf = ByteBuffer.wrap(randomBytes); - Mockito.doReturn(VALUES).when(selector).getLongVector(); - Mockito.doReturn(times).when(timeSelector).getLongVector(); + timeSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length) + { + }) + { + @Override + public long[] getLongVector() + { + return times; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + }; + + selector = new VectorObjectSelector() + { + @Override + public Object[] getObjectVector() + { + return pairs; + } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 0; + } + }; + + nonLongValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + LONG_VALUES.length, + 0, + LONG_VALUES.length + )) + { + @Override + public long[] getLongVector() + { + return LONG_VALUES; + } + + @Override + public float[] getFloatVector() + { + return FLOAT_VALUES; + } + + @Override + public double[] getDoubleVector() + { + return DOUBLE_VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + + @Override + public int getMaxVectorSize() + { + return 4; + } + + @Override + public int getCurrentVectorSize() + { + return 4; + } + }; + + selectorFactory = new VectorColumnSelectorFactory() + { + @Override + public ReadableVectorInspector getReadableVectorInspector() + { + return new NoFilterVectorOffset(VALUES.length, 0, VALUES.length); + } + + @Override + public SingleValueDimensionVectorSelector makeSingleValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public MultiValueDimensionVectorSelector makeMultiValueDimensionSelector(DimensionSpec dimensionSpec) + { + return null; + } + + @Override + public VectorValueSelector makeValueSelector(String column) + { + if (TIME_COL.equals(column)) { + return timeSelector; + } else if (FIELD_NAME_LONG.equals(column)) { + return nonLongValueSelector; + } + return null; + } + + + @Override + public VectorObjectSelector makeObjectSelector(String column) + { + if (FIELD_NAME.equals(column)) { + return selector; + } else { + return null; + } + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + if (FIELD_NAME.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.LONG); + } else if (FIELD_NAME_LONG.equals(column)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE); + } + return null; + } + }; + target = new LongLastVectorAggregator(timeSelector, selector); clearBufferForPositions(0, 0); + + longLastAggregatorFactory = new LongLastAggregatorFactory(NAME, FIELD_NAME, TIME_COL); + } + + @Test + public void testFactory() + { + Assert.assertTrue(longLastAggregatorFactory.canVectorize(selectorFactory)); + VectorAggregator vectorAggregator = longLastAggregatorFactory.factorizeVector(selectorFactory); + Assert.assertNotNull(vectorAggregator); + Assert.assertEquals(LongLastVectorAggregator.class, vectorAggregator.getClass()); } @Test @@ -73,20 +243,19 @@ public void initValueShouldInitZero() @Test public void aggregate() { - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[3], result.lhs.longValue()); - Assert.assertEquals(VALUES[3], result.rhs, EPSILON); + Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[3].rhs, result.rhs, EPSILON); } @Test public void aggregateWithNulls() { - mockNullsVector(); - target.aggregate(buf, 0, 0, VALUES.length); + target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); - Assert.assertEquals(times[3], result.lhs.longValue()); - Assert.assertEquals(VALUES[3], result.rhs, EPSILON); + Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); + Assert.assertEquals(pairs[3].rhs, result.rhs, EPSILON); } @Test @@ -98,8 +267,12 @@ public void aggregateBatchWithoutRows() target.aggregate(buf, 3, positions, null, positionOffset); for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); - Assert.assertEquals(times[i], result.lhs.longValue()); - Assert.assertEquals(VALUES[i], result.rhs, EPSILON); + Assert.assertEquals(pairs[i].getLhs().longValue(), result.lhs.longValue()); + if (!NullHandling.replaceWithDefault() && NULLS[i]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(pairs[i].rhs, result.rhs, EPSILON); + } } } @@ -114,7 +287,11 @@ public void aggregateBatchWithRows() for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); Assert.assertEquals(times[rows[i]], result.lhs.longValue()); - Assert.assertEquals(VALUES[rows[i]], result.rhs, EPSILON); + if (!NullHandling.replaceWithDefault() && NULLS[rows[i]]) { + Assert.assertNull(result.rhs); + } else { + Assert.assertEquals(pairs[rows[i]].rhs, result.rhs, EPSILON); + } } } @@ -124,11 +301,4 @@ private void clearBufferForPositions(int offset, int... positions) target.init(buf, offset + position); } } - - private void mockNullsVector() - { - if (!NullHandling.replaceWithDefault()) { - Mockito.doReturn(NULLS).when(selector).getNullVector(); - } - } } From 33c7d85d9c3306820279220c99bfa9d0523ca78b Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Thu, 12 Oct 2023 00:26:54 -0700 Subject: [PATCH 10/14] fix UT + static check failures --- .../query/aggregation/first/FirstLastUtils.java | 8 +++++--- .../first/LongFirstAggregatorFactory.java | 2 -- .../first/NumericFirstVectorAggregator.java | 13 +++++++------ .../last/FloatLastAggregatorFactory.java | 2 +- .../last/LongLastAggregatorFactory.java | 2 +- .../last/NumericLastVectorAggregator.java | 15 ++++++++------- .../last/StringLastAggregatorFactory.java | 1 - ...SerializablePairLongDoubleBufferStoreTest.java | 2 +- .../SerializablePairLongFloatBufferStoreTest.java | 2 +- .../SerializablePairLongLongBufferStoreTest.java | 2 +- 10 files changed, 25 insertions(+), 24 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java index 7120d3a4605f..35b52641ab76 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -70,14 +70,16 @@ public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) public static boolean[] getNullVector(Object[] objectVector) { - boolean containsNonNullValues = false; + boolean containsNullValues = false; boolean[] nullValueVector = new boolean[objectVector.length]; for (int i = 0; i < objectVector.length; i++) { if (objectVector[i] != null) { - containsNonNullValues = true; nullValueVector[i] = false; + } else { + nullValueVector[i] = true; + containsNullValues = true; } } - return containsNonNullValues ? null : nullValueVector; + return containsNullValues ? nullValueVector : null; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 39afef69c300..c4e8c72ff30d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -59,7 +58,6 @@ public class LongFirstAggregatorFactory extends AggregatorFactory { public static final ColumnType TYPE = ColumnType.ofComplex(SerializablePairLongLongComplexMetricSerde.TYPE_NAME); - private static final Logger log = new Logger(LongFirstAggregatorFactory.class); private static final Aggregator NIL_AGGREGATOR = new LongFirstAggregator( NilColumnValueSelector.instance(), diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java index 46d311ed45fa..72da1c687c5f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java @@ -89,7 +89,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) if (foldNeeded) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; - if (inPair.lhs < firstTime) { + if (inPair.lhs != null && inPair.lhs < firstTime) { firstTime = inPair.lhs; if (useDefault || inPair.rhs != null) { updateTimeWithValue(buf, position, firstTime, inPair.getRhs()); @@ -158,11 +158,13 @@ public void aggregate( if (foldNeeded) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; if (useDefault || inPair != null) { - if (inPair.lhs < firstTime) { - updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + if (inPair.lhs != null && inPair.lhs < firstTime) { + if (inPair.rhs != null) { + updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + } else { + updateTimeWithNull(buf, position, inPair.lhs); + } } - } else { - updateTimeWithNull(buf, position, inPair.lhs); } } else { if (timeVector[row] < firstTime) { @@ -173,7 +175,6 @@ public void aggregate( } } } - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index d8d71765a25c..e240af081261 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -33,8 +33,8 @@ import org.apache.druid.query.aggregation.SerializablePairLongFloatComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.FirstLastUtils; +import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 751de9fbb7ef..9996bd624729 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -33,8 +33,8 @@ import org.apache.druid.query.aggregation.SerializablePairLongLongComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.any.NilVectorAggregator; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.FirstLastUtils; +import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java index 0cda99dfba56..a758ca3fd29b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java @@ -92,7 +92,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeNumeric[index], pairClass); if (foldNeeded) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; - if (inPair.lhs >= lastTime) { + if (inPair.lhs != null && inPair.lhs >= lastTime) { lastTime = inPair.lhs; if (useDefault || inPair.rhs != null) { updateTimeWithValue(buf, position, lastTime, inPair.getRhs()); @@ -159,11 +159,13 @@ public void aggregate( if (foldNeeded) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; - if (inPair.lhs >= lastTime) { - if (useDefault || inPair.rhs != null) { - updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); - } else { - updateTimeWithNull(buf, position, inPair.lhs); + if (useDefault || inPair != null) { + if (inPair.lhs != null && inPair.lhs >= lastTime) { + if (inPair.rhs != null) { + updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + } else { + updateTimeWithNull(buf, position, inPair.lhs); + } } } } else { @@ -175,7 +177,6 @@ public void aggregate( } } } - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index c8282529091c..16180a1f4cd3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -34,7 +34,6 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.segment.BaseObjectColumnValueSelector; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java index 5e6344dd3368..0e9ab9b4b80e 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStoreTest.java @@ -328,7 +328,7 @@ private void assertTransferredValuesEqual(SerializablePairLongDouble[] input) th private static SerializablePairLongDoubleComplexColumn createComplexColumn( AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, HeapByteBufferWriteOutBytes resultChannel - ) throws IOException + ) { ByteBuffer byteBuffer = ByteBuffer.allocate(Ints.checkedCast(transferredBuffer.getSerializedSize())); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java index 7cf12af184dc..c670bca9684c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStoreTest.java @@ -328,7 +328,7 @@ private void assertTransferredValuesEqual(SerializablePairLongFloat[] input) thr private static SerializablePairLongFloatComplexColumn createComplexColumn( AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, HeapByteBufferWriteOutBytes resultChannel - ) throws IOException + ) { ByteBuffer byteBuffer = ByteBuffer.allocate(Ints.checkedCast(transferredBuffer.getSerializedSize())); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java index 61d4391c45d8..22ffd82a1688 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStoreTest.java @@ -328,7 +328,7 @@ private void assertTransferredValuesEqual(SerializablePairLongLong[] input) thro private static SerializablePairLongLongComplexColumn createComplexColumn( AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, HeapByteBufferWriteOutBytes resultChannel - ) throws IOException + ) { ByteBuffer byteBuffer = ByteBuffer.allocate(Ints.checkedCast(transferredBuffer.getSerializedSize())); From c4fa03e059b21e7461eb6d96b807c4517f176f8c Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Thu, 30 Nov 2023 23:50:59 -0800 Subject: [PATCH 11/14] address comments --- ...erializablePairLongObjectColumnHeader.java | 1 - ...SerializablePairLongDoubleBufferStore.java | 4 +-- ...rializablePairLongDoubleComplexColumn.java | 4 +-- ...zablePairLongDoubleComplexMetricSerde.java | 1 + .../SerializablePairLongFloatBufferStore.java | 4 +-- ...erializablePairLongFloatComplexColumn.java | 4 +-- ...izablePairLongFloatComplexMetricSerde.java | 1 + .../SerializablePairLongLongBufferStore.java | 4 +-- ...SerializablePairLongLongComplexColumn.java | 4 +-- ...lizablePairLongLongComplexMetricSerde.java | 1 + .../first/DoubleFirstAggregatorFactory.java | 24 ++---------------- .../aggregation/first/FirstLastUtils.java | 9 +++++-- .../first/FloatFirstAggregatorFactory.java | 23 +---------------- .../first/LongFirstAggregatorFactory.java | 22 +--------------- .../first/StringFirstAggregatorFactory.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 24 +----------------- .../last/FloatLastAggregatorFactory.java | 24 +----------------- .../last/LongLastAggregatorFactory.java | 25 +------------------ .../last/StringLastAggregatorFactory.java | 2 +- 19 files changed, 31 insertions(+), 152 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java index 16c4dfe375b5..a9839460d9eb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java @@ -32,7 +32,6 @@ public abstract class AbstractSerializablePairLongObjectColumnHeader> { - public static final int EXPECTED_VERSION = 3; private static final int HEADER_SIZE_BYTES = 4; private static final int USE_INTEGER_MASK = 0x80; private static final int VERSION_INDEX = 0; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java index b5c22c159dfe..c5d84f62fc43 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java @@ -46,13 +46,13 @@ public AbstractSerializablePairLongObjectColumnHeader { + public static final int EXPECTED_VERSION = 1; public static final String TYPE_NAME = "serializablePairLongDouble"; private static final SerializablePairLongDoubleSimpleStagedSerde SERDE = new SerializablePairLongDoubleSimpleStagedSerde(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java index 9fe268da2023..1314d1987dc8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java @@ -46,13 +46,13 @@ public AbstractSerializablePairLongObjectColumnHeader if (maxDelta <= Integer.MAX_VALUE) { columnHeader = new SerializablePairLongFloatColumnHeader( - AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + SerializablePairLongFloatComplexMetricSerde.EXPECTED_VERSION, true, minValue ); } else { columnHeader = new SerializablePairLongFloatColumnHeader( - AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + SerializablePairLongFloatComplexMetricSerde.EXPECTED_VERSION, false, minValue ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java index 14b7f268c2d5..fa841b240722 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexColumn.java @@ -101,9 +101,9 @@ public Builder(ByteBuffer buffer) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(masterByteBuffer, SerializablePairLongFloat.class); Preconditions.checkArgument( - columnHeader.getVersion() == AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + columnHeader.getVersion() == SerializablePairLongFloatComplexMetricSerde.EXPECTED_VERSION, "version %s expected, got %s", - AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + SerializablePairLongFloatComplexMetricSerde.EXPECTED_VERSION, columnHeader.getVersion() ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java index 2ffcb635b82e..7578d06c275e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java @@ -32,6 +32,7 @@ public class SerializablePairLongFloatComplexMetricSerde extends AbstractSerializableLongObjectPairSerde { + public static final int EXPECTED_VERSION = 1; public static final String TYPE_NAME = "serializablePairLongFloat"; private static final SerializablePairLongFloatSimpleStagedSerde SERDE = new SerializablePairLongFloatSimpleStagedSerde(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java index 1dd1465dde4c..fd9efe668da7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java @@ -45,13 +45,13 @@ public AbstractSerializablePairLongObjectColumnHeader if (maxDelta <= Integer.MAX_VALUE) { columnHeader = new SerializablePairLongLongColumnHeader( - AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + SerializablePairLongLongComplexMetricSerde.EXPECTED_VERSION, true, minValue ); } else { columnHeader = new SerializablePairLongLongColumnHeader( - AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + SerializablePairLongLongComplexMetricSerde.EXPECTED_VERSION, false, minValue ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java index 1a80541f2966..b5a3806f5642 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexColumn.java @@ -101,9 +101,9 @@ public Builder(ByteBuffer buffer) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(masterByteBuffer, SerializablePairLongLong.class); Preconditions.checkArgument( - columnHeader.getVersion() == AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + columnHeader.getVersion() == SerializablePairLongLongComplexMetricSerde.EXPECTED_VERSION, "version %s expected, got %s", - AbstractSerializablePairLongObjectColumnHeader.EXPECTED_VERSION, + SerializablePairLongLongComplexMetricSerde.EXPECTED_VERSION, columnHeader.getVersion() ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java index a5f79f007279..3914e38d2660 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java @@ -32,6 +32,7 @@ public class SerializablePairLongLongComplexMetricSerde extends AbstractSerializableLongObjectPairSerde { + public static final int EXPECTED_VERSION = 1; public static final String TYPE_NAME = "serializablePairLongLong"; private static final SerializablePairLongLongSimpleStagedSerde SERDE = new SerializablePairLongLongSimpleStagedSerde(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 20251410f38f..4b004cd0b254 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -32,20 +32,16 @@ import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.SerializablePairLongDoubleComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -161,25 +157,9 @@ public VectorAggregator factorizeVector( ) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); - - if (Types.isNumeric(capabilities)) { - VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( - columnSelectorFactory.getReadableVectorInspector(), - fieldName, - valueSelector, - capabilities.toColumnType(), - ColumnType.DOUBLE - ); - return new DoubleFirstVectorAggregator(timeSelector, objectSelector); - } - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - if (capabilities != null) { - return new DoubleFirstVectorAggregator(timeSelector, vSelector); - } - return NilVectorAggregator.of(NilVectorAggregator.DOUBLE_NIL_PAIR); + return new DoubleFirstVectorAggregator(timeSelector, vSelector); + } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java index 35b52641ab76..2b2ceb389e85 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -30,7 +30,7 @@ public class FirstLastUtils { /** - * Returns whether a given value selector *might* contain SerializablePairLongString objects. + * Returns whether a given value selector *might* contain object assignable from pairClass (SerializablePairLong*). */ public static boolean selectorNeedsFoldCheck( final BaseObjectColumnValueSelector valueSelector, @@ -48,7 +48,7 @@ public static boolean selectorNeedsFoldCheck( return false; } - // Check if the selector class could possibly be a SerializablePairLongString (either a superclass or subclass). + // Check if the selector class could possibly be of pairClass* (either a superclass or subclass). final Class clazz = valueSelector.classOfObject(); return clazz.isAssignableFrom(pairClass) || pairClass.isAssignableFrom(clazz); @@ -68,6 +68,11 @@ public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) } + /** + * Returns boolean array indicating null values for the given objectVector. + * Returns null is no null objects are found in objectVector + */ + @Nullable public static boolean[] getNullVector(Object[] objectVector) { boolean containsNullValues = false; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 51eb0345d77c..2c0831e96b6c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -32,20 +32,16 @@ import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongFloatComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -150,25 +146,8 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); - if (Types.isNumeric(capabilities)) { - VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( - columnSelectorFactory.getReadableVectorInspector(), - fieldName, - valueSelector, - capabilities.toColumnType(), - ColumnType.FLOAT - ); - return new FloatFirstVectorAggregator(timeSelector, objectSelector); - } - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - if (capabilities != null) { - return new FloatFirstVectorAggregator(timeSelector, vSelector); - } - - return NilVectorAggregator.of(NilVectorAggregator.FLOAT_NIL_PAIR); + return new FloatFirstVectorAggregator(timeSelector, vSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index c4e8c72ff30d..ce85f4a2bcc3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -32,20 +32,16 @@ import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongLongComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -149,24 +145,8 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); - if (Types.isNumeric(capabilities)) { - VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( - columnSelectorFactory.getReadableVectorInspector(), - fieldName, - valueSelector, - capabilities.toColumnType(), - ColumnType.LONG - ); - return new LongFirstVectorAggregator(timeSelector, objectSelector); - } - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - if (capabilities != null) { - return new LongFirstVectorAggregator(timeSelector, vSelector); - } - return NilVectorAggregator.of(NilVectorAggregator.LONG_NIL_PAIR); + return new LongFirstVectorAggregator(timeSelector, vSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index 8dbe12d4ddc5..61486e0936ad 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -245,7 +245,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new StringFirstAggregateCombiner(); + return new GenericFirstAggregateCombiner(SerializablePairLongString.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index bac1109e570c..e60121e6daf0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -32,7 +32,6 @@ import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.SerializablePairLongDoubleComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.cache.CacheKeyBuilder; @@ -40,14 +39,11 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -138,26 +134,8 @@ public VectorAggregator factorizeVector( ) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); - - if (Types.isNumeric(capabilities)) { - VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( - columnSelectorFactory.getReadableVectorInspector(), - fieldName, - valueSelector, - capabilities.toColumnType(), - ColumnType.DOUBLE - ); - return new DoubleLastVectorAggregator(timeSelector, objectSelector); - } - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - if (capabilities != null) { - return new DoubleLastVectorAggregator(timeSelector, vSelector); - } else { - return NilVectorAggregator.of(NilVectorAggregator.DOUBLE_NIL_PAIR); - } + return new DoubleLastVectorAggregator(timeSelector, vSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index e240af081261..c35138f35737 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -32,7 +32,6 @@ import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongFloatComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; @@ -40,14 +39,11 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -154,27 +150,9 @@ public VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory ) { - final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - - if (Types.isNumeric(capabilities)) { - VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( - columnSelectorFactory.getReadableVectorInspector(), - fieldName, - valueSelector, - capabilities.toColumnType(), - ColumnType.FLOAT - ); - return new FloatLastVectorAggregator(timeSelector, objectSelector); - } - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - if (capabilities != null) { - return new FloatLastVectorAggregator(timeSelector, vSelector); - } else { - return NilVectorAggregator.of(NilVectorAggregator.FLOAT_NIL_PAIR); - } + return new FloatLastVectorAggregator(timeSelector, vSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 9996bd624729..222ef70e3389 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -32,7 +32,6 @@ import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongLongComplexMetricSerde; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.any.NilVectorAggregator; import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; @@ -40,14 +39,11 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -155,28 +151,9 @@ public VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory ) { - final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - - if (Types.isNumeric(capabilities)) { - VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); - VectorObjectSelector objectSelector = ExpressionVectorSelectors.castValueSelectorToObject( - columnSelectorFactory.getReadableVectorInspector(), - fieldName, - valueSelector, - capabilities.toColumnType(), - ColumnType.LONG - ); - - return new LongLastVectorAggregator(timeSelector, objectSelector); - } - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - if (capabilities != null) { - return new LongLastVectorAggregator(timeSelector, vSelector); - } else { - return NilVectorAggregator.of(NilVectorAggregator.LONG_NIL_PAIR); - } + return new LongLastVectorAggregator(timeSelector, vSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index 16180a1f4cd3..a9bb2f0bbd6f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -210,7 +210,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new StringLastAggregateCombiner(); + return new GenericLastAggregateCombiner(SerializablePairLongString.class); } @Override From e3cc206d1e4baa24b7355eed865953013f5fa99b Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Fri, 1 Dec 2023 13:37:42 -0800 Subject: [PATCH 12/14] address comments + fix failure checks --- .../aggregation/any/NilVectorAggregator.java | 7 -- .../first/DoubleFirstAggregatorFactory.java | 11 +++- .../first/DoubleFirstVectorAggregator.java | 13 ++-- .../first/FloatFirstAggregatorFactory.java | 11 +++- .../first/FloatFirstVectorAggregator.java | 13 ++-- .../first/LongFirstAggregatorFactory.java | 11 +++- .../first/LongFirstVectorAggregator.java | 13 ++-- .../first/NumericFirstVectorAggregator.java | 64 +++++++++---------- .../first/StringFirstAggregateCombiner.java | 59 ----------------- .../last/DoubleLastAggregatorFactory.java | 11 +++- .../last/DoubleLastVectorAggregator.java | 14 ++-- .../last/FloatLastAggregatorFactory.java | 11 +++- .../last/FloatLastVectorAggregator.java | 14 ++-- .../last/LongLastAggregatorFactory.java | 11 +++- .../last/LongLastVectorAggregator.java | 14 ++-- .../last/NumericLastVectorAggregator.java | 59 +++++++++-------- .../last/StringLastAggregateCombiner.java | 60 ----------------- 17 files changed, 174 insertions(+), 222 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregateCombiner.java delete mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregateCombiner.java diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java index 80a1e0f7e12a..67d11d820a64 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java @@ -20,9 +20,6 @@ package org.apache.druid.query.aggregation.any; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.query.aggregation.SerializablePairLongDouble; -import org.apache.druid.query.aggregation.SerializablePairLongFloat; -import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; import javax.annotation.Nullable; @@ -45,10 +42,6 @@ public class NilVectorAggregator implements VectorAggregator NullHandling.defaultLongValue() ); - public static final SerializablePairLongDouble DOUBLE_NIL_PAIR = new SerializablePairLongDouble(0L, NullHandling.defaultDoubleValue()); - public static final SerializablePairLongLong LONG_NIL_PAIR = new SerializablePairLongLong(0L, NullHandling.defaultLongValue()); - public static final SerializablePairLongFloat FLOAT_NIL_PAIR = new SerializablePairLongFloat(0L, NullHandling.defaultFloatValue()); - /** * @return A vectorized aggregator that returns the default double value. */ diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 4b004cd0b254..92880be84573 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -37,8 +37,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -157,8 +159,13 @@ public VectorAggregator factorizeVector( ) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - return new DoubleFirstVectorAggregator(timeSelector, vSelector); + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + return new DoubleFirstVectorAggregator(timeSelector, valueSelector); + } + VectorObjectSelector objectSelector = columnSelectorFactory.makeObjectSelector(fieldName); + return new DoubleFirstVectorAggregator(timeSelector, objectSelector); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java index d58882e8439d..0fdd06231feb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregator.java @@ -29,9 +29,14 @@ public class DoubleFirstVectorAggregator extends NumericFirstVectorAggregator { - public DoubleFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) + public DoubleFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { - super(timeSelector, valueSelector, SerializablePairLongDouble.class); + super(timeSelector, null, objectSelector); + } + + public DoubleFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector, null); } @Override @@ -42,9 +47,9 @@ public void initValue(ByteBuffer buf, int position) @Override - void putValue(ByteBuffer buf, int position, Number number) + void putValue(ByteBuffer buf, int position, int index) { - double firstValue = number.doubleValue(); + double firstValue = valueSelector != null ? valueSelector.getDoubleVector()[index] : ((SerializablePairLongDouble) (objectSelector.getObjectVector()[index])).getRhs(); buf.putDouble(position, firstValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 2c0831e96b6c..5faf04607df7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -37,8 +37,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -146,8 +148,13 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - return new FloatFirstVectorAggregator(timeSelector, vSelector); + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + return new FloatFirstVectorAggregator(timeSelector, valueSelector); + } + VectorObjectSelector objectSelector = columnSelectorFactory.makeObjectSelector(fieldName); + return new FloatFirstVectorAggregator(timeSelector, objectSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java index 44ddf24c0319..82a679095946 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregator.java @@ -29,9 +29,14 @@ public class FloatFirstVectorAggregator extends NumericFirstVectorAggregator { - public FloatFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) + public FloatFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { - super(timeSelector, valueSelector, SerializablePairLongFloat.class); + super(timeSelector, null, objectSelector); + } + + public FloatFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector, null); } @Override @@ -41,9 +46,9 @@ public void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position, Number number) + void putValue(ByteBuffer buf, int position, int index) { - float firstValue = number.floatValue(); + float firstValue = valueSelector != null ? valueSelector.getFloatVector()[index] : ((SerializablePairLongFloat) objectSelector.getObjectVector()[index]).getRhs(); buf.putFloat(position, firstValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index ce85f4a2bcc3..6b8836050ef5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -37,8 +37,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -145,8 +147,13 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - return new LongFirstVectorAggregator(timeSelector, vSelector); + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + return new LongFirstVectorAggregator(timeSelector, valueSelector); + } + VectorObjectSelector objectSelector = columnSelectorFactory.makeObjectSelector(fieldName); + return new LongFirstVectorAggregator(timeSelector, objectSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java index 961f124c5df3..8c7631db3c12 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregator.java @@ -28,9 +28,14 @@ public class LongFirstVectorAggregator extends NumericFirstVectorAggregator { - public LongFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) + public LongFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { - super(timeSelector, valueSelector, SerializablePairLongLong.class); + super(timeSelector, null, objectSelector); + } + + public LongFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector, null); } @Override @@ -40,9 +45,9 @@ public void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position, Number number) + void putValue(ByteBuffer buf, int position, int index) { - long firstValue = number.longValue(); + long firstValue = valueSelector != null ? valueSelector.getLongVector()[index] : ((SerializablePairLongLong) objectSelector.getObjectVector()[index]).getRhs(); buf.putLong(position, firstValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java index 72da1c687c5f..b487ac817d54 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java @@ -35,17 +35,17 @@ public abstract class NumericFirstVectorAggregator implements VectorAggregator { static final int NULL_OFFSET = Long.BYTES; static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; - final VectorObjectSelector valueSelector; - private final Class pairClass; + final VectorObjectSelector objectSelector; + final VectorValueSelector valueSelector; private final boolean useDefault = NullHandling.replaceWithDefault(); private final VectorValueSelector timeSelector; private long firstTime; - NumericFirstVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector, Class pairClass) + NumericFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector, VectorObjectSelector objectSelector) { this.timeSelector = timeSelector; + this.objectSelector = objectSelector; this.valueSelector = valueSelector; - this.pairClass = pairClass; firstTime = Long.MAX_VALUE; } @@ -62,8 +62,16 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) { final long[] timeVector = timeSelector.getLongVector(); final boolean[] nullTimeVector = timeSelector.getNullVector(); - final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); - final boolean[] nullValueVector = FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); + + Object[] objectsWhichMightBeNumeric = null; + boolean[] nullValueVector = null; + + if (objectSelector != null) { + objectsWhichMightBeNumeric = objectSelector.getObjectVector(); + } else if (valueSelector != null) { + nullValueVector = valueSelector.getNullVector(); + } + firstTime = buf.getLong(position); // the time vector is already sorted @@ -85,14 +93,12 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) } index = i; - final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeNumeric[index], pairClass); - if (foldNeeded) { + if (objectsWhichMightBeNumeric != null) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; - if (inPair.lhs != null && inPair.lhs < firstTime) { firstTime = inPair.lhs; if (useDefault || inPair.rhs != null) { - updateTimeWithValue(buf, position, firstTime, inPair.getRhs()); + updateTimeWithValue(buf, position, firstTime, index); } else { updateTimeWithNull(buf, position, firstTime); } @@ -102,9 +108,8 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) if (earliestTime < firstTime) { firstTime = earliestTime; - if (useDefault || nullValueVector == null || !nullValueVector[index]) { - updateTimeWithValue(buf, position, earliestTime, (Number) objectsWhichMightBeNumeric[index]); + updateTimeWithValue(buf, position, earliestTime, index); } else { updateTimeWithNull(buf, position, earliestTime); } @@ -136,18 +141,13 @@ public void aggregate( ) { final long[] timeVector = timeSelector.getLongVector(); - final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); - final boolean[] nullValueVector = FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); - boolean[] nulls = useDefault ? null : nullValueVector; - - // iterate once over the object vector to find first non null element and - // determine if the type is Pair or not - boolean foldNeeded = false; - for (Object obj : objectsWhichMightBeNumeric) { - if (obj != null) { - foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, pairClass); - break; - } + + Object[] objectsWhichMightBeNumeric = null; + boolean[] nulls = null; + if (objectSelector != null) { + objectsWhichMightBeNumeric = objectSelector.getObjectVector(); + } else if (valueSelector != null) { + nulls = useDefault ? null : valueSelector.getNullVector(); } for (int i = 0; i < numRows; i++) { @@ -155,12 +155,12 @@ public void aggregate( int row = rows == null ? i : rows[i]; firstTime = buf.getLong(position); - if (foldNeeded) { + if (objectsWhichMightBeNumeric != null) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; if (useDefault || inPair != null) { if (inPair.lhs != null && inPair.lhs < firstTime) { if (inPair.rhs != null) { - updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + updateTimeWithValue(buf, position, inPair.lhs, row); } else { updateTimeWithNull(buf, position, inPair.lhs); } @@ -168,8 +168,8 @@ public void aggregate( } } else { if (timeVector[row] < firstTime) { - if (useDefault || nulls == null || objectsWhichMightBeNumeric[row] != null) { - updateTimeWithValue(buf, position, timeVector[row], (Number) objectsWhichMightBeNumeric[row]); + if (useDefault || nulls == null || !nulls[row]) { + updateTimeWithValue(buf, position, timeVector[row], row); } else { updateTimeWithNull(buf, position, timeVector[row]); } @@ -184,13 +184,13 @@ public void aggregate( * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the current aggregate value is stored * @param time the time to be updated in the buffer as the first time - * @param number number which is the first value + * @param index he index of the vectorized vector which is the first value */ - void updateTimeWithValue(ByteBuffer buf, int position, long time, Number number) + void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) { buf.putLong(position, time); buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, number); + putValue(buf, position + VALUE_OFFSET, index); } /** @@ -215,7 +215,7 @@ void updateTimeWithNull(ByteBuffer buf, int position, long time) * Abstract function which needs to be overridden by subclasses to set the * latest value in the buffer depending on the datatype */ - abstract void putValue(ByteBuffer buf, int position, Number number); + abstract void putValue(ByteBuffer buf, int position, int index); @Override public void close() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregateCombiner.java deleted file mode 100644 index 4ccf92e2b381..000000000000 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregateCombiner.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation.first; - -import org.apache.druid.query.aggregation.ObjectAggregateCombiner; -import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.segment.ColumnValueSelector; - -import javax.annotation.Nullable; - -public class StringFirstAggregateCombiner extends ObjectAggregateCombiner -{ - private SerializablePairLongString firstValue; - - @Override - public void reset(ColumnValueSelector selector) - { - firstValue = (SerializablePairLongString) selector.getObject(); - } - - @Override - public void fold(ColumnValueSelector selector) - { - SerializablePairLongString newValue = (SerializablePairLongString) selector.getObject(); - if (StringFirstAggregatorFactory.TIME_COMPARATOR.compare(firstValue, newValue) > 0) { - firstValue = newValue; - } - } - - @Nullable - @Override - public SerializablePairLongString getObject() - { - return firstValue; - } - - @Override - public Class classOfObject() - { - return SerializablePairLongString.class; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index e60121e6daf0..56b58a33896e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -39,8 +39,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -134,8 +136,13 @@ public VectorAggregator factorizeVector( ) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - return new DoubleLastVectorAggregator(timeSelector, vSelector); + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + return new DoubleLastVectorAggregator(timeSelector, valueSelector); + } + VectorObjectSelector objectSelector = columnSelectorFactory.makeObjectSelector(fieldName); + return new DoubleLastVectorAggregator(timeSelector, objectSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java index ab02d8e86d22..d40ba750476b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java @@ -33,16 +33,22 @@ public class DoubleLastVectorAggregator extends NumericLastVectorAggregator { double lastValue; - public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) + public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { - super(timeSelector, valueSelector, SerializablePairLongDouble.class); + super(timeSelector, null, objectSelector); + lastValue = 0; + } + + public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector, null); lastValue = 0; } @Override - void putValue(ByteBuffer buf, int position, Number number) + void putValue(ByteBuffer buf, int position, int index) { - lastValue = number.doubleValue(); + lastValue = valueSelector != null ? valueSelector.getDoubleVector()[index] : ((SerializablePairLongDouble) objectSelector.getObjectVector()[index]).getRhs(); buf.putDouble(position, lastValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index c35138f35737..0eefc4586452 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -39,8 +39,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -151,8 +153,13 @@ public VectorAggregator factorizeVector( ) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - return new FloatLastVectorAggregator(timeSelector, vSelector); + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + return new FloatLastVectorAggregator(timeSelector, valueSelector); + } + VectorObjectSelector objectSelector = columnSelectorFactory.makeObjectSelector(fieldName); + return new FloatLastVectorAggregator(timeSelector, objectSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java index c9bde26266c9..a5500c19cbe8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java @@ -33,16 +33,22 @@ public class FloatLastVectorAggregator extends NumericLastVectorAggregator { float lastValue; - public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) + public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { - super(timeSelector, valueSelector, SerializablePairLongFloat.class); + super(timeSelector, null, objectSelector); + lastValue = 0; + } + + public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector, null); lastValue = 0; } @Override - void putValue(ByteBuffer buf, int position, Number number) + void putValue(ByteBuffer buf, int position, int index) { - lastValue = number.floatValue(); + lastValue = valueSelector != null ? valueSelector.getFloatVector()[index] : ((SerializablePairLongFloat) objectSelector.getObjectVector()[index]).getRhs(); buf.putFloat(position, lastValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 222ef70e3389..492df2136492 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -39,8 +39,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.Types; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -152,8 +154,13 @@ public VectorAggregator factorizeVector( ) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); - VectorObjectSelector vSelector = columnSelectorFactory.makeObjectSelector(fieldName); - return new LongLastVectorAggregator(timeSelector, vSelector); + ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); + if (Types.isNumeric(capabilities)) { + VectorValueSelector valueSelector = columnSelectorFactory.makeValueSelector(fieldName); + return new LongLastVectorAggregator(timeSelector, valueSelector); + } + VectorObjectSelector objectSelector = columnSelectorFactory.makeObjectSelector(fieldName); + return new LongLastVectorAggregator(timeSelector, objectSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java index c0446e9e8e32..65fadf549a76 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java @@ -33,9 +33,15 @@ public class LongLastVectorAggregator extends NumericLastVectorAggregator { long lastValue; - public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector) + public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { - super(timeSelector, valueSelector, SerializablePairLongLong.class); + super(timeSelector, null, objectSelector); + lastValue = 0; + } + + public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + { + super(timeSelector, valueSelector, null); lastValue = 0; } @@ -46,9 +52,9 @@ public void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position, Number number) + void putValue(ByteBuffer buf, int position, int index) { - lastValue = number.longValue(); + lastValue = valueSelector != null ? valueSelector.getLongVector()[index] : ((SerializablePairLongLong) objectSelector.getObjectVector()[index]).getRhs(); buf.putLong(position, lastValue); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java index a758ca3fd29b..ff5fa3af9e98 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java @@ -22,7 +22,6 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -36,18 +35,18 @@ public abstract class NumericLastVectorAggregator implements VectorAggregator { static final int NULL_OFFSET = Long.BYTES; static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; - final VectorObjectSelector valueSelector; - private final Class pairClass; + final VectorValueSelector valueSelector; + final VectorObjectSelector objectSelector; private final boolean useDefault = NullHandling.replaceWithDefault(); private final VectorValueSelector timeSelector; private long lastTime; - NumericLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector valueSelector, Class pairClass) + NumericLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector, VectorObjectSelector objectSelector) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; - this.pairClass = pairClass; + this.objectSelector = objectSelector; lastTime = Long.MIN_VALUE; } @@ -67,10 +66,16 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) } final long[] timeVector = timeSelector.getLongVector(); - final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); - final boolean[] nullValueVector = FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); - + Object[] objectsWhichMightBeNumeric = null; + boolean[] nullValueVector = null; boolean nullAbsent = false; + + if (objectSelector != null) { + objectsWhichMightBeNumeric = objectSelector.getObjectVector(); + } else if (valueSelector != null) { + nullValueVector = valueSelector.getNullVector(); + } + lastTime = buf.getLong(position); if (nullValueVector == null) { @@ -89,13 +94,12 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) } } - final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeNumeric[index], pairClass); - if (foldNeeded) { + if (objectsWhichMightBeNumeric != null) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; if (inPair.lhs != null && inPair.lhs >= lastTime) { lastTime = inPair.lhs; if (useDefault || inPair.rhs != null) { - updateTimeWithValue(buf, position, lastTime, inPair.getRhs()); + updateTimeWithValue(buf, position, lastTime, index); } else { updateTimeWithNull(buf, position, lastTime); } @@ -105,7 +109,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) if (latestTime >= lastTime) { lastTime = latestTime; if (useDefault || nullValueVector == null || !nullValueVector[index]) { - updateTimeWithValue(buf, position, lastTime, (Number) objectsWhichMightBeNumeric[index]); + updateTimeWithValue(buf, position, lastTime, index); } else { updateTimeWithNull(buf, position, lastTime); } @@ -140,29 +144,28 @@ public void aggregate( } final long[] timeVector = timeSelector.getLongVector(); + Object[] objectsWhichMightBeNumeric = null; + boolean[] nulls = null; - final Object[] objectsWhichMightBeNumeric = valueSelector.getObjectVector(); - boolean[] nulls = useDefault ? null : FirstLastUtils.getNullVector(objectsWhichMightBeNumeric); - - boolean foldNeeded = false; - for (Object obj : objectsWhichMightBeNumeric) { - if (obj != null) { - foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, pairClass); - break; - } + if (objectSelector != null) { + objectsWhichMightBeNumeric = objectSelector.getObjectVector(); + } else if (valueSelector != null) { + nulls = useDefault ? null : valueSelector.getNullVector(); } + + for (int i = 0; i < numRows; i++) { int position = positions[i] + positionOffset; int row = rows == null ? i : rows[i]; long lastTime = buf.getLong(position); - if (foldNeeded) { + if (objectsWhichMightBeNumeric != null) { final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; if (useDefault || inPair != null) { if (inPair.lhs != null && inPair.lhs >= lastTime) { if (inPair.rhs != null) { - updateTimeWithValue(buf, position, inPair.lhs, inPair.rhs); + updateTimeWithValue(buf, position, inPair.lhs, row); } else { updateTimeWithNull(buf, position, inPair.lhs); } @@ -171,7 +174,7 @@ public void aggregate( } else { if (timeVector[row] >= lastTime) { if (useDefault || nulls == null || !nulls[row]) { - updateTimeWithValue(buf, position, timeVector[row], (Number) objectsWhichMightBeNumeric[row]); + updateTimeWithValue(buf, position, timeVector[row], row); } else { updateTimeWithNull(buf, position, timeVector[row]); } @@ -185,13 +188,13 @@ public void aggregate( * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the current aggregate value is stored * @param time the time to be updated in the buffer as the last time - * @param number number which is the last value + * @param index he index of the vectorized vector which is the last value */ - void updateTimeWithValue(ByteBuffer buf, int position, long time, Number number) + void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) { buf.putLong(position, time); buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, number); + putValue(buf, position + VALUE_OFFSET, index); } /** @@ -215,7 +218,7 @@ void updateTimeWithNull(ByteBuffer buf, int position, long time) *Abstract function which needs to be overridden by subclasses to set the * latest value in the buffer depending on the datatype */ - abstract void putValue(ByteBuffer buf, int position, Number number); + abstract void putValue(ByteBuffer buf, int position, int index); @Override public void close() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregateCombiner.java deleted file mode 100644 index 47f21190395f..000000000000 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregateCombiner.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation.last; - -import org.apache.druid.query.aggregation.ObjectAggregateCombiner; -import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; -import org.apache.druid.segment.ColumnValueSelector; - -import javax.annotation.Nullable; - -public class StringLastAggregateCombiner extends ObjectAggregateCombiner -{ - private SerializablePairLongString lastValue; - - @Override - public void reset(ColumnValueSelector selector) - { - lastValue = (SerializablePairLongString) selector.getObject(); - } - - @Override - public void fold(ColumnValueSelector selector) - { - SerializablePairLongString newValue = (SerializablePairLongString) selector.getObject(); - if (StringFirstAggregatorFactory.TIME_COMPARATOR.compare(lastValue, newValue) < 0) { - lastValue = (SerializablePairLongString) selector.getObject(); - } - } - - @Nullable - @Override - public SerializablePairLongString getObject() - { - return lastValue; - } - - @Override - public Class classOfObject() - { - return SerializablePairLongString.class; - } -} From edd7a8845419792810076f57076a038f115edbd4 Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Thu, 7 Dec 2023 11:05:23 -0800 Subject: [PATCH 13/14] intellij-inspections failure fix --- .../aggregation/any/NilVectorAggregator.java | 5 ----- .../aggregation/first/FirstLastUtils.java | 21 ------------------- 2 files changed, 26 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java index 67d11d820a64..b4468813f529 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/NilVectorAggregator.java @@ -69,11 +69,6 @@ public static NilVectorAggregator longNilVectorAggregator() @Nullable private final Object returnValue; - public static NilVectorAggregator of(Object returnValue) - { - return new NilVectorAggregator(returnValue); - } - private NilVectorAggregator(@Nullable Object returnValue) { this.returnValue = returnValue; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java index 2b2ceb389e85..f6ae31b6ccc6 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -66,25 +66,4 @@ public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) return clazz.isAssignableFrom(pairClass) || pairClass.isAssignableFrom(clazz); } - - - /** - * Returns boolean array indicating null values for the given objectVector. - * Returns null is no null objects are found in objectVector - */ - @Nullable - public static boolean[] getNullVector(Object[] objectVector) - { - boolean containsNullValues = false; - boolean[] nullValueVector = new boolean[objectVector.length]; - for (int i = 0; i < objectVector.length; i++) { - if (objectVector[i] != null) { - nullValueVector[i] = false; - } else { - nullValueVector[i] = true; - containsNullValues = true; - } - } - return containsNullValues ? nullValueVector : null; - } } From 3397717d6b3b103bf054027ce3c1df054b93176e Mon Sep 17 00:00:00 2001 From: Ankit Kothari Date: Thu, 7 Dec 2023 22:12:53 -0800 Subject: [PATCH 14/14] addressing comments + extending LongString to the abstract classes --- ...stractSerializableLongObjectPairSerde.java | 3 + ...SerializablePairLongObjectBufferStore.java | 23 ++++ ...erializablePairLongObjectColumnHeader.java | 5 + ...lizablePairLongObjectColumnSerializer.java | 9 ++ ...PairLongObjectDeltaEncodedStagedSerde.java | 8 ++ ...izablePairLongObjectSimpleStagedSerde.java | 10 +- ...SerializablePairLongDoubleBufferStore.java | 3 - ...zablePairLongDoubleComplexMetricSerde.java | 2 +- .../SerializablePairLongFloatBufferStore.java | 3 - ...izablePairLongFloatComplexMetricSerde.java | 2 +- .../SerializablePairLongLongBufferStore.java | 3 - ...lizablePairLongLongComplexMetricSerde.java | 2 +- ...SerializablePairLongStringBufferStore.java | 112 ++---------------- ...erializablePairLongStringColumnHeader.java | 82 +------------ ...lizablePairLongStringColumnSerializer.java | 63 +--------- ...rializablePairLongStringComplexColumn.java | 2 +- ...PairLongStringDeltaEncodedStagedSerde.java | 9 +- ...izablePairLongStringSimpleStagedSerde.java | 9 +- .../aggregation/first/FirstLastUtils.java | 4 +- .../last/GenericLastAggregateCombiner.java | 2 +- ...alizablePairLongStringBufferStoreTest.java | 26 ++-- 21 files changed, 100 insertions(+), 282 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java index 049dc4a4f6db..75274555cf3c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java @@ -26,6 +26,9 @@ import javax.annotation.Nullable; +// To maintain parity with the longString pair serde, +// we have made EXPECTED_VERSION as 3 because it uses the same delta & block encoding technique. +// 0,1,2 versions for numeric serdes do not exist. public abstract class AbstractSerializableLongObjectPairSerde> extends ComplexMetricSerde { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java index e5808b444044..71cef886c67f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectBufferStore.java @@ -27,6 +27,7 @@ import org.apache.druid.segment.serde.cell.IOIterator; import org.apache.druid.segment.writeout.SegmentWriteOutMedium; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.nio.channels.WritableByteChannel; @@ -53,6 +54,15 @@ public void store(@Nullable T pairLongObject) throws IOException serializedStorage.store(pairLongObject); } + /** + * each call transfers the temporary buffer into an encoded, block-compessed buffer of the segment. It is ready to be + * transferred to a {@link WritableByteChannel} + * + * @param byteBufferProvider - provides a ByteBuffer used for block compressed encoding + * @param segmentWriteOutMedium - used to create temporary storage + * @return encoded buffer ready to be stored + * @throws IOException + */ public TransferredBuffer transferToRowWriter( ByteBufferProvider byteBufferProvider, SegmentWriteOutMedium segmentWriteOutMedium @@ -61,6 +71,11 @@ public TransferredBuffer transferToRowWriter( AbstractSerializablePairLongObjectColumnHeader columnHeader = createColumnHeader(); AbstractSerializablePairLongObjectDeltaEncodedStagedSerde deltaEncodedSerde = createDeltaEncodedSerde(columnHeader); + // try-with-resources will call cellWriter.close() an extra time in the normal case, but it protects against + // buffer leaking in the case of an exception (close() is idempotent). In the normal path, close() performs some + // finalization of the CellWriter object. We want that object state finalized before creating the TransferredBuffer + // as a point of good style (though strictly speaking, it works fine to pass it in before calling close since + // TransferredBuffer does not do anything in the constructor with the object) try (CellWriter cellWriter = new CellWriter.Builder(segmentWriteOutMedium).setByteBufferProvider(byteBufferProvider) .build()) { try (IOIterator bufferIterator = iterator()) { @@ -78,6 +93,10 @@ public TransferredBuffer transferToRowWriter( } } + // 1. we have overflow in our range || 2. we have only seen null values + // in this case, effectively disable delta encoding by using longs and a min value of 0 + // else we shoudl return columnHeader with delta encding enabled + @Nonnull public abstract AbstractSerializablePairLongObjectColumnHeader createColumnHeader(); public abstract AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createDeltaEncodedSerde(AbstractSerializablePairLongObjectColumnHeader columnHeader); @@ -86,6 +105,10 @@ public IOIterator iterator() throws IOException return serializedStorage.iterator(); } + /** + * contains serialized data that is compressed and delta-encoded (Long) + * It's ready to be transferred to a {@link WritableByteChannel} + */ public static class TransferredBuffer implements Serializer { private final CellWriter cellWriter; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java index a9839460d9eb..677059adb680 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnHeader.java @@ -32,6 +32,7 @@ public abstract class AbstractSerializablePairLongObjectColumnHeader> { + // header size is 4 bytes for word alignment for LZ4 (minmatch) compression private static final int HEADER_SIZE_BYTES = 4; private static final int USE_INTEGER_MASK = 0x80; private static final int VERSION_INDEX = 0; @@ -76,6 +77,10 @@ static AbstractSerializablePairLongObjectColumnHeader fromBuffer(ByteBuffer b return new SerializablePairLongFloatColumnHeader(bytes, minTimestamp); } + if (pairClass.isAssignableFrom(SerializablePairLongString.class)) { + return new SerializablePairLongStringColumnHeader(bytes, minTimestamp); + } + throw new RE(String.format(Locale.ENGLISH, "Unsupported pairClass type: %s", pairClass.getSimpleName())); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java index 21ba824f9e26..6ef61946db8e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectColumnSerializer.java @@ -31,6 +31,15 @@ import java.io.IOException; import java.nio.channels.WritableByteChannel; +/** + * valid call sequence + *

+ * open()+serialize()*(getSerializedSize()|writeTo())* + *

+ * getSerializedSize() / writeTo() effectively function as a close call, but each may be called multiple times and has + * no effect on one another. + */ +@SuppressWarnings("NotNullFieldNotInitialized") public abstract class AbstractSerializablePairLongObjectColumnSerializer> implements GenericColumnSerializer { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java index 62b0a52642e7..f276cd7b67f3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectDeltaEncodedStagedSerde.java @@ -30,6 +30,14 @@ import java.nio.ByteBuffer; import java.util.Locale; +/** + * serializes a Long/Object(Number) pair in the context of a column/segment. Uses the minValue to perform delta + * encoding/decoding and if the range of the segment fits in an integer (useIntegerDelta), the format is + * Integer:Byte:Integer + * + * otherwise + * Long:Integer:bytes + */ public abstract class AbstractSerializablePairLongObjectDeltaEncodedStagedSerde> implements StagedSerde { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java index cf43cbc34c5c..8a7857163e46 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairLongObjectSimpleStagedSerde.java @@ -27,8 +27,14 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.Locale; +/** + * serializes a Long/Object pair as + * Long:Byte:Object + *

+ * or + * Long:isNullByte:ObjectBytes + */ public abstract class AbstractSerializablePairLongObjectSimpleStagedSerde> implements StagedSerde { @@ -55,7 +61,7 @@ public StorableBuffer serializeDelayed( @Override public void store(ByteBuffer byteBuffer) { - Preconditions.checkNotNull(value.getLhs(), String.format(Locale.ENGLISH, "Long in %s must be non-null", pairCLass.getSimpleName())); + Preconditions.checkNotNull(value.getLhs(), "Long in %s must be non-null", pairCLass.getSimpleName()); byteBuffer.putLong(value.getLhs()); if (rhsObject != null) { byteBuffer.put(NullHandling.IS_NOT_NULL_BYTE); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java index c5d84f62fc43..d4063d0fff80 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleBufferStore.java @@ -37,9 +37,6 @@ public AbstractSerializablePairLongObjectColumnHeader maxValue) { - // true iff - // 1. we have overflow in our range || 2. we have only seen null values - // in this case, effectively disable delta encoding by using longs and a min value of 0 maxDelta = Long.MAX_VALUE; minValue = 0; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java index 0a40377b3cd5..7f8befd09275 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleComplexMetricSerde.java @@ -32,7 +32,7 @@ public class SerializablePairLongDoubleComplexMetricSerde extends AbstractSerializableLongObjectPairSerde { - public static final int EXPECTED_VERSION = 1; + public static final int EXPECTED_VERSION = 3; public static final String TYPE_NAME = "serializablePairLongDouble"; private static final SerializablePairLongDoubleSimpleStagedSerde SERDE = new SerializablePairLongDoubleSimpleStagedSerde(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java index 1314d1987dc8..7c3e9d62f325 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatBufferStore.java @@ -37,9 +37,6 @@ public AbstractSerializablePairLongObjectColumnHeader SerializablePairLongFloatColumnHeader columnHeader; if (minValue < maxValue && maxDelta < 0 || minValue > maxValue) { - // true iff - // 1. we have overflow in our range || 2. we have only seen null values - // in this case, effectively disable delta encoding by using longs and a min value of 0 maxDelta = Long.MAX_VALUE; minValue = 0; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java index 7578d06c275e..d342608a9e15 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatComplexMetricSerde.java @@ -32,7 +32,7 @@ public class SerializablePairLongFloatComplexMetricSerde extends AbstractSerializableLongObjectPairSerde { - public static final int EXPECTED_VERSION = 1; + public static final int EXPECTED_VERSION = 3; public static final String TYPE_NAME = "serializablePairLongFloat"; private static final SerializablePairLongFloatSimpleStagedSerde SERDE = new SerializablePairLongFloatSimpleStagedSerde(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java index fd9efe668da7..a249d8b9bcee 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongBufferStore.java @@ -36,9 +36,6 @@ public AbstractSerializablePairLongObjectColumnHeader SerializablePairLongLongColumnHeader columnHeader; if (minValue < maxValue && maxDelta < 0 || minValue > maxValue) { - // true iff - // 1. we have overflow in our range || 2. we have only seen null values - // in this case, effectively disable delta encoding by using longs and a min value of 0 maxDelta = Long.MAX_VALUE; minValue = 0; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java index 3914e38d2660..7541dfdb2a15 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongComplexMetricSerde.java @@ -32,7 +32,7 @@ public class SerializablePairLongLongComplexMetricSerde extends AbstractSerializableLongObjectPairSerde { - public static final int EXPECTED_VERSION = 1; + public static final int EXPECTED_VERSION = 3; public static final String TYPE_NAME = "serializablePairLongLong"; private static final SerializablePairLongLongSimpleStagedSerde SERDE = new SerializablePairLongLongSimpleStagedSerde(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStore.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStore.java index 843e1a4eff0f..59ef383446f7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStore.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStore.java @@ -19,93 +19,23 @@ package org.apache.druid.query.aggregation; -import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; -import org.apache.druid.segment.serde.Serializer; -import org.apache.druid.segment.serde.cell.ByteBufferProvider; -import org.apache.druid.segment.serde.cell.CellWriter; -import org.apache.druid.segment.serde.cell.IOIterator; -import org.apache.druid.segment.writeout.SegmentWriteOutMedium; - import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.IOException; -import java.nio.channels.WritableByteChannel; -public class SerializablePairLongStringBufferStore +public class SerializablePairLongStringBufferStore extends AbstractSerializablePairLongObjectBufferStore { - private final SerializedStorage serializedStorage; - - private long minValue = Long.MAX_VALUE; - private long maxValue = Long.MIN_VALUE; - public SerializablePairLongStringBufferStore(SerializedStorage serializedStorage) { - this.serializedStorage = serializedStorage; - } - - public void store(@Nullable SerializablePairLongString pairLongString) throws IOException - { - if (pairLongString != null && pairLongString.lhs != null) { - minValue = Math.min(minValue, pairLongString.lhs); - maxValue = Math.max(maxValue, pairLongString.lhs); - } - - serializedStorage.store(pairLongString); - } - - /** - * each call transfers the temporary buffer into an encoded, block-compessed buffer of the segment. It is ready to be - * transferred to a {@link WritableByteChannel} - * - * @param byteBufferProvider - provides a ByteBuffer used for block compressed encoding - * @param segmentWriteOutMedium - used to create temporary storage - * @return encoded buffer ready to be stored - * @throws IOException - */ - public TransferredBuffer transferToRowWriter( - ByteBufferProvider byteBufferProvider, - SegmentWriteOutMedium segmentWriteOutMedium - ) throws IOException - { - SerializablePairLongStringColumnHeader columnHeader = createColumnHeader(); - SerializablePairLongStringDeltaEncodedStagedSerde serde = - new SerializablePairLongStringDeltaEncodedStagedSerde( - columnHeader.getMinValue(), - columnHeader.isUseIntegerDeltas() - ); - - // try-with-resources will call cellWriter.close() an extra time in the normal case, but it protects against - // buffer leaking in the case of an exception (close() is idempotent). In the normal path, close() performs some - // finalization of the CellWriter object. We want that object state finalized before creating the TransferredBuffer - // as a point of good style (though strictly speaking, it works fine to pass it in before calling close since - // TransferredBuffer does not do anything in the constructor with the object) - try (CellWriter cellWriter = - new CellWriter.Builder(segmentWriteOutMedium).setByteBufferProvider(byteBufferProvider).build()) { - try (IOIterator bufferIterator = iterator()) { - while (bufferIterator.hasNext()) { - SerializablePairLongString pairLongString = bufferIterator.next(); - byte[] serialized = serde.serialize(pairLongString); - - cellWriter.write(serialized); - } - - cellWriter.close(); - - return new TransferredBuffer(cellWriter, columnHeader); - } - } + super(serializedStorage); } + @Override @Nonnull - public SerializablePairLongStringColumnHeader createColumnHeader() + public AbstractSerializablePairLongObjectColumnHeader createColumnHeader() { long maxDelta = maxValue - minValue; SerializablePairLongStringColumnHeader columnHeader; if (minValue < maxValue && maxDelta < 0 || minValue > maxValue) { - // true iff - // 1. we have overflow in our range || 2. we have only seen null values - // in this case, effectively disable delta encoding by using longs and a min value of 0 maxDelta = Long.MAX_VALUE; minValue = 0; } @@ -126,37 +56,9 @@ public SerializablePairLongStringColumnHeader createColumnHeader() return columnHeader; } - public IOIterator iterator() throws IOException - { - return serializedStorage.iterator(); - } - - /** - * contains serialized data that is compressed and delta-encoded (Long) - * It's ready to be transferred to a {@link WritableByteChannel} - */ - public static class TransferredBuffer implements Serializer + @Override + public AbstractSerializablePairLongObjectDeltaEncodedStagedSerde createDeltaEncodedSerde(AbstractSerializablePairLongObjectColumnHeader columnHeader) { - private final CellWriter cellWriter; - private final SerializablePairLongStringColumnHeader columnHeader; - - public TransferredBuffer(CellWriter cellWriter, SerializablePairLongStringColumnHeader columnHeader) - { - this.cellWriter = cellWriter; - this.columnHeader = columnHeader; - } - - @Override - public void writeTo(WritableByteChannel channel, @Nullable FileSmoosher smoosher) throws IOException - { - columnHeader.transferTo(channel); - cellWriter.writeTo(channel, smoosher); - } - - @Override - public long getSerializedSize() - { - return columnHeader.getSerializedSize() + cellWriter.getSerializedSize(); - } + return new SerializablePairLongStringDeltaEncodedStagedSerde(minValue, columnHeader.isUseIntegerDeltas()); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnHeader.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnHeader.java index e9ad87caee5d..65983bcdb959 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnHeader.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnHeader.java @@ -19,93 +19,21 @@ package org.apache.druid.query.aggregation; -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; -import org.apache.druid.segment.serde.cell.LongSerializer; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -public class SerializablePairLongStringColumnHeader +public class SerializablePairLongStringColumnHeader extends AbstractSerializablePairLongObjectColumnHeader { - // header size is 4 bytes for word alignment for LZ4 (minmatch) compression - private static final int HEADER_SIZE_BYTES = 4; - private static final int USE_INTEGER_MASK = 0x80; - private static final int VERSION_INDEX = 0; - private static final int ENCODING_INDEX = 1; - - private final byte[] bytes; - private final long minValue; - - private SerializablePairLongStringColumnHeader(byte[] bytes, long minTimestamp) + SerializablePairLongStringColumnHeader(byte[] bytes, long minTimestamp) { - this.bytes = bytes; - this.minValue = minTimestamp; + super(bytes, minTimestamp); } public SerializablePairLongStringColumnHeader(int version, boolean useIntegerDeltas, long minTimestamp) { - this.minValue = minTimestamp; - bytes = new byte[HEADER_SIZE_BYTES]; - Preconditions.checkArgument(version <= 255, "max version 255"); - bytes[VERSION_INDEX] = (byte) version; - - if (useIntegerDeltas) { - bytes[ENCODING_INDEX] |= USE_INTEGER_MASK; - } - } - - public static SerializablePairLongStringColumnHeader fromBuffer(ByteBuffer byteBuffer) - { - byte[] bytes = new byte[HEADER_SIZE_BYTES]; - - byteBuffer.get(bytes); - - long minTimestamp = byteBuffer.getLong(); - - return new SerializablePairLongStringColumnHeader(bytes, minTimestamp); + super(version, useIntegerDeltas, minTimestamp); } + @Override public SerializablePairLongStringDeltaEncodedStagedSerde createSerde() { return new SerializablePairLongStringDeltaEncodedStagedSerde(minValue, isUseIntegerDeltas()); } - - public void transferTo(WritableByteChannel channel) throws IOException - { - LongSerializer longSerializer = new LongSerializer(); - - channel.write(ByteBuffer.wrap(bytes)); - channel.write(longSerializer.serialize(minValue)); - } - - public int getVersion() - { - return 0XFF & bytes[VERSION_INDEX]; - } - - public boolean isUseIntegerDeltas() - { - return (bytes[ENCODING_INDEX] & USE_INTEGER_MASK) != 0; - } - - public long getMinValue() - { - return minValue; - } - - public int getSerializedSize() - { - return HEADER_SIZE_BYTES + Long.BYTES; - } - - @Override - public String toString() - { - return MoreObjects.toStringHelper(this) - .add("bytes", bytes) - .add("minValue", minValue) - .toString(); - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java index 5962aec928e5..d3b9900963ca 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringColumnSerializer.java @@ -20,16 +20,10 @@ package org.apache.druid.query.aggregation; import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.GenericColumnSerializer; import org.apache.druid.segment.serde.cell.ByteBufferProvider; -import org.apache.druid.segment.serde.cell.StagedSerde; import org.apache.druid.segment.writeout.SegmentWriteOutMedium; -import javax.annotation.Nullable; import java.io.IOException; -import java.nio.channels.WritableByteChannel; /** * valid call sequence @@ -39,26 +33,15 @@ * getSerializedSize() / writeTo() effectively function as a close call, but each may be called multiple times and has * no effect on one another. */ -@SuppressWarnings("NotNullFieldNotInitialized") -public class SerializablePairLongStringColumnSerializer implements GenericColumnSerializer +public class SerializablePairLongStringColumnSerializer extends AbstractSerializablePairLongObjectColumnSerializer { - public static final StagedSerde STAGED_SERDE = - new SerializablePairLongStringSimpleStagedSerde(); - - private final SegmentWriteOutMedium segmentWriteOutMedium; - private final ByteBufferProvider byteBufferProvider; - - private AbstractSerializablePairLongObjectColumnSerializer.State state = AbstractSerializablePairLongObjectColumnSerializer.State.START; - private SerializablePairLongStringBufferStore bufferStore; - private SerializablePairLongStringBufferStore.TransferredBuffer transferredBuffer; public SerializablePairLongStringColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, ByteBufferProvider byteBufferProvider ) { - this.segmentWriteOutMedium = segmentWriteOutMedium; - this.byteBufferProvider = byteBufferProvider; + super(new SerializablePairLongStringSimpleStagedSerde(), segmentWriteOutMedium, byteBufferProvider); } @Override @@ -68,49 +51,9 @@ public void open() throws IOException if (state == AbstractSerializablePairLongObjectColumnSerializer.State.START) { bufferStore = new SerializablePairLongStringBufferStore( - new SerializedStorage<>(segmentWriteOutMedium.makeWriteOutBytes(), STAGED_SERDE) + new SerializedStorage<>(segmentWriteOutMedium.makeWriteOutBytes(), stagedSerde) ); state = AbstractSerializablePairLongObjectColumnSerializer.State.OPEN; } } - - @Override - public void serialize(ColumnValueSelector selector) throws IOException - { - Preconditions.checkState(state == AbstractSerializablePairLongObjectColumnSerializer.State.OPEN, "serialize called in invalid state %s", state); - - SerializablePairLongString pairLongString = selector.getObject(); - - bufferStore.store(pairLongString); - } - - @Override - public long getSerializedSize() throws IOException - { - Preconditions.checkState( - state != AbstractSerializablePairLongObjectColumnSerializer.State.START, - "getSerializedSize called in invalid state %s (must have opened at least)", - state - ); - - transferToRowWriterIfNecessary(); - - return transferredBuffer.getSerializedSize(); - } - - @Override - public void writeTo(WritableByteChannel channel, @Nullable FileSmoosher smoosher) throws IOException - { - Preconditions.checkState(state != AbstractSerializablePairLongObjectColumnSerializer.State.START, "writeTo called in invalid state %s", state); - transferToRowWriterIfNecessary(); - transferredBuffer.writeTo(channel, smoosher); - } - - private void transferToRowWriterIfNecessary() throws IOException - { - if (state == AbstractSerializablePairLongObjectColumnSerializer.State.OPEN) { - transferredBuffer = bufferStore.transferToRowWriter(byteBufferProvider, segmentWriteOutMedium); - state = AbstractSerializablePairLongObjectColumnSerializer.State.CLOSED; - } - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java index d3b4dcfbb4e8..264dfc4a8534 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringComplexColumn.java @@ -101,7 +101,7 @@ public Builder(ByteBuffer buffer) serializedSize = masterByteBuffer.remaining(); SerializablePairLongStringColumnHeader columnHeader = - SerializablePairLongStringColumnHeader.fromBuffer(masterByteBuffer); + (SerializablePairLongStringColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(masterByteBuffer, SerializablePairLongString.class); Preconditions.checkArgument( columnHeader.getVersion() == SerializablePairLongStringComplexMetricSerde.EXPECTED_VERSION, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringDeltaEncodedStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringDeltaEncodedStagedSerde.java index e1c070127b71..5c188a0b9a80 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringDeltaEncodedStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringDeltaEncodedStagedSerde.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.segment.serde.cell.StagedSerde; import org.apache.druid.segment.serde.cell.StorableBuffer; import javax.annotation.Nullable; @@ -37,15 +36,11 @@ * otherwise * Long:Integer:bytes */ -public class SerializablePairLongStringDeltaEncodedStagedSerde implements StagedSerde +public class SerializablePairLongStringDeltaEncodedStagedSerde extends AbstractSerializablePairLongObjectDeltaEncodedStagedSerde { - private final long minValue; - private final boolean useIntegerDelta; - public SerializablePairLongStringDeltaEncodedStagedSerde(long minValue, boolean useIntegerDelta) { - this.minValue = minValue; - this.useIntegerDelta = useIntegerDelta; + super(minValue, useIntegerDelta, SerializablePairLongString.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSimpleStagedSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSimpleStagedSerde.java index acdd937d2aac..0e67b190bd05 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSimpleStagedSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSimpleStagedSerde.java @@ -21,7 +21,6 @@ import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.segment.serde.cell.StagedSerde; import org.apache.druid.segment.serde.cell.StorableBuffer; import javax.annotation.Nullable; @@ -35,8 +34,14 @@ * or * Long:StringSize:StringData */ -public class SerializablePairLongStringSimpleStagedSerde implements StagedSerde +public class SerializablePairLongStringSimpleStagedSerde extends AbstractSerializablePairLongObjectSimpleStagedSerde { + + public SerializablePairLongStringSimpleStagedSerde() + { + super(SerializablePairLongString.class); + } + @Override public StorableBuffer serializeDelayed(@Nullable SerializablePairLongString value) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java index f6ae31b6ccc6..b9ef88dfdcf9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -44,7 +44,7 @@ public static boolean selectorNeedsFoldCheck( } if (valueSelector instanceof NilColumnValueSelector) { - // Nil column, definitely no SerializablePairLongStrings. + // Nil column, definitely no SerializablePairLongObject. return false; } @@ -55,7 +55,7 @@ public static boolean selectorNeedsFoldCheck( } /** - * Returns whether an object *might* contain SerializablePairLongString objects. + * Returns whether an object *might* is assignable to/from the pairClass. */ public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java index 7ad091199df4..1b1e2dbd128b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -46,7 +46,7 @@ public void reset(ColumnValueSelector selector) public void fold(ColumnValueSelector selector) { T newValue = (T) selector.getObject(); - if (Longs.compare(lastValue.lhs, newValue.lhs) <= 0) { + if (Longs.compare(lastValue.lhs, newValue.lhs) < 0) { lastValue = newValue; } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStoreTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStoreTest.java index 8075e273c287..8b05f8a094c8 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStoreTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/SerializablePairLongStringBufferStoreTest.java @@ -69,7 +69,7 @@ public void setup() throws Exception bufferStore = new SerializablePairLongStringBufferStore( new SerializedStorage<>( writeOutMedium.makeWriteOutBytes(), - SerializablePairLongStringColumnSerializer.STAGED_SERDE + new SerializablePairLongStringSimpleStagedSerde() )); } @@ -139,7 +139,7 @@ public void testMinValueUsesInteger() throws Exception bufferStore.store(value); } - SerializablePairLongStringColumnHeader columnHeader = bufferStore.createColumnHeader(); + SerializablePairLongStringColumnHeader columnHeader = (SerializablePairLongStringColumnHeader) bufferStore.createColumnHeader(); Assert.assertEquals(integerRangeArr[0].lhs.longValue(), columnHeader.getMinValue()); Assert.assertTrue(columnHeader.isUseIntegerDeltas()); } @@ -151,7 +151,7 @@ public void testMinValueUsesLong() throws Exception bufferStore.store(value); } - SerializablePairLongStringColumnHeader columnHeader = bufferStore.createColumnHeader(); + SerializablePairLongStringColumnHeader columnHeader = (SerializablePairLongStringColumnHeader) bufferStore.createColumnHeader(); Assert.assertEquals(MIN_LONG, columnHeader.getMinValue()); Assert.assertFalse(columnHeader.isUseIntegerDeltas()); } @@ -163,7 +163,7 @@ public void testMinValueUsesIntegerSerialization() throws Exception bufferStore.store(value); } - SerializablePairLongStringColumnHeader columnHeader = bufferStore.createColumnHeader(); + SerializablePairLongStringColumnHeader columnHeader = (SerializablePairLongStringColumnHeader) bufferStore.createColumnHeader(); HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { @@ -174,7 +174,7 @@ public void testMinValueUsesIntegerSerialization() throws Exception byteBuffer.flip(); SerializablePairLongStringColumnHeader deserializedColumnhHeader = - SerializablePairLongStringColumnHeader.fromBuffer(byteBuffer); + (SerializablePairLongStringColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongString.class); Assert.assertEquals(MIN_INTEGER, deserializedColumnhHeader.getMinValue()); Assert.assertTrue(deserializedColumnhHeader.isUseIntegerDeltas()); } @@ -188,7 +188,7 @@ public void testMinValueSerialization() throws Exception bufferStore.store(value); } - SerializablePairLongStringColumnHeader columnHeader = bufferStore.createColumnHeader(); + SerializablePairLongStringColumnHeader columnHeader = (SerializablePairLongStringColumnHeader) bufferStore.createColumnHeader(); HeapByteBufferWriteOutBytes channel = new HeapByteBufferWriteOutBytes(); try (ResourceHolder resourceHolder = NativeClearedByteBufferProvider.INSTANCE.get()) { @@ -200,7 +200,7 @@ public void testMinValueSerialization() throws Exception byteBuffer.flip(); SerializablePairLongStringColumnHeader deserializedColumnhHeader = - SerializablePairLongStringColumnHeader.fromBuffer(byteBuffer); + (SerializablePairLongStringColumnHeader) AbstractSerializablePairLongObjectColumnHeader.fromBuffer(byteBuffer, SerializablePairLongString.class); Assert.assertEquals(MIN_LONG, deserializedColumnhHeader.getMinValue()); Assert.assertFalse(deserializedColumnhHeader.isUseIntegerDeltas()); } @@ -271,11 +271,11 @@ public void testOverflowTransfer() throws Exception bufferStore.store(new SerializablePairLongString(Long.MIN_VALUE, "fuu")); bufferStore.store(new SerializablePairLongString(Long.MAX_VALUE, "fuu")); - SerializablePairLongStringColumnHeader columnHeader = bufferStore.createColumnHeader(); + SerializablePairLongStringColumnHeader columnHeader = (SerializablePairLongStringColumnHeader) bufferStore.createColumnHeader(); Assert.assertEquals(0, columnHeader.getMinValue()); - SerializablePairLongStringBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( NativeClearedByteBufferProvider.INSTANCE, writeOutMedium ); @@ -292,11 +292,11 @@ public void testNullOnlyTransfer() throws Exception bufferStore.store(null); - SerializablePairLongStringColumnHeader columnHeader = bufferStore.createColumnHeader(); + SerializablePairLongStringColumnHeader columnHeader = (SerializablePairLongStringColumnHeader) bufferStore.createColumnHeader(); Assert.assertEquals(0, columnHeader.getMinValue()); - SerializablePairLongStringBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter( NativeClearedByteBufferProvider.INSTANCE, writeOutMedium ); @@ -351,7 +351,7 @@ private void assertBufferedValuesEqual(List input) t private void assertTransferredValuesEqual(SerializablePairLongString[] input) throws IOException { - SerializablePairLongStringBufferStore.TransferredBuffer transferredBuffer = + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer = bufferStore.transferToRowWriter(NativeClearedByteBufferProvider.INSTANCE, writeOutMedium); HeapByteBufferWriteOutBytes resultChannel = new HeapByteBufferWriteOutBytes(); @@ -365,7 +365,7 @@ private void assertTransferredValuesEqual(SerializablePairLongString[] input) th } private static SerializablePairLongStringComplexColumn createComplexColumn( - SerializablePairLongStringBufferStore.TransferredBuffer transferredBuffer, + AbstractSerializablePairLongObjectBufferStore.TransferredBuffer transferredBuffer, HeapByteBufferWriteOutBytes resultChannel ) {