From 8ce30621abda3458be22cf547220351ac2bd9037 Mon Sep 17 00:00:00 2001 From: Xiang Fu Date: Tue, 26 Sep 2023 16:01:19 -0700 Subject: [PATCH] fixing ValueReaderComparisonTest during to accidentally hitting surrogates (#11681) --- .../ValueReaderComparisonTest.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readerwriter/ValueReaderComparisonTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readerwriter/ValueReaderComparisonTest.java index 245c63c95343..f788bdb37821 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readerwriter/ValueReaderComparisonTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readerwriter/ValueReaderComparisonTest.java @@ -24,8 +24,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; @@ -44,15 +47,25 @@ public class ValueReaderComparisonTest { + // Number of rounds to run the test for, change this number to test locally for catching the corner cases. + private static final int NUM_ROUNDS = 1; + @DataProvider public static Object[] text() { - return Stream.of(Pair.of(ByteOrder.BIG_ENDIAN, true), Pair.of(ByteOrder.LITTLE_ENDIAN, true), - // there is no little endian support for var length at the time of writing - Pair.of(ByteOrder.BIG_ENDIAN, false)).flatMap( - pair -> Stream.of(new AsciiTestCase(pair.getLeft(), pair.getRight()), - new Utf8TestCase(pair.getLeft(), pair.getRight()), - new RandomBytesTextTestCase(pair.getLeft(), pair.getRight()), - new OrderedInvalidUtf8TestCase(pair.getLeft(), pair.getRight()))).toArray(Object[]::new); + return Collections.nCopies(NUM_ROUNDS, + Stream.of( + Pair.of(ByteOrder.BIG_ENDIAN, true), + Pair.of(ByteOrder.LITTLE_ENDIAN, true), + Pair.of(ByteOrder.BIG_ENDIAN, false)) + .flatMap( + pair -> Stream.of( + new AsciiTestCase(pair.getLeft(), pair.getRight()), + new Utf8TestCase(pair.getLeft(), pair.getRight()), + new RandomBytesTextTestCase(pair.getLeft(), pair.getRight()), + new OrderedInvalidUtf8TestCase(pair.getLeft(), pair.getRight()))) + .collect(Collectors.toList())) + .stream() + .flatMap(List::stream).toArray(Object[]::new); } static abstract class TestCase { @@ -142,7 +155,8 @@ void testSuperiorPrefixes(ValueReader reader, int numBytesPerValue, String... st char[] chars = strings[i].toCharArray(); for (int j = 0; j < strings[i].length(); j++) { // this test's ordering assumption is not valid for surrogates - if (!Character.isLowSurrogate(chars[j])) { + char nextChar = (char) (chars[j] + 1); + if (!Character.isSurrogate(chars[j]) && !Character.isSurrogate(nextChar)) { chars[j]++; String string = new String(chars); int signum = strings[i].compareTo(string);