From e5637c9711a8788ea2f35926f8d91ce25fa9e46d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 3 Oct 2023 03:32:19 -0700 Subject: [PATCH 1/2] urlencode nested serializer temp file names so they dont explode stuff --- .../druid/segment/nested/DictionaryIdLookup.java | 14 ++++++++++---- .../nested/NestedDataColumnSupplierTest.java | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java b/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java index a4fd0907066b..a865f680421a 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java @@ -99,7 +99,7 @@ public int lookupString(@Nullable String value) // for strings because of this. if other type dictionary writers could potentially use multiple internal files // in the future, we should transition them to using this approach as well (or build a combination smoosher and // mapper so that we can have a mutable smoosh) - File stringSmoosh = FileUtils.createTempDir(name + "__stringTempSmoosh"); + File stringSmoosh = FileUtils.createTempDir(StringUtils.urlEncode(name) + "__stringTempSmoosh"); final String fileName = NestedCommonFormatColumnSerializer.getInternalFileName( name, NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME @@ -135,7 +135,9 @@ public int lookupString(@Nullable String value) public int lookupLong(@Nullable Long value) { if (longDictionary == null) { - Path longFile = makeTempFile(name + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME); + final Path longFile = makeTempFile( + StringUtils.urlEncode(name) + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME + ); longBuffer = mapWriter(longFile, longDictionaryWriter); longDictionary = FixedIndexed.read(longBuffer, TypeStrategies.LONG, ByteOrder.nativeOrder(), Long.BYTES).get(); // reset position @@ -151,7 +153,9 @@ public int lookupLong(@Nullable Long value) public int lookupDouble(@Nullable Double value) { if (doubleDictionary == null) { - Path doubleFile = makeTempFile(name + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME); + final Path doubleFile = makeTempFile( + StringUtils.urlEncode(name) + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME + ); doubleBuffer = mapWriter(doubleFile, doubleDictionaryWriter); doubleDictionary = FixedIndexed.read(doubleBuffer, TypeStrategies.DOUBLE, ByteOrder.nativeOrder(), Double.BYTES).get(); // reset position @@ -167,7 +171,9 @@ public int lookupDouble(@Nullable Double value) public int lookupArray(@Nullable int[] value) { if (arrayDictionary == null) { - Path arrayFile = makeTempFile(name + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME); + final Path arrayFile = makeTempFile( + StringUtils.urlEncode(name) + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME + ); arrayBuffer = mapWriter(arrayFile, arrayDictionaryWriter); arrayDictionary = FrontCodedIntArrayIndexed.read(arrayBuffer, ByteOrder.nativeOrder()).get(); // reset position diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index 80daa3549dcf..653b39ff9bf7 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -171,7 +171,7 @@ public static void staticSetup() @Before public void setup() throws IOException { - final String fileNameBase = "test"; + final String fileNameBase = "test/column"; final String arrayFileNameBase = "array"; fileMapper = smooshify(fileNameBase, tempFolder.newFolder(), data); baseBuffer = fileMapper.mapFile(fileNameBase); From ce316de96e0034d41c80c9e7e2800bfcbde84d3b Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 3 Oct 2023 13:50:38 -0700 Subject: [PATCH 2/2] adjust --- .../segment/nested/DictionaryIdLookup.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java b/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java index a865f680421a..24f9ca757808 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java @@ -135,9 +135,7 @@ public int lookupString(@Nullable String value) public int lookupLong(@Nullable Long value) { if (longDictionary == null) { - final Path longFile = makeTempFile( - StringUtils.urlEncode(name) + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME - ); + final Path longFile = makeTempFile(name + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME); longBuffer = mapWriter(longFile, longDictionaryWriter); longDictionary = FixedIndexed.read(longBuffer, TypeStrategies.LONG, ByteOrder.nativeOrder(), Long.BYTES).get(); // reset position @@ -153,11 +151,14 @@ public int lookupLong(@Nullable Long value) public int lookupDouble(@Nullable Double value) { if (doubleDictionary == null) { - final Path doubleFile = makeTempFile( - StringUtils.urlEncode(name) + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME - ); + final Path doubleFile = makeTempFile(name + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME); doubleBuffer = mapWriter(doubleFile, doubleDictionaryWriter); - doubleDictionary = FixedIndexed.read(doubleBuffer, TypeStrategies.DOUBLE, ByteOrder.nativeOrder(), Double.BYTES).get(); + doubleDictionary = FixedIndexed.read( + doubleBuffer, + TypeStrategies.DOUBLE, + ByteOrder.nativeOrder(), + Double.BYTES + ).get(); // reset position doubleBuffer.position(0); } @@ -171,9 +172,7 @@ public int lookupDouble(@Nullable Double value) public int lookupArray(@Nullable int[] value) { if (arrayDictionary == null) { - final Path arrayFile = makeTempFile( - StringUtils.urlEncode(name) + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME - ); + final Path arrayFile = makeTempFile(name + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME); arrayBuffer = mapWriter(arrayFile, arrayDictionaryWriter); arrayDictionary = FrontCodedIntArrayIndexed.read(arrayBuffer, ByteOrder.nativeOrder()).get(); // reset position @@ -245,7 +244,7 @@ private int arrayOffset() private Path makeTempFile(String name) { try { - return Files.createTempFile(name, ".tmp"); + return Files.createTempFile(StringUtils.urlEncode(name), null); } catch (IOException e) { throw new RuntimeException(e);