Skip to content

Commit

Permalink
One unit test and integration tests for the lookup operator.
Browse files Browse the repository at this point in the history
Signed-off-by: Lukasz Soszynski <lukasz.soszynski@eliatra.com>
  • Loading branch information
lukasz-soszynski-eliatra authored Jul 2, 2024
1 parent d6a1fb4 commit 5b65e61
Showing 10 changed files with 276 additions and 25 deletions.
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@ public PhysicalPlan visitLookup(LogicalLookup node, C context) {
node.getAppendOnly(),
node.getCopyFieldMap(),
(a, b) -> {
throw new RuntimeException("not implemented by DefaultImplementor");
throw new UnsupportedOperationException("Lookup not implemented by DefaultImplementor");
});
}

Original file line number Diff line number Diff line change
@@ -143,7 +143,7 @@ public static LogicalPlan lookup(
LogicalPlan input,
String indexName,
Map<ReferenceExpression, ReferenceExpression> matchFieldMap,
boolean appendOnly,
Boolean appendOnly,
Map<ReferenceExpression, ReferenceExpression> copyFields) {
return new LogicalLookup(input, indexName, matchFieldMap, appendOnly, copyFields);
}
Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@
import org.opensearch.sql.ast.tree.RareTopN.CommandType;
import org.opensearch.sql.ast.tree.Sort;
import org.opensearch.sql.data.model.ExprBooleanValue;
import org.opensearch.sql.data.model.ExprValueUtils;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.executor.pagination.PlanSerializer;
import org.opensearch.sql.expression.DSL;
@@ -59,6 +60,7 @@
import org.opensearch.sql.expression.window.WindowDefinition;
import org.opensearch.sql.expression.window.ranking.RowNumberFunction;
import org.opensearch.sql.planner.logical.LogicalCloseCursor;
import org.opensearch.sql.planner.logical.LogicalLookup;
import org.opensearch.sql.planner.logical.LogicalPaginate;
import org.opensearch.sql.planner.logical.LogicalPlan;
import org.opensearch.sql.planner.logical.LogicalPlanDSL;
@@ -308,4 +310,19 @@ public void visitLookup_should_build_LookupOperator() {

assertEquals(expectedPhysicalPlan, lookupOperator);
}

@Test
public void visitLookup_should_throw_unsupportedOperationException() {
LogicalLookup input = mock(LogicalLookup.class);
LogicalPlan dataSource = mock(LogicalPlan.class);
PhysicalPlan physicalSource = mock(PhysicalPlan.class);
when(dataSource.accept(implementor, null)).thenReturn(physicalSource);
when(input.getChild()).thenReturn(List.of(dataSource));
PhysicalPlan lookupOperator = implementor.visitLookup(input, null);
when(physicalSource.next()).thenReturn(ExprValueUtils.tupleValue(Map.of("field", "value")));

var ex = assertThrows(UnsupportedOperationException.class, () -> lookupOperator.next());

assertEquals("Lookup not implemented by DefaultImplementor", ex.getMessage());
}
}
Original file line number Diff line number Diff line change
@@ -709,7 +709,17 @@ public enum Index {
TestsConstants.TEST_INDEX_NESTED_WITH_NULLS,
"multi_nested",
getNestedTypeIndexMapping(),
"src/test/resources/nested_with_nulls.json");
"src/test/resources/nested_with_nulls.json"),
IOT_READINGS(
TestsConstants.TEST_INDEX_IOT_READINGS,
"iot_readings",
getMappingFile("iot_readings_index_mapping.json"),
"src/test/resources/iot_readings.json"),
IOT_SENSORS(
TestsConstants.TEST_INDEX_IOT_SENSORS,
"iot_sensors",
getMappingFile("iot_sensors_index_mapping.json"),
"src/test/resources/iot_sensors.json");

private final String name;
private final String type;
Original file line number Diff line number Diff line change
@@ -57,6 +57,8 @@ public class TestsConstants {
public static final String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard";
public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested";
public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
public static final String TEST_INDEX_IOT_READINGS = TEST_INDEX + "_iot_readings";
public static final String TEST_INDEX_IOT_SENSORS = TEST_INDEX + "_iot_sensors";
public static final String DATASOURCES = ".ql-datasources";

public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
187 changes: 165 additions & 22 deletions integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java
Original file line number Diff line number Diff line change
@@ -5,8 +5,8 @@

package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_IOT_READINGS;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_IOT_SENSORS;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;

@@ -18,47 +18,190 @@ public class LookupCommandIT extends PPLIntegTestCase {

@Override
public void init() throws IOException {
loadIndex(Index.BANK);
loadIndex(Index.BANK_WITH_NULL_VALUES);
loadIndex(Index.IOT_READINGS);
loadIndex(Index.IOT_SENSORS);
}

@Test
public void testLookup() throws IOException {
JSONObject result = executeQuery(String.format("source=%s | lookup %s male", TEST_INDEX_BANK));
verifyDataRows(result, rows(true), rows(false));
JSONObject result =
executeQuery(
String.format(
"source=%s | lookup %s did as device-id | sort @timestamp",
TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
verifyDataRows(
result,
rows(
28.1,
"2015-01-20 15:31:32.406431",
255,
"temperature-basement",
"meter",
255,
"VendorOne"),
rows(
27.8,
"2016-01-20 15:31:33.509334",
256,
"temperature-living-room",
"temperature meter",
256,
"VendorTwo"),
rows(
27.4,
"2017-01-20 15:31:35.732436",
257,
"temperature-bedroom",
"camcorder",
257,
"VendorThree"),
rows(
28.5,
"2018-01-20 15:32:32.406431",
255,
"temperature-basement",
"meter",
255,
"VendorOne"),
rows(
27.9,
"2019-01-20 15:32:33.509334",
256,
"temperature-living-room",
"temperature meter",
256,
"VendorTwo"),
rows(
27.4,
"2020-01-20 15:32:35.732436",
257,
"temperature-bedroom",
"camcorder",
257,
"VendorThree"));
}

@Test
public void testConsecutiveDedup() throws IOException {
public void testLookupSelectedAttribute() throws IOException {
JSONObject result =
executeQuery(
String.format(
"source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK));
verifyDataRows(result, rows(true), rows(false), rows(true), rows(false));
"source=%s | lookup %s did as device-id type, vendor | sort @timestamp",
TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
verifyDataRows(
result,
rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter", "VendorOne"),
rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter", "VendorTwo"),
rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder", "VendorThree"),
rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter", "VendorOne"),
rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter", "VendorTwo"),
rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder", "VendorThree"));
}

@Test
public void testAllowMoreDuplicates() throws IOException {
public void testLookupRenameSelectedAttributes() throws IOException {
JSONObject result =
executeQuery(String.format("source=%s | dedup 2 male | fields male", TEST_INDEX_BANK));
verifyDataRows(result, rows(true), rows(true), rows(false), rows(false));
executeQuery(
String.format(
"source=%s | lookup %s did as device-id did as dev_id, type as kind, vendor | sort"
+ " @timestamp",
TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
verifyDataRows(
result,
rows(28.1, "2015-01-20 15:31:32.406431", 255, 255, "meter", "VendorOne"),
rows(27.8, "2016-01-20 15:31:33.509334", 256, 256, "temperature meter", "VendorTwo"),
rows(27.4, "2017-01-20 15:31:35.732436", 257, 257, "camcorder", "VendorThree"),
rows(28.5, "2018-01-20 15:32:32.406431", 255, 255, "meter", "VendorOne"),
rows(27.9, "2019-01-20 15:32:33.509334", 256, 256, "temperature meter", "VendorTwo"),
rows(27.4, "2020-01-20 15:32:35.732436", 257, 257, "camcorder", "VendorThree"));
}

@Test
public void testLookupSelectedMultipleAttributes() throws IOException {
JSONObject result =
executeQuery(
String.format(
"source=%s | lookup %s did as device-id type | sort @timestamp",
TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
verifyDataRows(
result,
rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter"),
rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter"),
rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder"),
rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter"),
rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter"),
rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder"));
}

@Test
public void testLookupShouldAppendOnlyShouldBeFalseByDefault() throws IOException {
JSONObject result =
executeQuery(
String.format(
"source=%s | rename temperature as vendor | lookup %s did as device-id | sort"
+ " @timestamp",
TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
verifyDataRows(
result,
rows("2015-01-20 15:31:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255),
rows(
"2016-01-20 15:31:33.509334",
256,
"VendorTwo",
"temperature-living-room",
"temperature meter",
256),
rows(
"2017-01-20 15:31:35.732436",
257,
"VendorThree",
"temperature-bedroom",
"camcorder",
257),
rows("2018-01-20 15:32:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255),
rows(
"2019-01-20 15:32:33.509334",
256,
"VendorTwo",
"temperature-living-room",
"temperature meter",
256),
rows(
"2020-01-20 15:32:35.732436",
257,
"VendorThree",
"temperature-bedroom",
"camcorder",
257));
}

@Test
public void testKeepEmptyDedup() throws IOException {
public void testLookupWithAppendOnlyFalse() throws IOException {
JSONObject result =
executeQuery(
String.format(
"source=%s | dedup balance keepempty=true | fields firstname, balance",
TEST_INDEX_BANK_WITH_NULL_VALUES));
"source=%s | rename temperature as vendor | lookup %s did as device-id appendonly ="
+ " true | sort @timestamp",
TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
verifyDataRows(
result,
rows("Amber JOHnny", 39225),
rows("Hattie", null),
rows("Nanette", 32838),
rows("Dale", 4180),
rows("Elinor", null),
rows("Virginia", null),
rows("Dillard", 48086));
rows("2015-01-20 15:31:32.406431", 255, 28.1, "temperature-basement", "meter", 255),
rows(
"2016-01-20 15:31:33.509334",
256,
27.8,
"temperature-living-room",
"temperature meter",
256),
rows("2017-01-20 15:31:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257),
rows("2018-01-20 15:32:32.406431", 255, 28.5, "temperature-basement", "meter", 255),
rows(
"2019-01-20 15:32:33.509334",
256,
27.9,
"temperature-living-room",
"temperature meter",
256),
rows("2020-01-20 15:32:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"mappings": {
"properties": {
"device-id": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"temperature": {
"type": "float"
},
"timestamp": {
"type": "date"
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"mappings": {
"properties": {
"did": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"vendor": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
13 changes: 13 additions & 0 deletions integ-test/src/test/resources/iot_readings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{ "index" : { "_id" : "1" } }
{ "device-id":255, "temperature":28.1, "@timestamp":"2015-01-20T15:31:32.406431+00:00" }
{ "index" : { "_id" : "2" } }
{ "device-id":256, "temperature":27.8, "@timestamp":"2016-01-20T15:31:33.509334+00:00" }
{ "index" : { "_id" : "3" } }
{ "device-id":257, "temperature":27.4, "@timestamp":"2017-01-20T15:31:35.732436+00:00" }
{ "index" : { "_id" : "4" } }
{ "device-id":255, "temperature":28.5, "@timestamp":"2018-01-20T15:32:32.406431+00:00" }
{ "index" : { "_id" : "5" } }
{ "device-id":256, "temperature":27.9, "@timestamp":"2019-01-20T15:32:33.509334+00:00" }
{ "index" : { "_id" : "6" } }
{ "device-id":257, "temperature":27.4, "@timestamp":"2020-01-20T15:32:35.732436+00:00" }
{ "index" : { "_id" : "7" } }
6 changes: 6 additions & 0 deletions integ-test/src/test/resources/iot_sensors.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{ "index" : { "_id" : "1" } }
{ "did" : 255, "name":"temperature-basement", "vendor":"VendorOne", "type":"meter"}
{ "index" : { "_id" : "2" } }
{ "did" : 256, "name":"temperature-living-room", "vendor":"VendorTwo", "type":"temperature meter" }
{ "index" : { "_id" : "3" } }
{ "did" : 257, "name":"temperature-bedroom", "vendor":"VendorThree", "type":"camcorder"}

0 comments on commit 5b65e61

Please sign in to comment.