Skip to content

Commit

Permalink
[Tests] IT tests and test utils update to fix failing tests for serve…
Browse files Browse the repository at this point in the history 
…rless

Signed-off-by: Manasvini B S <[email protected]>
  • Loading branch information
@manasvinibs
manasvinibs committed Aug 8, 2024
1 parent 7022a09 commit 2881b6b
Show file tree
Hide file tree
Showing 11 changed files with 463 additions and 89 deletions.
25 changes: 20 additions & 5 deletions integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -690,12 +691,20 @@ public void sanitizeTest() throws IOException {
Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE),
false);
List<String> lines = csvResult.getLines();
Collections.sort(lines);
assertEquals(5, lines.size());
assertEquals(lines.get(0), "'+Amber JOHnny,Duke Willmington+");
assertEquals(lines.get(1), "'-Hattie,Bond-");
assertEquals(lines.get(2), "'=Nanette,Bates=");
assertEquals(lines.get(3), "'@Dale,Adams@");
assertEquals(lines.get(4), "\",Elinor\",\"Ratliff,,,\"");

List<String> expectedLines =
Arrays.asList(
"'+Amber JOHnny,Duke Willmington+",
"'-Hattie,Bond-",
"'=Nanette,Bates=",
"'@Dale,Adams@",
"\",Elinor\",\"Ratliff,,,\"");

Collections.sort(expectedLines);
assertEquals(expectedLines.size(), lines.size());
assertContainsSameItems(expectedLines, lines);
}

@Test
Expand All @@ -719,6 +728,12 @@ private void verifyFieldOrder(final String[] expectedFields) throws IOException
verifyFieldOrder(expectedFields, query);
}

private void assertContainsSameItems(List<String> expectedLines, List<String> actualLines) {
for (int i = 0; i < expectedLines.size(); i++) {
assertEquals(expectedLines.get(i), actualLines.get(i));
}
}

private void verifyFieldOrder(final String[] expectedFields, final String query)
throws IOException {

Expand Down
5 changes: 3 additions & 2 deletions integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE;
import static org.opensearch.sql.util.TestUtils.assertRowsEqual;

import java.io.IOException;
import java.util.Locale;
Expand All @@ -27,7 +28,7 @@ public void sanitizeTest() throws IOException {
Locale.ROOT,
"source=%s | fields firstname, lastname",
TEST_INDEX_BANK_CSV_SANITIZE));
assertEquals(
assertRowsEqual(
StringUtils.format(
"firstname,lastname%n"
+ "'+Amber JOHnny,Duke Willmington+%n"
Expand All @@ -47,7 +48,7 @@ public void escapeSanitizeTest() throws IOException {
"source=%s | fields firstname, lastname",
TEST_INDEX_BANK_CSV_SANITIZE),
false);
assertEquals(
assertRowsEqual(
StringUtils.format(
"firstname,lastname%n"
+ "+Amber JOHnny,Duke Willmington+%n"
Expand Down
44 changes: 43 additions & 1 deletion integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;

Expand All @@ -35,7 +38,16 @@ public void testConsecutiveDedup() throws IOException {
executeQuery(
String.format(
"source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK));
verifyDataRows(result, rows(true), rows(false), rows(true), rows(false));
List<Object[]> actualRows = extractActualRows(result);
List<Object[]> expectedRows = getExpectedDedupRows(actualRows);
assertTrue("Deduplication was not consecutive", expectedRows != null);
assertEquals(
"Row count after deduplication does not match", expectedRows.size(), actualRows.size());

// Verify the expected and actual rows match
for (int i = 0; i < expectedRows.size(); i++) {
assertArrayEquals(expectedRows.get(i), actualRows.get(i));
}
}

@Test
Expand All @@ -62,4 +74,34 @@ public void testKeepEmptyDedup() throws IOException {
rows("Virginia", null),
rows("Dillard", 48086));
}

private List<Object[]> extractActualRows(JSONObject result) {
JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> actualRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
actualRows.add(new Object[] {row.get(0)});
}
return actualRows;
}

// Verify consecutive deduplication and create the expected rows
private List<Object[]> getExpectedDedupRows(List<Object[]> actualRows) {
boolean isConsecutive = true;
Object previousValue = null;
List<Object[]> expectedRows = new ArrayList<>();

for (Object[] currentRow : actualRows) {
Object currentValue = currentRow[0];
if (previousValue == null || !currentValue.equals(previousValue)) {
expectedRows.add(currentRow);
} else {
isConsecutive = false;
break;
}
previousValue = currentValue;
}

return isConsecutive ? expectedRows : null;
}
}
131 changes: 102 additions & 29 deletions integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.verifyOrder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
import org.junit.Test;

public class ParseCommandIT extends PPLIntegTestCase {
Expand All @@ -26,15 +28,38 @@ public void testParseCommand() throws IOException {
executeQuery(
String.format(
"source=%s | parse email '.+@(?<host>.+)' | fields email, host", TEST_INDEX_BANK));
verifyOrder(
result,
rows("[email protected]", "pyrami.com"),
rows("[email protected]", "netagy.com"),
rows("[email protected]", "quility.com"),
rows("[email protected]", "boink.com"),
rows("[email protected]", "scentric.com"),
rows("[email protected]", "filodyne.com"),
rows("[email protected]", "quailcom.com"));

// Create the expected rows
List<Object[]> expectedRows =
new ArrayList<>(
List.of(
new Object[] {"[email protected]", "pyrami.com"},
new Object[] {"[email protected]", "netagy.com"},
new Object[] {"[email protected]", "quility.com"},
new Object[] {"[email protected]", "boink.com"},
new Object[] {"[email protected]", "scentric.com"},
new Object[] {"[email protected]", "filodyne.com"},
new Object[] {"[email protected]", "quailcom.com"}));

// Actual rows from the response
JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> actualRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
actualRows.add(new Object[] {row.getString(0), row.getString(1)});
}

if (expectedRows.size() != actualRows.size()) {
Assert.fail("Row count is different " + expectedRows + " " + actualRows);
}
// Sort the lists before compare
expectedRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));
actualRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));

// Compare the sorted lists by iterating over elements and using assertArrayEquals
for (int i = 0; i < expectedRows.size(); i++) {
assertArrayEquals(expectedRows.get(i), actualRows.get(i));
}
}

@Test
Expand All @@ -43,15 +68,39 @@ public void testParseCommandReplaceOriginalField() throws IOException {
executeQuery(
String.format(
"source=%s | parse email '.+@(?<email>.+)' | fields email", TEST_INDEX_BANK));
verifyOrder(
result,
rows("pyrami.com"),
rows("netagy.com"),
rows("quility.com"),
rows("boink.com"),
rows("scentric.com"),
rows("filodyne.com"),
rows("quailcom.com"));

// Create the expected rows
List<Object[]> expectedRows =
new ArrayList<>(
List.of(
new Object[] {"pyrami.com"},
new Object[] {"netagy.com"},
new Object[] {"quility.com"},
new Object[] {"boink.com"},
new Object[] {"scentric.com"},
new Object[] {"filodyne.com"},
new Object[] {"quailcom.com"}));

// Actual rows from the response
JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> actualRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
actualRows.add(new Object[] {row.getString(0)});
}

// Sort the lists using natural ordering
expectedRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));
actualRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));

if (expectedRows.size() != actualRows.size()) {
Assert.fail("Row count is different " + expectedRows + " " + actualRows);
}

// Compare the sorted lists by iterating over elements and using assertArrayEquals
for (int i = 0; i < expectedRows.size(); i++) {
assertArrayEquals(expectedRows.get(i), actualRows.get(i));
}
}

@Test
Expand All @@ -62,14 +111,38 @@ public void testParseCommandWithOtherRunTimeFields() throws IOException {
"source=%s | parse email '.+@(?<host>.+)' | "
+ "eval eval_result=1 | fields host, eval_result",
TEST_INDEX_BANK));
verifyOrder(
result,
rows("pyrami.com", 1),
rows("netagy.com", 1),
rows("quility.com", 1),
rows("boink.com", 1),
rows("scentric.com", 1),
rows("filodyne.com", 1),
rows("quailcom.com", 1));

// Create the expected rows as List<Object[]>
List<Object[]> expectedRows =
new ArrayList<>(
List.of(
new Object[] {"pyrami.com", 1},
new Object[] {"netagy.com", 1},
new Object[] {"quility.com", 1},
new Object[] {"boink.com", 1},
new Object[] {"scentric.com", 1},
new Object[] {"filodyne.com", 1},
new Object[] {"quailcom.com", 1}));

// Actual rows from the response
JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> actualRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
actualRows.add(new Object[] {row.getString(0), row.getInt(1)});
}

if (expectedRows.size() != actualRows.size()) {
Assert.fail("Row count is different " + expectedRows + " " + actualRows);
}

// Sort both expected and actual rows
expectedRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));
actualRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));

// Compare the sorted lists by iterating over elements and using assertArrayEquals
for (int i = 0; i < expectedRows.size(); i++) {
assertArrayEquals(expectedRows.get(i), actualRows.get(i));
}
}
}
59 changes: 55 additions & 4 deletions integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@
import static org.opensearch.sql.util.MatcherUtils.verifyOrder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Test;

Expand All @@ -38,17 +44,62 @@ public void testSortWithNullValue() throws IOException {
String.format(
"source=%s | sort balance | fields firstname, balance",
TEST_INDEX_BANK_WITH_NULL_VALUES));

JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> nullRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
if (row.isNull(1)) {
nullRows.add(new Object[] {row.getString(0), null});
}
}
// Verify the set values for null balances as rows with null balance can return in any order
List<Object[]> expectedNullRows =
Arrays.asList(
new Object[] {"Hattie", null},
new Object[] {"Elinor", null},
new Object[] {"Virginia", null});
assertSetEquals(expectedNullRows, nullRows);

// Create a new JSONArray to store not null balance rows
JSONArray nonNullBalanceRows = new JSONArray();

for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
if (!row.isNull(1)) { // Check if the balance (index 1) is not null
nonNullBalanceRows.put(row);
}
}

// Create a new JSONObject with the filtered data
JSONObject filteredResult = new JSONObject();
filteredResult.put("schema", result.getJSONArray("schema"));
filteredResult.put("total", nonNullBalanceRows.length()); // Update total count
filteredResult.put("datarows", nonNullBalanceRows);
filteredResult.put("size", nonNullBalanceRows.length()); // Update size count

verifyOrder(
result,
rows("Hattie", null),
rows("Elinor", null),
rows("Virginia", null),
filteredResult,
rows("Dale", 4180),
rows("Nanette", 32838),
rows("Amber JOHnny", 39225),
rows("Dillard", 48086));
}

private void assertSetEquals(List<Object[]> expected, List<Object[]> actual) {
Set<List<Object>> expectedSet = new HashSet<>();
for (Object[] arr : expected) {
expectedSet.add(Arrays.asList(arr));
}

Set<List<Object>> actualSet = new HashSet<>();
for (Object[] arr : actual) {
actualSet.add(Arrays.asList(arr));
}

assertEquals(expectedSet, actualSet);
}

@Test
public void testSortStringField() throws IOException {
JSONObject result =
Expand Down
Loading

0 comments on commit 2881b6b

Please sign in to comment.