Skip to content

Commit

Permalink
Review realization row limit (#3208)
Browse files Browse the repository at this point in the history
* Add Realize In Memory Row Limit and Handle Streaming Object Result

* Remove ROW_LIMIT_PROPERTY_NAME - Now standardized from streaming result

* Realizaed Row Limit fix for test

* use executeLegendQuery
  • Loading branch information
hbalasu2 authored Oct 28, 2024
1 parent e7e709b commit 0c01cf7
Show file tree
Hide file tree
Showing 9 changed files with 219 additions and 19 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public static Test suite() throws Exception
suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStoreUnion::inMemoryAndRelational", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::ordered", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::pureToSqlQuery::calendarAggregations", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));
suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::platform", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport));

return suite;
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ else if (result instanceof StreamingObjectResult)
{
return (T) ((StreamingObjectResult) result).getObjectStream();
}
else if (rawType.equals(List.class))
{
return this.getResultAsParameterizedType(type, result.realizeInMemory());
}
}
throw new IllegalArgumentException("Unable to convert " + result.getClass().getSimpleName() + " to " + TypeUtils.toString(type));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,15 @@ public String flush(Serializer serializer)
throw new RuntimeException(e);
}
}

public static long getRealizeRowLimit()
{
return Long.getLong(
"org.finos.legend.engine.realizedResultRowLimit",
Long.getLong(
"org.finos.legend.engine.realizedRelationalResultRowLimit",
1_000L
)
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -100,17 +100,6 @@ public Result realizeInMemory()
}
}

private static long getRealizeRowLimit()
{
return Long.getLong(
"org.finos.legend.engine.realizedResultRowLimit",
Long.getLong(
"org.finos.legend.engine.realizedRelationalResultRowLimit",
1_000L
)
);
}

@Override
public void cancel()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package org.finos.legend.engine.plan.execution.result.object;

import org.eclipse.collections.api.factory.Lists;
import org.finos.legend.engine.plan.dependencies.store.shared.IResult;
import org.finos.legend.engine.plan.execution.result.ConstantResult;
import org.finos.legend.engine.plan.execution.result.Result;
Expand All @@ -23,6 +24,7 @@
import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat;
import org.finos.legend.engine.plan.execution.result.serialization.Serializer;

import java.util.List;
import java.util.stream.Stream;

public class StreamingObjectResult<T> extends StreamingResult
Expand Down Expand Up @@ -91,4 +93,22 @@ public Serializer getSerializer(SerializationFormat format)
throw new RuntimeException(format.toString() + " format not currently supported with StreamingObjectResult");
}
}

@Override
public Result realizeInMemory()
{
List<T> res = Lists.mutable.empty();
this.getObjectStream().forEach(x ->
{
if (res.size() > StreamingResult.getRealizeRowLimit())
{
throw new RuntimeException("Too many rows returned. Realization of relational results currently supports results with up to " + getRealizeRowLimit() + " rows.");
}
else
{
res.add(x);
}
});
return new ConstantResult(res);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ public class RealizedRelationalResult extends StreamingResult
public List<SQLResultColumn> columns;
public List<List<Object>> resultSetRows;
public List<List<Object>> transformedRows;

private static final int DEFAULT_ROW_LIMIT = 1000;
public static final String ROW_LIMIT_PROPERTY_NAME = "org.finos.legend.engine.realizedRelationalResultRowLimit";

public RealizedRelationalResult(RelationalResult relationalResult) throws SQLException
Expand All @@ -52,7 +50,7 @@ public RealizedRelationalResult(RelationalResult relationalResult) throws SQLExc
this.transformedRows = Lists.mutable.empty();
this.resultSetRows = Lists.mutable.empty();
ResultSet resultSet = relationalResult.resultSet;
int SUPPORTED_RESULT_ROWS = getRowLimit();
long SUPPORTED_RESULT_ROWS = getRealizeRowLimit();
int rowCount = 0;
try
{
Expand Down Expand Up @@ -84,11 +82,6 @@ public RealizedRelationalResult(RelationalResult relationalResult) throws SQLExc
}
}

public int getRowLimit()
{
return Integer.getInteger(ROW_LIMIT_PROPERTY_NAME, DEFAULT_ROW_LIMIT);
}

private RealizedRelationalResult()
{
super(Lists.mutable.empty());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,21 @@
<artifactId>legend-engine-language-pure-compiler</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-functions-planExecution-pure</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-runtime-java-extension-compiled-functions-planExecution</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-runtime-java-extension-shared-functions-planExecution</artifactId>
<scope>test</scope>
</dependency>

<!-- TEST -->
</dependencies>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@

import meta::pure::profiles::*;
import meta::pure::mapping::*;
import meta::pure::profiles::*;


function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsNotEmptyForRelational_returnsTrue():Boolean[1]
{
//CreateAndFill
meta::relational::tests::platform::operations::createTablesAndFillDbUS();

let finalQuery =
{|

let employees = meta::relational::tests::platform::operations::Person.all()
->filter( x | $x.region =='NYC')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());

$employees->isNotEmpty();
};

let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
assertEquals('true', $result);
}

function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsNotEmptyForRelational_returnsFalse():Boolean[1]
{
//CreateAndFill
meta::relational::tests::platform::operations::createTablesAndFillDbUS();


let finalQuery =
{|

let employees = meta::relational::tests::platform::operations::Person.all()
->filter( x | $x.region =='Chennai')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());

$employees->isNotEmpty();
};

let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
assertEquals('false', $result);

}

function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsEmptyForRelational_returnsFalse():Boolean[1]
{
//CreateAndFill
meta::relational::tests::platform::operations::createTablesAndFillDbUS();


let finalQuery =
{|

let employees = meta::relational::tests::platform::operations::Person.all()
->filter( x | $x.region =='NYC')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());

$employees->isEmpty();
};

let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
assertEquals('false', $result);

}

function <<test.Test,test.AlloyOnly>> meta::relational::tests::platform::operations::testIsEmptyForRelational_returnsTrue():Boolean[1]
{
//CreateAndFill
meta::relational::tests::platform::operations::createTablesAndFillDbUS();


let finalQuery =
{|

let employees = meta::relational::tests::platform::operations::Person.all()
->filter( x | $x.region =='Chennai')->from(meta::relational::tests::platform::operations::mapping::EmployeeUSMapping,meta::relational::tests::platform::operations::runtime::testRuntimeUS());

$employees->isEmpty();
};

let result = meta::legend::executeLegendQuery($finalQuery,[],^meta::pure::runtime::ExecutionContext(),meta::relational::extension::relationalExtensions());
assertEquals('true', $result);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
Class meta::relational::tests::platform::operations::Person
{
firstName: String[1];
lastName: String[*];
isDataEng:Boolean[1];
region:String[1];
}

function meta::relational::tests::platform::operations::createTablesAndFillDbUS():Boolean[1]
{
let connection = meta::relational::tests::platform::operations::runtime::testRuntimeUS().connectionByElement(meta::relational::tests::platform::operations::database::EmployeeUSDB)->cast(@meta::external::store::relational::runtime::TestDatabaseConnection);
meta::relational::metamodel::execute::executeInDb('Drop table if exists Person;', $connection);
meta::relational::metamodel::execute::executeInDb('Create Table Person(PKEY INT PRIMARY KEY,FIRST_NAME VARCHAR(200),LAST_NAME VARCHAR(200),IS_FULLTIME BOOLEAN,REGION VARCHAR(200));', $connection);
meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (1,\'John\', \'Smith\',True, \'NYC\');', $connection);
meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (2,\'Jane\', \'Doe\',True, \'Chicago\');', $connection);
meta::relational::metamodel::execute::executeInDb('insert into Person (PKEY, FIRST_NAME, LAST_NAME,IS_FULLTIME,REGION) values (3,\'Christopher\', \'Nolan\',True, \'NYC\');', $connection);
true;
}

###Relational
Database meta::relational::tests::platform::operations::database::EmployeeUSDB
(
Table Person (
PKEY INT PRIMARY KEY,
FIRST_NAME VARCHAR(200) ,
LAST_NAME VARCHAR(200) ,
IS_FULLTIME BIT,
REGION VARCHAR(200)
)
)


###Mapping
import meta::relational::tests::*;
import meta::external::store::relational::tests::*;
import meta::relational::tests::model::simple::*;

Mapping meta::relational::tests::platform::operations::mapping::EmployeeUSMapping
(
meta::relational::tests::platform::operations::Person: Relational{
~primaryKey
(
[meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.PKEY
)
~mainTable [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person
firstName:[meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.FIRST_NAME,
lastName: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.LAST_NAME,
isDataEng: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.IS_FULLTIME,
region: [meta::relational::tests::platform::operations::database::EmployeeUSDB]Person.REGION
}
)


###Pure
import meta::relational::metamodel::*;
import meta::json::*;
import meta::json::tests::*;
import meta::relational::tests::*;
import meta::external::store::relational::tests::*;
import meta::pure::profiles::*;
import meta::relational::metamodel::execute::*;
import meta::core::runtime::*;
import meta::external::store::relational::runtime::*;
import meta::relational::runtime::*;


function meta::relational::tests::platform::operations::runtime::testRuntimeUS():Runtime[1]
{
^Runtime(connectionStores = meta::relational::tests::platform::operations::testDatabaseConnectionPerson(meta::relational::tests::platform::operations::database::EmployeeUSDB,'GMT'))
}

function meta::relational::tests::platform::operations::testDatabaseConnectionPerson(db:Database[1], timeZone:String[0..1]) : ConnectionStore[1]
{
^ConnectionStore(
connection = ^meta::external::store::relational::runtime::TestDatabaseConnection(
type = DatabaseType.H2,
timeZone = if($timeZone->isEmpty(), |'GMT', |$timeZone)
),
element = $db);
}




0 comments on commit 0c01cf7

Please sign in to comment.