Skip to content

Commit

Permalink
Support renameColumns and handle null on painless scripting on elasti…
Browse files Browse the repository at this point in the history 
…csearch (#2486)

* Support renameColumns on elasticsearch

* Handle null on painless script
  • Loading branch information
@rafaelbey
rafaelbey authored Nov 30, 2023
1 parent 7fa7306 commit 0ce46c0
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 7 deletions.
4 changes: 4 additions & 0 deletions ...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,9 @@ public Handlers(PureModel pureModel)

register("meta::pure::functions::collection::size_Any_MANY__Integer_1_", true, ps -> res("Integer", "one"));

register("meta::pure::functions::collection::pair_U_1__V_1__Pair_1_", false, ps -> res(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, this.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType"))
._rawType(this.pureModel.getType("meta::pure::functions::collection::Pair"))
._typeArguments(Lists.fixedSize.ofAll(ps.stream().map(ValueSpecificationAccessor::_genericType).collect(Collectors.toList()))), "one"));

register("meta::pure::functions::multiplicity::toOne_T_MANY__T_1_", true, ps -> res(ps.get(0)._genericType(), "one"));

Expand Down Expand Up @@ -1796,6 +1799,7 @@ private Map<String, Dispatch> buildDispatch()
map.put("meta::pure::functions::collection::removeDuplicates_T_MANY__T_MANY_", (List<ValueSpecification> ps) -> ps.size() == 1);
map.put("meta::pure::functions::collection::reverse_T_m__T_m_", (List<ValueSpecification> ps) -> ps.size() == 1);
map.put("meta::pure::functions::collection::size_Any_MANY__Integer_1_", (List<ValueSpecification> ps) -> ps.size() == 1);
map.put("meta::pure::functions::collection::pair_U_1__V_1__Pair_1_", (List<ValueSpecification> ps) -> ps.size() == 2);
map.put("meta::pure::functions::collection::slice_T_MANY__Integer_1__Integer_1__T_MANY_", (List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::paginated_T_MANY__Integer_1__Integer_1__T_MANY_",(List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::sortBy_T_m__Function_$0_1$__T_m_", (List<ValueSpecification> ps) -> ps.size() == 2 && matchZeroOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && check(ft._parameters().toList(), (List<? extends VariableExpression> nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity())))));
Expand Down
4 changes: 4 additions & 0 deletions .../legend/engine/plan/execution/stores/elasticsearch/v7/result/ExecutionRequestVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,10 @@ else if (aggregationContainer.multi_terms != null)
}
else
{
if (!this.activities.isEmpty())
{
return false;
}
processor = ExecutionRequestVisitor.this::processNotAggregateResponse;
}

Expand Down
36 changes: 30 additions & 6 deletions ...rc/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,26 @@ function meta::external::store::elasticsearch::v7::pureToEs::processRestrict(vs
);
}

function meta::external::store::elasticsearch::v7::pureToEs::processRenameColumns(vs : FunctionExpression[1], initReq: State[1]): State[1]
{
$initReq.debug(|'Processing ->renameColumns');
let currReq = process($vs.parametersValues->at(0), $initReq);

let toRename = $vs->instanceValuesAtParameter(1, $currReq.sq.inScopeVars)->match([
p:Pair<String,String>[*] | $p,
vss:ValueSpecification[*] | $vss->map(vs|$vs->reactivate($currReq.sq.inScopeVars)->evaluateAndDeactivate()->cast(@Pair<String,String>));
]);

let toRenameMap = $toRename->newMap();

let newESDetails = $currReq.tdsESDetails->map(c |
let newName = $toRenameMap->get($c.name);
if($newName->isNotEmpty(), | ^$c(name=$newName->toOne()), | $c);
);

^$currReq(tdsESDetails = $newESDetails);
}

function meta::external::store::elasticsearch::v7::pureToEs::processDistinct(vs : FunctionExpression[1], initReq: State[1]): State[1]
{
$initReq.debug(|'Processing ->distinct');
Expand Down Expand Up @@ -600,7 +620,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::supportedRoutingFun
pair(supportedIfEqual(extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_), processExtend_FunctionExpression_1__State_1__State_1_),
pair(supportedIfEqual(restrict_TabularDataSet_1__String_MANY__TabularDataSet_1_), processRestrict_FunctionExpression_1__State_1__State_1_),
pair(supportedIfEqual(distinct_TabularDataSet_1__TabularDataSet_1_), processDistinct_FunctionExpression_1__State_1__State_1_),

pair(supportedIfEqual(renameColumns_TabularDataSet_1__Pair_MANY__TabularDataSet_1_), processRenameColumns_FunctionExpression_1__State_1__State_1_),

pair(supportedIfEqual(filter_TabularDataSet_1__Function_1__TabularDataSet_1_), processFilter_FunctionExpression_1__State_1__State_1_),

Expand Down Expand Up @@ -1369,12 +1389,16 @@ function meta::external::store::elasticsearch::v7::pureToEs::processPainlessIsNo
let field = $fieldPair.first.values->cast(@TDSESDetail)->toOne('isNotEmpty only works on tds columns for now');
let req = $fieldPair.second;

let script = $field.resultPath.property->match([
pair(^InlineScript(source = $field->painlessIsNotEmpty()->literal(), params = newMap([])), $req);
}


function meta::external::store::elasticsearch::v7::pureToEs::painlessIsNotEmpty(field: TDSESDetail[1]): String[1]
{
$field.resultPath.property->match([
text: TextProperty[1] | 'params[\'_source\'][\'%s\'] != null',
any: Any[1] | 'doc[\'%s\'].size() != 0';
])->format($field.path());

pair(^InlineScript(source = $script->literal(), params = newMap([])), $req);
}

function meta::external::store::elasticsearch::v7::pureToEs::processPainlessBooleanComparison(fe: FunctionExpression[1], initReq: State[1]): Pair<InlineScript, State>[1]
Expand Down Expand Up @@ -1439,12 +1463,12 @@ function meta::external::store::elasticsearch::v7::pureToEs::processPainlessBool
any: Any[*] | 'params[\'%s\']'
])->format($param.first);

let script = '%s.toInstant().toEpochMilli() %s %s'->format([painlessExtractField($tdsESDetail), $operation, $paramScript]);
let script = '(%s && %s.toInstant().toEpochMilli() %s %s)'->format([painlessIsNotEmpty($tdsESDetail), painlessExtractField($tdsESDetail), $operation, $paramScript]);
let literalOrExpression = $newValue->literalOrExpression(true)->toOne();
pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $literalOrExpression))), $param.second);
},
{|
let script = '%s %s params[\'%s\']'->format([painlessExtractField($tdsESDetail), $operation, $param.first]);
let script = '(%s && %s %s params[\'%s\'])'->format([painlessIsNotEmpty($tdsESDetail), painlessExtractField($tdsESDetail), $operation, $param.first]);
let literalOrExpression = $value->literalOrExpression(true)->toOne();
pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $literalOrExpression))), $param.second);
}
Expand Down
8 changes: 8 additions & 0 deletions ...st/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,4 +145,12 @@ meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testSl
$config->testTdsExpression(x | $x->sort('Title')->slice($var3, $var4));
$config->testTdsExpression(x | $x->sort('Title')->slice(3, $var4));
$config->testTdsExpression(x | $x->sort('Title')->slice($var3, 4));
}

function
<<paramTest.Test>>
{doc.doc = 'Test rename column on Elasticsearch'}
meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testRenameColumn(config:TestConfig[1]):Boolean[1]
{
$config->testTdsExpression(x | $x->renameColumns(pair('Title', 'Titulo')));
}
8 changes: 8 additions & 0 deletions ...src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ meta::external::store::elasticsearch::executionTest::testCase::tds::project::mis
$config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | if($x.getDate('ReleaseDate') >= %1990-01-01, |'90s and newer', |'Older than 90s'), 'Bucket')]));
}

function
<<paramTest.Test>>
{doc.doc = 'Test projection on Elasticsearch with pure if expressions on nullable field'}
meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionOnNullable(config:TestConfig[1]):Boolean[1]
{
$config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.getNullableString('MPAA') == 'PG-13', 'Bucket')]));
}

function
<<paramTest.Test>>
{doc.doc = 'Test projection on Elasticsearch with pure if expressions'}
Expand Down
1 change: 0 additions & 1 deletion ...ngine/plan/execution/stores/elasticsearch/test/Test_Pure_ElasticSearch_ExecutionPlan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ public static TestSuite suite()
TestSuite suite = new TestSuite();
if (DockerClientFactory.instance().isDockerAvailable())
{
System.setProperty("org.finos.legend.engine.plan.execution.stores.elasticsearch.test.password", "s3cret");
CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport();
suite.addTest(PureTestBuilderCompiled.buildSuite(
TestCollection.collectTests(
Expand Down

0 comments on commit 0ce46c0

Please sign in to comment.