diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index d83ee915ee5..28fa3047e01 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -490,6 +490,9 @@ public Handlers(PureModel pureModel) register("meta::pure::functions::collection::size_Any_MANY__Integer_1_", true, ps -> res("Integer", "one")); + register("meta::pure::functions::collection::pair_U_1__V_1__Pair_1_", false, ps -> res(new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, this.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) + ._rawType(this.pureModel.getType("meta::pure::functions::collection::Pair")) + ._typeArguments(Lists.fixedSize.ofAll(ps.stream().map(ValueSpecificationAccessor::_genericType).collect(Collectors.toList()))), "one")); register("meta::pure::functions::multiplicity::toOne_T_MANY__T_1_", true, ps -> res(ps.get(0)._genericType(), "one")); @@ -1796,6 +1799,7 @@ private Map buildDispatch() map.put("meta::pure::functions::collection::removeDuplicates_T_MANY__T_MANY_", (List ps) -> ps.size() == 1); map.put("meta::pure::functions::collection::reverse_T_m__T_m_", (List ps) -> ps.size() == 1); map.put("meta::pure::functions::collection::size_Any_MANY__Integer_1_", (List ps) -> ps.size() == 1); + map.put("meta::pure::functions::collection::pair_U_1__V_1__Pair_1_", (List ps) -> ps.size() == 2); map.put("meta::pure::functions::collection::slice_T_MANY__Integer_1__Integer_1__T_MANY_", (List ps) -> ps.size() == 3 && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name()))); map.put("meta::pure::functions::collection::paginated_T_MANY__Integer_1__Integer_1__T_MANY_",(List ps) -> ps.size() == 3 && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name()))); map.put("meta::pure::functions::collection::sortBy_T_m__Function_$0_1$__T_m_", (List ps) -> ps.size() == 2 && matchZeroOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && check(ft._parameters().toList(), (List nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity()))))); diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/v7/result/ExecutionRequestVisitor.java b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/v7/result/ExecutionRequestVisitor.java index c433d87f0c0..87137ea1076 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/v7/result/ExecutionRequestVisitor.java +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/v7/result/ExecutionRequestVisitor.java @@ -577,6 +577,10 @@ else if (aggregationContainer.multi_terms != null) } else { + if (!this.activities.isEmpty()) + { + return false; + } processor = ExecutionRequestVisitor.this::processNotAggregateResponse; } diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure index 28c81d44034..a9897f75476 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/src/main/resources/core_elasticsearch_seven_metamodel/functions/pure_to_elasticsearch.pure @@ -149,6 +149,26 @@ function meta::external::store::elasticsearch::v7::pureToEs::processRestrict(vs ); } +function meta::external::store::elasticsearch::v7::pureToEs::processRenameColumns(vs : FunctionExpression[1], initReq: State[1]): State[1] +{ + $initReq.debug(|'Processing ->renameColumns'); + let currReq = process($vs.parametersValues->at(0), $initReq); + + let toRename = $vs->instanceValuesAtParameter(1, $currReq.sq.inScopeVars)->match([ + p:Pair[*] | $p, + vss:ValueSpecification[*] | $vss->map(vs|$vs->reactivate($currReq.sq.inScopeVars)->evaluateAndDeactivate()->cast(@Pair)); + ]); + + let toRenameMap = $toRename->newMap(); + + let newESDetails = $currReq.tdsESDetails->map(c | + let newName = $toRenameMap->get($c.name); + if($newName->isNotEmpty(), | ^$c(name=$newName->toOne()), | $c); + ); + + ^$currReq(tdsESDetails = $newESDetails); +} + function meta::external::store::elasticsearch::v7::pureToEs::processDistinct(vs : FunctionExpression[1], initReq: State[1]): State[1] { $initReq.debug(|'Processing ->distinct'); @@ -600,7 +620,7 @@ function meta::external::store::elasticsearch::v7::pureToEs::supportedRoutingFun pair(supportedIfEqual(extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_), processExtend_FunctionExpression_1__State_1__State_1_), pair(supportedIfEqual(restrict_TabularDataSet_1__String_MANY__TabularDataSet_1_), processRestrict_FunctionExpression_1__State_1__State_1_), pair(supportedIfEqual(distinct_TabularDataSet_1__TabularDataSet_1_), processDistinct_FunctionExpression_1__State_1__State_1_), - + pair(supportedIfEqual(renameColumns_TabularDataSet_1__Pair_MANY__TabularDataSet_1_), processRenameColumns_FunctionExpression_1__State_1__State_1_), pair(supportedIfEqual(filter_TabularDataSet_1__Function_1__TabularDataSet_1_), processFilter_FunctionExpression_1__State_1__State_1_), @@ -1369,12 +1389,16 @@ function meta::external::store::elasticsearch::v7::pureToEs::processPainlessIsNo let field = $fieldPair.first.values->cast(@TDSESDetail)->toOne('isNotEmpty only works on tds columns for now'); let req = $fieldPair.second; - let script = $field.resultPath.property->match([ + pair(^InlineScript(source = $field->painlessIsNotEmpty()->literal(), params = newMap([])), $req); +} + + +function meta::external::store::elasticsearch::v7::pureToEs::painlessIsNotEmpty(field: TDSESDetail[1]): String[1] +{ + $field.resultPath.property->match([ text: TextProperty[1] | 'params[\'_source\'][\'%s\'] != null', any: Any[1] | 'doc[\'%s\'].size() != 0'; ])->format($field.path()); - - pair(^InlineScript(source = $script->literal(), params = newMap([])), $req); } function meta::external::store::elasticsearch::v7::pureToEs::processPainlessBooleanComparison(fe: FunctionExpression[1], initReq: State[1]): Pair[1] @@ -1439,12 +1463,12 @@ function meta::external::store::elasticsearch::v7::pureToEs::processPainlessBool any: Any[*] | 'params[\'%s\']' ])->format($param.first); - let script = '%s.toInstant().toEpochMilli() %s %s'->format([painlessExtractField($tdsESDetail), $operation, $paramScript]); + let script = '(%s && %s.toInstant().toEpochMilli() %s %s)'->format([painlessIsNotEmpty($tdsESDetail), painlessExtractField($tdsESDetail), $operation, $paramScript]); let literalOrExpression = $newValue->literalOrExpression(true)->toOne(); pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $literalOrExpression))), $param.second); }, {| - let script = '%s %s params[\'%s\']'->format([painlessExtractField($tdsESDetail), $operation, $param.first]); + let script = '(%s && %s %s params[\'%s\'])'->format([painlessIsNotEmpty($tdsESDetail), painlessExtractField($tdsESDetail), $operation, $param.first]); let literalOrExpression = $value->literalOrExpression(true)->toOne(); pair(^InlineScript(source = $script->literal(), params = newMap(pair($param.first, $literalOrExpression))), $param.second); } diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure index 7257c358512..271a369afae 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_misc.pure @@ -145,4 +145,12 @@ meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testSl $config->testTdsExpression(x | $x->sort('Title')->slice($var3, $var4)); $config->testTdsExpression(x | $x->sort('Title')->slice(3, $var4)); $config->testTdsExpression(x | $x->sort('Title')->slice($var3, 4)); +} + +function + <> + {doc.doc = 'Test rename column on Elasticsearch'} +meta::external::store::elasticsearch::executionTest::testCase::tds::misc::testRenameColumn(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x | $x->renameColumns(pair('Title', 'Titulo'))); } \ No newline at end of file diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure index 0bb015433d2..8089ed3ccf0 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/main/resources/core_elasticsearch_execution_test/elasticsearch_plan_test_project.pure @@ -43,6 +43,14 @@ meta::external::store::elasticsearch::executionTest::testCase::tds::project::mis $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | if($x.getDate('ReleaseDate') >= %1990-01-01, |'90s and newer', |'Older than 90s'), 'Bucket')])); } +function + <> + {doc.doc = 'Test projection on Elasticsearch with pure if expressions on nullable field'} +meta::external::store::elasticsearch::executionTest::testCase::tds::project::misc::testProjectExpressionOnNullable(config:TestConfig[1]):Boolean[1] +{ + $config->testTdsExpression(x|$x->extend([col(x: TDSRow[1] | $x.getNullableString('MPAA') == 'PG-13', 'Bucket')])); +} + function <> {doc.doc = 'Test projection on Elasticsearch with pure if expressions'} diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/Test_Pure_ElasticSearch_ExecutionPlan.java b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/Test_Pure_ElasticSearch_ExecutionPlan.java index 65409eeb950..bcc0b71b20e 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/Test_Pure_ElasticSearch_ExecutionPlan.java +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/src/test/java/org/finos/legend/engine/plan/execution/stores/elasticsearch/test/Test_Pure_ElasticSearch_ExecutionPlan.java @@ -29,7 +29,6 @@ public static TestSuite suite() TestSuite suite = new TestSuite(); if (DockerClientFactory.instance().isDockerAvailable()) { - System.setProperty("org.finos.legend.engine.plan.execution.stores.elasticsearch.test.password", "s3cret"); CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); suite.addTest(PureTestBuilderCompiled.buildSuite( TestCollection.collectTests(