From 5a43b2e62b6e64b68df3a93368177692b253e459 Mon Sep 17 00:00:00 2001 From: Jake Kim <60619826+jake-kim1@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:05:02 -0500 Subject: [PATCH] Implement write API for read/write from/to same connection (#3303) * Implement write API for read/write from/to same connection * Generalize processing of JSONElement return value --- .../toPureGraph/handlers/Handlers.java | 2 + .../src/main/resources/core.definition.json | 3 +- .../relation/functions/write/write.pure | 39 +++----- .../Test_JAVA_RelationFunction_PCT.java | 2 +- .../pct_relational.pure | 98 ++++++++++--------- ...lational_DuckDB_RelationFunctions_PCT.java | 5 +- ...t_Relational_H2_RelationFunctions_PCT.java | 5 +- ...tional_Postgres_RelationFunctions_PCT.java | 5 +- .../stores/relational/RelationalExecutor.java | 5 + .../RelationalExecutionNodeExecutor.java | 6 ++ .../executionPlan/nodes/SQLExecutionNode.java | 1 + .../relational/contract/storeContract.pure | 10 +- .../executionPlan/executionPlan.pure | 1 + .../relational/metamodel/metamodel.pure | 7 ++ .../mutation/relationalMutation.pure | 20 ++++ .../extension/extension_relational.pure | 3 +- .../models/executionPlan_relational.pure | 1 + .../relationalMappingExecution.pure | 3 +- .../sqlQueryToString/dbExtension.pure | 5 + 19 files changed, 130 insertions(+), 91 deletions(-) diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index 362156381c5..da9e718045d 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -1366,6 +1366,8 @@ private void registerTDS() register("meta::pure::tds::asc_String_1__SortInformation_1_", false, ps -> res("meta::pure::tds::SortInformation", "one")); register("meta::pure::tds::desc_String_1__SortInformation_1_", false, ps -> res("meta::pure::tds::SortInformation", "one")); + register("meta::pure::functions::relation::write_Relation_1__RelationElementAccessor_1__Integer_1_", true, ps -> res("Integer", "one")); + register(h("meta::pure::functions::relation::ascending_ColSpec_1__SortInfo_1_", false, ps -> res("meta::pure::functions::relation::SortInfo", "one"), ps -> Lists.fixedSize.of(ps.get(0)._genericType()._typeArguments().getFirst()), ps -> true)); register(h("meta::pure::functions::relation::descending_ColSpec_1__SortInfo_1_", false, ps -> res("meta::pure::functions::relation::SortInfo", "one"), ps -> Lists.fixedSize.of(ps.get(0)._genericType()._typeArguments().getFirst()), ps -> true)); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core.definition.json b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core.definition.json index cc98e87c94c..a88b3e70d42 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core.definition.json +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core.definition.json @@ -11,6 +11,7 @@ "platform_store_relational", "core_functions_standard", "core_functions_unclassified", - "core_functions_json" + "core_functions_json", + "core_functions_relation" ] } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/write/write.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/write/write.pure index eace90a8331..a0c12deecbc 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/write/write.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/write/write.pure @@ -22,31 +22,20 @@ native function <> meta::pure::fu function <> meta::pure::functions::relation::testWrite(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] { let expr = { - | - let targetTDS = meta::pure::metamodel::relation::newTDSRelationAccessor(#TDS - val,str,other - 1,aaa,a - #); - let rowsAdded = #TDS - val,str,other - 1,a,a - 3,ewe,b - 4,qw,c - 5,wwe,d - 6,weq,e - #->write($targetTDS); - $targetTDS->select(); + | #TDS + val,str,other + 1,a,a + 3,ewe,b + 4,qw,c + 5,wwe,d + 6,weq,e + #->select() + ->write( + #TDS + val,str,other + 1,aaa,a + #->newTDSRelationAccessor()); }; - let res = $f->eval($expr); - - assertEquals( '#TDS\n'+ - ' val,str,other\n'+ - ' 1,aaa,a\n'+ - ' 1,a,a\n'+ - ' 3,ewe,b\n'+ - ' 4,qw,c\n'+ - ' 5,wwe,d\n'+ - ' 6,weq,e\n'+ - '#', $res->toString()); + assertEquals(5, $res); } diff --git a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java index d572cda012c..12d32348206 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java +++ b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java @@ -137,7 +137,7 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration one("meta::pure::functions::relation::tests::composition::testWindowFunctionsAfterProject_Function_1__Boolean_1_", "\"meta::pure::functions::relation::sort_Relation_1__SortInfo_MANY__Relation_1_ is not supported yet!\""), // Write - one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"meta::pure::functions::relation::select_Relation_1__Relation_1_ is not supported yet!\"") + one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"meta::pure::functions::relation::write_Relation_1__RelationElementAccessor_1__Integer_1_ is not supported yet!\"") ); public static Test suite() diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure index 28aa66a71f1..6a505f422e4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure @@ -95,48 +95,49 @@ function meta::relational::tests::pct::testAdapterForRelationalExecution(f: print(if($debug.debug,|'\nResult JSON:\n'+$x+'\n',|'')); // ---- + let resultType = $f->functionReturnType().rawType; let json = $x->meta::json::parseJSON(); - - if ($json->cast(@JSONObject).keyValuePairs->filter(k|$k.key.value=='result').value->isEmpty(), - | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult)->toOne(); - let x= meta::protocols::pure::vX_X_X::invocation::execution::execute::processRelationalClassResult($res, $repocessed.second.mapping->toOne(), $repocessed.second.runtime, [], $extensions); - $x.first.values;, - | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult)->toOne(); - // processResult - let resultType = $f->functionReturnType().rawType; - if ($resultType->toOne()->instanceOf(PrimitiveType), - | let preResultFinal = $res.result.rows->at(0).values->at(0); - let resultFinal = - if ([ - pair(|$resultType->toOne()->subTypeOf(Date), |buildDate($preResultFinal->toString())), - pair(|$resultType->toOne()->subTypeOf(Decimal), |$preResultFinal->cast(@Number)->toDecimal()), - pair(|$resultType->toOne()->subTypeOf(Integer), |$preResultFinal->cast(@Number)->round()), - pair(|$resultType->toOne()->subTypeOf(Float), |$preResultFinal->cast(@Number)->toFloat()) - ], - | $preResultFinal->toOne() + if (!$json->instanceOf(JSONObject), + | $json->meta::json::fromJSON($resultType), + | if ($json->cast(@JSONObject).keyValuePairs->filter(k|$k.key.value=='result').value->isEmpty(), + | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult)->toOne(); + let x= meta::protocols::pure::vX_X_X::invocation::execution::execute::processRelationalClassResult($res, $repocessed.second.mapping->toOne(), $repocessed.second.runtime, [], $extensions); + $x.first.values;, + | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult)->toOne(); + // processResult + if ($resultType->toOne()->instanceOf(PrimitiveType), + | let preResultFinal = $res.result.rows->at(0).values->at(0); + let resultFinal = + if ([ + pair(|$resultType->toOne()->subTypeOf(Date), |buildDate($preResultFinal->toString())), + pair(|$resultType->toOne()->subTypeOf(Decimal), |$preResultFinal->cast(@Number)->toDecimal()), + pair(|$resultType->toOne()->subTypeOf(Integer), |$preResultFinal->cast(@Number)->round()), + pair(|$resultType->toOne()->subTypeOf(Float), |$preResultFinal->cast(@Number)->toFloat()) + ], + | $preResultFinal->toOne() + ); + print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->makeString(','),|'')); + $resultFinal;, + | let tdsString = $res.result.columns->joinStrings(',') + '\n' + + $res.result.rows->map(x| + range($x.values->size())->map(z | if($x.values->at($z) == TDSNull, + | let type = $res.builder->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::TDSBuilder).columns->at($z).type; + if ([ + pair(|$type == 'Integer', |-2147483648), + pair(|$type == 'String', |'null'), + pair(|$type == 'DateTime', |'') + ], + |fail();0; + );, + |$x.values->at($z) + ) + )->makeString(',') + )->joinStrings('\n'); + let resultFinal = $tdsString->stringToTDS(); + print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->toString(),|'')); + $resultFinal; ); - print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->makeString(','),|'')); - $resultFinal;, - | let tdsString = $res.result.columns->joinStrings(',') + '\n' + - $res.result.rows->map(x| - range($x.values->size())->map(z | if($x.values->at($z) == TDSNull, - | let type = $res.builder->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::TDSBuilder).columns->at($z).type; - if ([ - pair(|$type == 'Integer', |-2147483648), - pair(|$type == 'String', |'null'), - pair(|$type == 'DateTime', |'') - ], - |fail();0; - );, - |$x.values->at($z) - ) - )->makeString(',') - )->joinStrings('\n'); - let resultFinal = $tdsString->stringToTDS(); - print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->toString(),|'')); - $resultFinal; - ); - )->cast(@X)->toMultiplicity(@[o]); + ))->cast(@X)->toMultiplicity(@[o]); } function meta::relational::tests::pct::process::buildDate(dateStr:String[1]):Date[1] @@ -256,8 +257,6 @@ function meta::relational::tests::pct::process::printNew(a:Any[1]):String[1] function meta::relational::tests::pct::process::reprocess(f:Function[1], dbc:meta::external::store::relational::runtime::DatabaseConnection[1], debug: DebugContext[1]):Pair,ProcessingState>[1] { - let otherFuncs = $f->cast(@FunctionDefinition)->meta::relational::tests::pct::extractDependentFunctions(); - let ext = ^meta::pure::metamodel::serialization::grammar::Configuration ( fullPath = true, @@ -308,8 +307,7 @@ function meta::relational::tests::pct::process::reprocess(f:Function[1], db functionName = 'from', genericType = $returnGenericType, multiplicity = $returnMultiplicity, - parametersValues = - $reprocessed.current->toOne()->cast(@ValueSpecification) + parametersValues = $reprocessed.current->toOne()->cast(@ValueSpecification) ->concatenate(if ($reprocessed.mapping->isEmpty(), |[], |^InstanceValue @@ -356,7 +354,7 @@ function meta::relational::tests::pct::process::reprocess(f:Function[1], db meta::legend::executePlanAsJSON($planAsJson, []); // Update the database ------------ - + let otherFuncs = $reprocessed.current->cast(@FunctionExpression)->toOne()->meta::relational::tests::pct::extractDependentFunctions(); let code = '\n###Relational\n'+ $reprocessed.database->toOne()->printDatabase()+ if ($reprocessed.mapping->isEmpty(), @@ -370,7 +368,6 @@ function meta::relational::tests::pct::process::reprocess(f:Function[1], db ' '+meta::pure::metamodel::serialization::grammar::printFunctionDefinition($newFunction, $ext, '')+'\n'+ '}' ; - //println($code); let alloyCompiled = meta::legend::compile($code)->filter(x|$x.name == 'lambdaContainer__Any_MANY_')->cast(@FunctionDefinition).expressionSequence->cast(@InstanceValue)->evaluateAndDeactivate().values->toOne()->cast(@FunctionDefinition); @@ -448,7 +445,7 @@ import meta::core::runtime::*; import meta::relational::metamodel::relation::*; import meta::relational::metamodel::*; import meta::pure::metamodel::relation::*; -import meta::relational::tests::pct::process::*; +import meta::relational::tests::pct::process::*; import meta::pure::store::*; function meta::relational::tests::pct::process::reprocess(a:Any[1], state:ProcessingState[1]):ProcessingState[1] @@ -471,7 +468,12 @@ function meta::relational::tests::pct::process::reprocess(a:Any[1], state:Proces | let repro = $z.parametersValues->evaluateAndDeactivate()->map(x|$x->reprocess($state)); ^$state ( - current = ^$z(parametersValues = $repro.current->cast(@ValueSpecification))->evaluateAndDeactivate(), + current = ^$z(parametersValues = if ($z->cast(@FunctionExpression).func == write_Relation_1__RelationElementAccessor_1__Integer_1_, + | let reproCurrent = $repro.current->cast(@ValueSpecification); + let tds = $reproCurrent->at(1)->cast(@FunctionExpression).parametersValues->at(0); + $reproCurrent->at(0)->concatenate($tds);, + | $repro.current->cast(@ValueSpecification)) + )->evaluateAndDeactivate(), mapping = $repro.mapping->first(), replaced = $repro.replaced, csvs = $repro.csvs diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java index c0de2943445..f69194f95d7 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java @@ -39,10 +39,7 @@ public class Test_Relational_DuckDB_RelationFunctions_PCT extends PCTReportConfi // BUG: unsupported compositions one("meta::pure::functions::relation::tests::composition::test_Distinct_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column newCol must appear in the GROUP BY clause or be used in an aggregate function"), one("meta::pure::functions::relation::tests::composition::test_GroupBy_Distinct_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: Referenced table \"restrict__d#2\" not found!\nCandidate tables: \"tb"), - one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column \"newCol\" must appear in the GROUP BY clause or must be part of an aggregate function.\nEither add it to the GROUP BY list, or use \"ANY_VALUE(newCol)\" if the exact value of \"newCol\" is not important.\nLINE 2: select \"str\" as \"str\", \"newCol\" as \"newCol\" from (select \"tb"), - - // Write test contains multiple expressions - one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"") + one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column \"newCol\" must appear in the GROUP BY clause or must be part of an aggregate function.\nEither add it to the GROUP BY list, or use \"ANY_VALUE(newCol)\" if the exact value of \"newCol\" is not important.\nLINE 2: select \"str\" as \"str\", \"newCol\" as \"newCol\" from (select \"tb") ); public static Test suite() diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java index 96325d3cbe2..3ae29a21a34 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java @@ -50,10 +50,7 @@ public class Test_Relational_H2_RelationFunctions_PCT extends PCTReportConfigura one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema.tb"), one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""), - one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""), - - // Write test contains multiple expressions - one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"") + one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\"") ); public static Test suite() diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java index c078ea4006b..5f2e8211a22 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java @@ -54,10 +54,7 @@ public class Test_Relational_Postgres_RelationFunctions_PCT extends PCTReportCon // Postgres doesn't support asOf Join (May want to compensate with an OLAP equivalent if required one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""), - one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""), - - // Write test contains multiple expressions - one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"") + one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\"") ); public static Test suite() diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java index a53e2ba2555..ae58c407150 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java @@ -245,6 +245,11 @@ public Result execute(SQLExecutionNode node, Identity identity, ExecutionState e return new VoidRelationalResult(executionState.activities, node, connectionManagerConnection, identity, executionState.logSQLWithParamValues()); } + if (node.isMutationSQL != null && node.isMutationSQL) + { + return new SQLUpdateResult(executionState.activities, databaseType, connectionManagerConnection, node.connection, identity, tempTableList, executionState.getRequestContext()); + } + return new SQLExecutionResult(executionState.activities, node, databaseType, databaseTimeZone, connectionManagerConnection, identity, tempTableList, executionState.topSpan, executionState.getRequestContext(), executionState.logSQLWithParamValues()); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java index 383e3330eef..f7b4b37b342 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java @@ -324,6 +324,12 @@ else if (executionNode instanceof SQLExecutionNode) { scope.span().setTag("executedSql", ((SQLExecutionResult) result).getExecutedSql()); } + + if (result instanceof SQLUpdateResult) + { + return new ConstantResult(((SQLUpdateResult) result).getUpdateCount()); + } + return result; } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/SQLExecutionNode.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/SQLExecutionNode.java index b78002c5c25..0bcbb39263c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/SQLExecutionNode.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/SQLExecutionNode.java @@ -31,6 +31,7 @@ public class SQLExecutionNode extends ExecutionNode public DatabaseConnection connection; public List resultColumns = Collections.emptyList(); public Boolean isResultColumnsDynamic; + public Boolean isMutationSQL; @Override public T accept(ExecutionNodeVisitor executionNodeVisitor) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/contract/storeContract.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/contract/storeContract.pure index b8eb71fca5c..578d513cee3 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/contract/storeContract.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/contract/storeContract.pure @@ -203,7 +203,8 @@ function meta::relational::contract::supports(f:FunctionExpression[1]):Boolean[1 meta::pure::functions::math::sum_Float_MANY__Float_1_, meta::pure::functions::math::sum_Integer_MANY__Integer_1_, meta::pure::functions::math::sum_Number_MANY__Number_1_, - meta::pure::tds::func_String_1__FunctionDefinition_1__TdsOlapAggregation_1_ + meta::pure::tds::func_String_1__FunctionDefinition_1__TdsOlapAggregation_1_, + meta::pure::functions::relation::write_Relation_1__RelationElementAccessor_1__Integer_1_ ]->contains($f.func); ); } @@ -230,8 +231,13 @@ function meta::relational::contract::planExecution(sq:meta::pure::mapping::Store ), pair( |$func->in(meta::pure::mutation::mutationFunctions()), - | // Write Flow + | // Save Flow $sq->meta::relational::mutation::executionPlan::planMutationExecution($ext, $m->toOne(), $runtime->toOne(), $exeCtx, $extensions, $debug); + ), + pair( + |$func->in([meta::pure::functions::relation::write_Relation_1__RelationElementAccessor_1__Integer_1_]), + | // Write Flow + $sq->meta::relational::mutation::executionPlan::planWriteExecution($ext, $m, $runtime->toOne(), $exeCtx, $extensions, $debug); ) ] ,| diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure index e7fd2df771a..3d0d9f4094e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure @@ -71,6 +71,7 @@ Class meta::relational::mapping::SQLExecutionNode extends ExecutionNode connection: DatabaseConnection[1]; metadata: QueryMetadata[*]; isResultColumnsDynamic : Boolean[0..1]; + isMutationSQL: Boolean[0..1]; } Class meta::relational::mapping::RelationalSaveNode extends ExecutionNode diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/metamodel/metamodel.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/metamodel/metamodel.pure index ac2a2e4e986..70e5df6dc7a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/metamodel/metamodel.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/metamodel/metamodel.pure @@ -80,6 +80,13 @@ Class meta::relational::metamodel::LoadTableSQL extends SQLQuery absolutePathToFile : meta::relational::functions::pureToSqlQuery::metamodel::VarPlaceHolder[0..1]; } +Class meta::relational::metamodel::InsertIntoTableSQL extends SQLQuery +{ + table: Table[1]; + columnsToLoad : Column[*]; + selectQuery : RelationalOperationElement[1]; +} + function meta::relational::metamodel::loadWith(c:CreateTableSQL[1], absolutePathToFile:meta::relational::functions::pureToSqlQuery::metamodel::VarPlaceHolder[1]):SQLQuery[2] { [$c, ^LoadTableSQL(table = $c.table, absolutePathToFile=$absolutePathToFile, columnsToLoad=$c.table.columns->cast(@Column))]//TODO move function, asserts/constraints on columns diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/mutation/relationalMutation.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/mutation/relationalMutation.pure index 1dbc67d53b9..f30c4d6dd78 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/mutation/relationalMutation.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/mutation/relationalMutation.pure @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::functions::pureToSqlQuery::*; +import meta::pure::store::*; import meta::pure::router::clustering::*; import meta::pure::router::metamodel::clustering::*; import meta::pure::mapping::*; @@ -176,3 +178,21 @@ function meta::relational::mutation::executionPlan::planMutationExecution(sq: St columnValueGenerators = $columnValueGenerators ); } + +function meta::relational::mutation::executionPlan::planWriteExecution(sq: StoreQuery[1], ext: RoutedValueSpecification[0..1], mapping: Mapping[0..1], runtime: Runtime[1], exeCtx: meta::pure::runtime::ExecutionContext[1], extensions: Extension[*], debug: DebugContext[1]): ExecutionNode[1] +{ + let fe = $sq.vs->cast(@FunctionExpression); + let store = $sq.store->cast(@Database); + let dbConnection = $runtime.connectionByElement($store)->cast(@DatabaseConnection); + let table = $fe.parametersValues->at(1)->cast(@InstanceValue).values->cast(@RelationStoreAccessor).sourceElement->cast(@Table)->toOne(); + let queryExeCtx = if($exeCtx->instanceOf(RelationalExecutionContext),|$exeCtx,|[])->cast(@RelationalExecutionContext); + let originalQuery = $fe.parametersValues->at(0)->toOne()->toSQLQuery($mapping, $sq.inScopeVars, $queryExeCtx, $debug, $extensions); + let postProcessor = $originalQuery->postProcessSQLQuery($store, $ext, $mapping, $runtime, $exeCtx, $extensions); + let insertQuery = ^InsertIntoTableSQL( + selectQuery = $postProcessor.query, + table= $table + ); + ^$postProcessor( + query = $insertQuery + )->generateExecutionNodeForPostProcessedResult($sq, $store, $ext, $mapping, $runtime, $exeCtx, $debug, false, $extensions); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/extension/extension_relational.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/extension/extension_relational.pure index c33322de891..2fcf59d8203 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/extension/extension_relational.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/extension/extension_relational.pure @@ -125,7 +125,8 @@ function meta::protocols::pure::vX_X_X::extension::getRelationalExtension():meta onConnectionCloseCommitQuery = $rel.onConnectionCloseCommitQuery, onConnectionCloseRollbackQuery = $rel.onConnectionCloseRollbackQuery, connection = $rel.connection->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::connection::transformDatabaseConnection($extensions), - isResultColumnsDynamic = $rel.isResultColumnsDynamic + isResultColumnsDynamic = $rel.isResultColumnsDynamic, + isMutationSQL = $rel.isMutationSQL ), rel:meta::relational::mapping::RelationalSaveNode[1]| ^meta::protocols::pure::vX_X_X::metamodel::executionPlan::RelationalSaveNode( diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/models/executionPlan_relational.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/models/executionPlan_relational.pure index 5047c6ea80b..d69b156c5b2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/models/executionPlan_relational.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/protocols/pure/vX_X_X/models/executionPlan_relational.pure @@ -23,6 +23,7 @@ Class meta::protocols::pure::vX_X_X::metamodel::executionPlan::SQLExecutionNode resultColumns : meta::protocols::pure::vX_X_X::metamodel::executionPlan::SQLResultColumn[*]; connection : meta::protocols::pure::vX_X_X::metamodel::store::relational::connection::DatabaseConnection[1]; isResultColumnsDynamic : Boolean[0..1]; + isMutationSQL : Boolean[0..1]; } Class meta::protocols::pure::vX_X_X::metamodel::executionPlan::RelationalSaveNode extends meta::protocols::pure::vX_X_X::metamodel::executionPlan::ExecutionNode diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure index e332393c7e2..ee00ff40703 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure @@ -183,7 +183,8 @@ function meta::relational::mapping::generateSQLExecutionNode(query:SQLQuery[1], metadata = ^meta::relational::mapping::TableInfo(info = $query->meta::relational::milestoning::getQueryTableAliases()->map(a|$a.relationalElement)->cast(@Table)->removeDuplicatesBy(t|$t.schema.name+'.'+$t.name)->map(tab| ^meta::relational::mapping::TableIdentifier(schema=$tab.schema.name, table=$tab.name))), // the only case that we yield result with dynamic columns is when pivot() is used and no casting is done afterwards, resulting // in a select statement with no columns being specified - isResultColumnsDynamic = $query->match([sel:SelectSQLQuery[1] | if(isResultColumnsDynamic($sel), |true,|[]), a:Any[1] | []]) + isResultColumnsDynamic = $query->match([sel:SelectSQLQuery[1] | if(isResultColumnsDynamic($sel), |true,|[]), a:Any[1] | []]), + isMutationSQL = if($query->instanceOf(InsertIntoTableSQL), |true, |[]) ); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure index 5c4267fa19c..bb237780901 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure @@ -430,6 +430,11 @@ function meta::relational::functions::sqlQueryToString::processOperation(relatio ll:LiteralList[1] | $ll.values->map(e | $e->processOperation($dbConfig, $format, $generationState, $config, $extensions))->joinStrings('(', ', ', ')'), s:SelectSQLQuery[1]| $s->processSelectSQLQuery($sgc, false), u:UpsertSQLQuery[1]| $u->processUpsertSQLQuery($sgc), + i:InsertIntoTableSQL[1] | + ['INSERT INTO ', + $i.table->processOperation($sgc), + ' ', + $i.selectQuery->processOperation($sgc)]->joinStrings(''), u:UnionAll[1]| '('+$u.queries->map(q|$q->processSelectSQLQuery($sgc, false))->makeString(' UNION ALL ')+')', u:Union[1]| '('+$u.queries->map(q|$q->processSelectSQLQuery($sgc, false))->makeString(' UNION ')+')', f:FreeMarkerOperationHolder[1]|processFreeMarkerOperationHolder($f, $dbConfig, $format, $generationState, $config, false, $extensions),