Implement write API for read/write from/to same connection (#3303)

* Implement write API for read/write from/to same connection * Generalize processing of JSONElement return value
finos · Dec 13, 2024 · 5a43b2e · 5a43b2e
1 parent fb2b144
commit 5a43b2e
Show file tree

Hide file tree

Showing 19 changed files with 130 additions and 91 deletions.
diff --git a/...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
@@ -1366,6 +1366,8 @@ private void registerTDS()
         register("meta::pure::tds::asc_String_1__SortInformation_1_", false, ps -> res("meta::pure::tds::SortInformation", "one"));
         register("meta::pure::tds::desc_String_1__SortInformation_1_", false, ps -> res("meta::pure::tds::SortInformation", "one"));
 
+        register("meta::pure::functions::relation::write_Relation_1__RelationElementAccessor_1__Integer_1_", true, ps -> res("Integer", "one"));
+
         register(h("meta::pure::functions::relation::ascending_ColSpec_1__SortInfo_1_", false, ps -> res("meta::pure::functions::relation::SortInfo", "one"), ps -> Lists.fixedSize.of(ps.get(0)._genericType()._typeArguments().getFirst()), ps -> true));
         register(h("meta::pure::functions::relation::descending_ColSpec_1__SortInfo_1_", false, ps -> res("meta::pure::functions::relation::SortInfo", "one"), ps -> Lists.fixedSize.of(ps.get(0)._genericType()._typeArguments().getFirst()), ps -> true));
 

diff --git a/...e-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core.definition.json b/...e-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core.definition.json
@@ -11,6 +11,7 @@
     "platform_store_relational",
     "core_functions_standard",
     "core_functions_unclassified",
-    "core_functions_json"
+    "core_functions_json",
+    "core_functions_relation"
   ]
 }
diff --git a/...ation-pure/src/main/resources/core_functions_relation/relation/functions/write/write.pure b/...ation-pure/src/main/resources/core_functions_relation/relation/functions/write/write.pure
@@ -22,31 +22,20 @@ native function <<PCT.function, functionType.SideEffectFunction>> meta::pure::fu
 function <<PCT.test>> meta::pure::functions::relation::testWrite<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
 {
     let expr = {
-               |
-                 let targetTDS = meta::pure::metamodel::relation::newTDSRelationAccessor(#TDS
-                   val,str,other
-                   1,aaa,a
-                 #);
-                 let rowsAdded = #TDS
-                      val,str,other
-                      1,a,a
-                      3,ewe,b
-                      4,qw,c
-                      5,wwe,d
-                      6,weq,e
-                  #->write($targetTDS);
-                  $targetTDS->select();
+               |  #TDS
+                    val,str,other
+                    1,a,a
+                    3,ewe,b
+                    4,qw,c
+                    5,wwe,d
+                    6,weq,e
+                  #->select()
+                  ->write(
+                  #TDS
+                    val,str,other
+                    1,aaa,a
+                  #->newTDSRelationAccessor());
               };
-
     let res =  $f->eval($expr);
-
-    assertEquals( '#TDS\n'+
-                  '   val,str,other\n'+
-                  '   1,aaa,a\n'+
-                  '   1,a,a\n'+
-                  '   3,ewe,b\n'+
-                  '   4,qw,c\n'+
-                  '   5,wwe,d\n'+
-                  '   6,weq,e\n'+
-                  '#', $res->toString());
+    assertEquals(5, $res);
 }
diff --git a/...a/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java b/...a/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java
@@ -137,7 +137,7 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration
             one("meta::pure::functions::relation::tests::composition::testWindowFunctionsAfterProject_Function_1__Boolean_1_", "\"meta::pure::functions::relation::sort_Relation_1__SortInfo_MANY__Relation_1_ is not supported yet!\""),
 
             // Write
-            one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"meta::pure::functions::relation::select_Relation_1__Relation_1_ is not supported yet!\"")
+            one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"meta::pure::functions::relation::write_Relation_1__RelationElementAccessor_1__Integer_1_ is not supported yet!\"")
     );
 
     public static Test suite()

diff --git a/...tionalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure b/...tionalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure
@@ -95,48 +95,49 @@ function meta::relational::tests::pct::testAdapterForRelationalExecution<X|o>(f:
     print(if($debug.debug,|'\nResult JSON:\n'+$x+'\n',|''));
     // ----
 
+    let resultType = $f->functionReturnType().rawType;
     let json = $x->meta::json::parseJSON();
-
-    if ($json->cast(@JSONObject).keyValuePairs->filter(k|$k.key.value=='result').value->isEmpty(),
-      | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult)->toOne();
-        let x= meta::protocols::pure::vX_X_X::invocation::execution::execute::processRelationalClassResult($res, $repocessed.second.mapping->toOne(), $repocessed.second.runtime, [], $extensions);
-        $x.first.values;,
-      | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult)->toOne();
-        // processResult
-        let resultType = $f->functionReturnType().rawType;
-        if ($resultType->toOne()->instanceOf(PrimitiveType),
-          | let preResultFinal = $res.result.rows->at(0).values->at(0);
-            let resultFinal = 
-              if ([
-                    pair(|$resultType->toOne()->subTypeOf(Date), |buildDate($preResultFinal->toString())),
-                    pair(|$resultType->toOne()->subTypeOf(Decimal), |$preResultFinal->cast(@Number)->toDecimal()),
-                    pair(|$resultType->toOne()->subTypeOf(Integer), |$preResultFinal->cast(@Number)->round()),
-                    pair(|$resultType->toOne()->subTypeOf(Float), |$preResultFinal->cast(@Number)->toFloat())
-                  ],
-                | $preResultFinal->toOne()
+    if (!$json->instanceOf(JSONObject),
+      | $json->meta::json::fromJSON($resultType),
+      | if ($json->cast(@JSONObject).keyValuePairs->filter(k|$k.key.value=='result').value->isEmpty(),
+            | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalClassResult)->toOne();
+              let x= meta::protocols::pure::vX_X_X::invocation::execution::execute::processRelationalClassResult($res, $repocessed.second.mapping->toOne(), $repocessed.second.runtime, [], $extensions);
+              $x.first.values;,
+            | let res = $json->meta::json::fromJSON(meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult, ^meta::json::ExtendedJSONDeserializationConfig(typeKeyName='__TYPE', failOnUnknownProperties=true, nullReplacementInArray=TDSNull))->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::RelationalTDSResult)->toOne();
+              // processResult
+              if ($resultType->toOne()->instanceOf(PrimitiveType),
+                | let preResultFinal = $res.result.rows->at(0).values->at(0);
+                  let resultFinal = 
+                    if ([
+                          pair(|$resultType->toOne()->subTypeOf(Date), |buildDate($preResultFinal->toString())),
+                          pair(|$resultType->toOne()->subTypeOf(Decimal), |$preResultFinal->cast(@Number)->toDecimal()),
+                          pair(|$resultType->toOne()->subTypeOf(Integer), |$preResultFinal->cast(@Number)->round()),
+                          pair(|$resultType->toOne()->subTypeOf(Float), |$preResultFinal->cast(@Number)->toFloat())
+                        ],
+                      | $preResultFinal->toOne()
+                    );
+                  print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->makeString(','),|''));
+                  $resultFinal;,
+                | let tdsString = $res.result.columns->joinStrings(',') + '\n' +
+                                  $res.result.rows->map(x|
+                                      range($x.values->size())->map(z | if($x.values->at($z) == TDSNull,
+                                                                            | let type = $res.builder->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::TDSBuilder).columns->at($z).type;
+                                                                              if ([
+                                                                                    pair(|$type == 'Integer', |-2147483648),
+                                                                                    pair(|$type == 'String', |'null'),
+                                                                                    pair(|$type == 'DateTime', |'')
+                                                                                  ],
+                                                                                  |fail();0;
+                                                                              );,
+                                                                            |$x.values->at($z)
+                                                                        )
+                                                                )->makeString(',')
+                                      )->joinStrings('\n');
+                    let resultFinal = $tdsString->stringToTDS();
+                    print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->toString(),|''));
+                    $resultFinal;
               );
-            print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->makeString(','),|''));
-            $resultFinal;,
-          | let tdsString = $res.result.columns->joinStrings(',') + '\n' +
-                            $res.result.rows->map(x|
-                                range($x.values->size())->map(z | if($x.values->at($z) == TDSNull,
-                                                                      | let type = $res.builder->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::TDSBuilder).columns->at($z).type;
-                                                                        if ([
-                                                                              pair(|$type == 'Integer', |-2147483648),
-                                                                              pair(|$type == 'String', |'null'),
-                                                                              pair(|$type == 'DateTime', |'')
-                                                                            ],
-                                                                            |fail();0;
-                                                                        );,
-                                                                      |$x.values->at($z)
-                                                                  )
-                                                          )->makeString(',')
-                                )->joinStrings('\n');
-              let resultFinal = $tdsString->stringToTDS();
-              print(if($debug.debug,|'\nResult Final:\n'+$resultFinal->toString(),|''));
-              $resultFinal;
-        );
-    )->cast(@X)->toMultiplicity(@[o]);
+    ))->cast(@X)->toMultiplicity(@[o]);
 }
 
 function meta::relational::tests::pct::process::buildDate(dateStr:String[1]):Date[1]
@@ -256,8 +257,6 @@ function meta::relational::tests::pct::process::printNew(a:Any[1]):String[1]
 
 function meta::relational::tests::pct::process::reprocess(f:Function<Any>[1], dbc:meta::external::store::relational::runtime::DatabaseConnection[1], debug: DebugContext[1]):Pair<FunctionDefinition<Any>,ProcessingState>[1]
 { 
-  let otherFuncs = $f->cast(@FunctionDefinition<Any>)->meta::relational::tests::pct::extractDependentFunctions();
-
   let ext = ^meta::pure::metamodel::serialization::grammar::Configuration
             (
               fullPath = true,
@@ -308,8 +307,7 @@ function meta::relational::tests::pct::process::reprocess(f:Function<Any>[1], db
                                                           functionName = 'from',
                                                           genericType = $returnGenericType,
                                                           multiplicity = $returnMultiplicity,                                                        
-                                                          parametersValues = 
-                                                                                $reprocessed.current->toOne()->cast(@ValueSpecification)
+                                                          parametersValues =    $reprocessed.current->toOne()->cast(@ValueSpecification)
                                                                                 ->concatenate(if ($reprocessed.mapping->isEmpty(),
                                                                                               |[],
                                                                                               |^InstanceValue
@@ -356,7 +354,7 @@ function meta::relational::tests::pct::process::reprocess(f:Function<Any>[1], db
 
             meta::legend::executePlanAsJSON($planAsJson, []);
             // Update the database ------------
-
+            let otherFuncs = $reprocessed.current->cast(@FunctionExpression)->toOne()->meta::relational::tests::pct::extractDependentFunctions();
             let code =  '\n###Relational\n'+
                         $reprocessed.database->toOne()->printDatabase()+
                         if ($reprocessed.mapping->isEmpty(),
@@ -370,7 +368,6 @@ function meta::relational::tests::pct::process::reprocess(f:Function<Any>[1], db
                         '  '+meta::pure::metamodel::serialization::grammar::printFunctionDefinition($newFunction, $ext, '')+'\n'+
                         '}'
                         ;
-
             //println($code);
 
             let alloyCompiled = meta::legend::compile($code)->filter(x|$x.name == 'lambdaContainer__Any_MANY_')->cast(@FunctionDefinition<Any>).expressionSequence->cast(@InstanceValue)->evaluateAndDeactivate().values->toOne()->cast(@FunctionDefinition<Any>);
@@ -448,7 +445,7 @@ import meta::core::runtime::*;
 import meta::relational::metamodel::relation::*;
 import meta::relational::metamodel::*;
 import meta::pure::metamodel::relation::*;
-import  meta::relational::tests::pct::process::*;
+import meta::relational::tests::pct::process::*;
 import meta::pure::store::*;
 
 function meta::relational::tests::pct::process::reprocess(a:Any[1], state:ProcessingState[1]):ProcessingState[1]
@@ -471,7 +468,12 @@ function meta::relational::tests::pct::process::reprocess(a:Any[1], state:Proces
                                     | let repro = $z.parametersValues->evaluateAndDeactivate()->map(x|$x->reprocess($state));
                                       ^$state
                                       (
-                                          current = ^$z(parametersValues = $repro.current->cast(@ValueSpecification))->evaluateAndDeactivate(),
+                                          current = ^$z(parametersValues = if ($z->cast(@FunctionExpression).func == write_Relation_1__RelationElementAccessor_1__Integer_1_,
+                                                                              | let reproCurrent = $repro.current->cast(@ValueSpecification);
+                                                                                let tds = $reproCurrent->at(1)->cast(@FunctionExpression).parametersValues->at(0);
+                                                                                $reproCurrent->at(0)->concatenate($tds);,
+                                                                              | $repro.current->cast(@ValueSpecification))
+                                                       )->evaluateAndDeactivate(),
                                           mapping = $repro.mapping->first(),
                                           replaced = $repro.replaced,
                                           csvs = $repro.csvs

diff --git a/...ution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java b/...ution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_RelationFunctions_PCT.java
@@ -39,10 +39,7 @@ public class Test_Relational_DuckDB_RelationFunctions_PCT extends PCTReportConfi
             // BUG: unsupported compositions
             one("meta::pure::functions::relation::tests::composition::test_Distinct_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column newCol must appear in the GROUP BY clause or be used in an aggregate function"),
             one("meta::pure::functions::relation::tests::composition::test_GroupBy_Distinct_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: Referenced table \"restrict__d#2\" not found!\nCandidate tables: \"tb"),
-            one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column \"newCol\" must appear in the GROUP BY clause or must be part of an aggregate function.\nEither add it to the GROUP BY list, or use \"ANY_VALUE(newCol)\" if the exact value of \"newCol\" is not important.\nLINE 2: select \"str\" as \"str\", \"newCol\" as \"newCol\" from (select \"tb"),
-
-            // Write test contains multiple expressions
-            one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"")
+            one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Binder Error: column \"newCol\" must appear in the GROUP BY clause or must be part of an aggregate function.\nEither add it to the GROUP BY list, or use \"ANY_VALUE(newCol)\" if the exact value of \"newCol\" is not important.\nLINE 2: select \"str\" as \"str\", \"newCol\" as \"newCol\" from (select \"tb")
     );
 
     public static Test suite()

diff --git a/...lan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java b/...lan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java
@@ -50,10 +50,7 @@ public class Test_Relational_H2_RelationFunctions_PCT extends PCTReportConfigura
             one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema.tb"),
 
             one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""),
-            one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""),
-
-            // Write test contains multiple expressions
-            one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"")
+            one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\"")
         );
 
     public static Test suite()

diff --git a/...n/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java b/...n/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java
@@ -54,10 +54,7 @@ public class Test_Relational_Postgres_RelationFunctions_PCT extends PCTReportCon
 
             // Postgres doesn't support asOf Join (May want to compensate with an OLAP equivalent if required
             one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""),
-            one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""),
-
-            // Write test contains multiple expressions
-            one("meta::pure::functions::relation::testWrite_Function_1__Boolean_1_", "\"Cannot cast a collection of size 3 to multiplicity [1]\"")
+            one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\"")
             );
 
     public static Test suite()