SQL - assortment of updates (finos#2284)

- support parseDate in post tds flow - support getDecimal in post tds flow - support firstNotNull in tds filter -> where - support cross join - support date arithmetic on aggregated values
PrateekGarg-gs · Sep 21, 2023 · b06e1ee · b06e1ee
1 parent 83bc59e
commit b06e1ee
Show file tree

Hide file tree

Showing 9 changed files with 229 additions and 49 deletions.
diff --git a/...ore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/...ore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure
@@ -68,7 +68,9 @@ Class meta::relational::functions::pureToSqlQuery::PureFunctionToRelationalFunct
 {
 }
 
-
+Class meta::relational::functions::pureToSqlQuery::PureFunctionTDSToRelationalFunctionPair extends Pair<meta::pure::metamodel::function::Function<Any>, meta::pure::metamodel::function::Function<{->RelationalOperationElement[1]}>>
+{
+}
 
 Class meta::relational::functions::pureToSqlQuery::State
 {
@@ -2321,8 +2323,21 @@ function meta::relational::functions::pureToSqlQuery::processParseDate(f:Functio
 
 function meta::relational::functions::pureToSqlQuery::processFirstNotNull(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map<VariableExpression, ValueSpecification>[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List<ColumnGroup>[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1]
 {
-  fail('firstNotNull not supported outside of TDS context');
-  processNoOp($f, $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions);
+  let caller = $state.functionExpressionStack->first();
+  let error = 'firstNotNull not supported outside of TDS context';
+
+  //we check that the caller function is in the tabular space
+  $caller->match([
+    s:SimpleFunctionExpression[1] | $s->evaluateAndDeactivate().func.classifierGenericType.typeArguments.rawType->match([
+      f:FunctionType[1] | $f.parameters->first()->evaluateAndDeactivate().genericType.rawType == TabularDataSet,
+      a:Any[*] | fail($error)
+    ]),
+    a:Any[*] | fail($error)
+  ]);
+
+  let oldFunc = $f.func;
+  let newFunc = ^$oldFunc(functionName = 'coalesce');
+  processDynaFunction(^$f(func = $newFunc), $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions);
 }
 
 function meta::relational::functions::pureToSqlQuery::findAliasOrFail(columnName:String[1], select:SelectSQLQuery[1]):Alias[1]
@@ -4973,6 +4988,7 @@ function meta::relational::functions::pureToSqlQuery::processTdsLambda(mapFn:Val
                         let dispatch = [
                            pair('getNumber', | findColumnInTdsFromGetter($f, $a,$returnColumnName, $vars, $state)),
                            pair('getInteger', | findColumnInTdsFromGetter($f, $a,$returnColumnName, $vars, $state)),
+                           pair('getDecimal', | findColumnInTdsFromGetter($f, $a,$returnColumnName, $vars, $state)),
                            pair('getString', | findColumnInTdsFromGetter($f, $a,$returnColumnName, $vars, $state)),
                            pair('getFloat', | findColumnInTdsFromGetter($f, $a,$returnColumnName, $vars, $state)),
                            pair('getDate', | findColumnInTdsFromGetter($f, $a,$returnColumnName, $vars, $state)),
@@ -4990,25 +5006,42 @@ function meta::relational::functions::pureToSqlQuery::processTdsLambda(mapFn:Val
                                  |$func->toOne().second->cast(@meta::pure::metamodel::function::Function<{->RelationalOperationElement[*]}>)->eval(),
                                  |
                                     let supportedFunction = findSupportedFunction($f, $state.supportedFunctions, $state.contextBasedSupportedFunctions);
-
+
+
                                     if($supportedFunction->isNotEmpty(),
-                                       | if($f.func == extractEnumValue_Enumeration_1__String_1__T_1_,
-                                            | ^Literal(value=extractEnumValue($f, $currentPropertyMapping, $context)),
-                                            | if($f.func == meta::pure::functions::string::plus_String_MANY__String_1_,
-                                                 | ^JoinStrings(strings=$f.parametersValues->map(p|$p->processTdsLambda($a,$returnColumnName, $vars, $state, $currentPropertyMapping, $context))),
-                                                 | if ($f.func == meta::pure::functions::collection::in_Any_1__Any_MANY__Boolean_1_,
-                                                       | let arg1 = $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context);
-                                                         let arg2 = $f.parametersValues->at(1)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context);
-                                                         let dynaParams = $arg1->concatenate($arg2->match([ll:Literal[*] | ^LiteralList(values = $ll), r:RelationalOperationElement[*] | $r]));
-                                                         newDynaFunction($f.func.functionName->toOne(), $dynaParams);,
-                                                       | if ($supportedFunction == processNoOp_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_,
-                                                          | $f.parametersValues->at(0)->processTdsLambda($a,$returnColumnName, $vars, $state, $currentPropertyMapping, $context),
-                                                          | if ($f.func == meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_,
-                                                            | newDynaFunction('coalesce', $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context));,
-                                                            | newDynaFunction($f.func.functionName->toOne(), $f.parametersValues->map(p|$p->processTdsLambda($a,$returnColumnName, $vars, $state, $currentPropertyMapping, $context)))))
-                                                   )
-                                              )
-                                            ),
+                                       | 
+                                          let overrides = [
+                                            ^PureFunctionTDSToRelationalFunctionPair(first = extractEnumValue_Enumeration_1__String_1__T_1_, second = {| 
+                                              ^Literal(value=extractEnumValue($f, $currentPropertyMapping, $context))
+                                            }),
+                                            ^PureFunctionTDSToRelationalFunctionPair(first = plus_String_MANY__String_1_, second = {| 
+                                              ^JoinStrings(strings=$f.parametersValues->map(p|$p->processTdsLambda($a,$returnColumnName, $vars, $state, $currentPropertyMapping, $context)))
+                                            }),
+                                            ^PureFunctionTDSToRelationalFunctionPair(first = in_Any_1__Any_MANY__Boolean_1_, second = {| 
+                                              let arg1 = $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context);
+                                              let arg2 = $f.parametersValues->at(1)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context);
+                                              let dynaParams = $arg1->concatenate($arg2->match([ll:Literal[*] | ^LiteralList(values = $ll), r:RelationalOperationElement[*] | $r]));
+                                              newDynaFunction($f.func.functionName->toOne(), $dynaParams);                                   
+                                            }),
+                                            ^PureFunctionTDSToRelationalFunctionPair(first = meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_, second = {| 
+                                              newDynaFunction('coalesce', $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context))
+                                            }),
+                                            ^PureFunctionTDSToRelationalFunctionPair(first = parseDate_String_1__Date_1_, second = {|
+                                              let format = ^InstanceValue(multiplicity = PureOne, genericType = ^GenericType(rawType=String), values = 'YYYY-MM-DD HH24:MI:SS');
+                                              newDynaFunction('toTimestamp', [
+                                                $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)->toOne(),
+                                                ^Literal(value = 'YYYY-MM-DD HH24:MI:SS')
+                                              ]);                                        
+                                            })
+                                          ];
+
+                                          let override = $overrides->filter(o | $o.first == $f.func)->first();
+                                          if ($override->isNotEmpty(), 
+                                              | $override->toOne().second->eval(),
+                                              | if ($supportedFunction == processNoOp_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_,
+                                                      | $f.parametersValues->at(0)->processTdsLambda($a,$returnColumnName, $vars, $state, $currentPropertyMapping, $context),
+                                                      | newDynaFunction($f.func.functionName->toOne(), $f.parametersValues->map(p|$p->processTdsLambda($a,$returnColumnName, $vars, $state, $currentPropertyMapping, $context)))));,
+
                                        | fail('function ' + $f.func.name->makeString() + ' is not yet supported'); ^DynaFunction(name='fail');
                                        );
                          );                        

diff --git a/...onalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTDSExtend.pure b/...onalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTDSExtend.pure
@@ -254,6 +254,34 @@ function <<test.Test>> meta::relational::tests::tds::tdsExtend::testFirstNotNull
       $result->sqlRemoveFormatting());   
 }
 
+function <<test.Test>> meta::relational::tests::tds::tdsExtend::testParseDate():Boolean[1]
+{
+   let sql = meta::relational::functions::sqlstring::toSQLStringPretty(
+      |Person.all()
+         ->project(p|$p.firstName,'firstName')
+         ->extend([
+          col({row:TDSRow[1]| parseDate('2023-01-01')}, 'date')
+          ]),
+      simpleRelationalMapping, 
+      testRuntime(), meta::relational::extension::relationalExtensions());
+
+  assert($sql->contains('parsedatetime'));
+}
+
+function <<test.Test>> meta::relational::tests::tds::tdsExtend::testDecimal():Boolean[1]
+{
+   let result = meta::relational::functions::sqlstring::toSQLStringPretty(
+      |Person.all()
+         ->project(p|1d,'decimal')
+         ->extend([
+          col({row:TDSRow[1]| $row.getDecimal('decimal') + 1}, 'plus')
+          ]),
+      simpleRelationalMapping, 
+      testRuntime(), meta::relational::extension::relationalExtensions());
+
+  assertEquals('select 1 as "decimal", (1 + 1) as "plus" from personTable as "root"', $result->sqlRemoveFormatting());
+}
+
 
 // Alloy exclusion reason: 10. Tricky usage of variables
 function <<test.Test, test.ExcludeAlloy>> {test.excludePlatform = 'Java compiled'} meta::relational::tests::tds::tdsExtend::testExtendWithVariables1():Boolean[1]

diff --git a/...onalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTDSFilter.pure b/...onalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTDSFilter.pure
@@ -232,3 +232,21 @@ function <<test.Test>> meta::relational::tests::tds::tdsFilter::testFilterOnQuot
    let resWithQuotes = execute($queryWithQuotes, $mapping, $runtime, meta::relational::extension::relationalExtensions());
    assertEquals($expectedSql, $resWithQuotes->sqlRemoveFormatting());
 }
+
+function <<test.Test>> meta::relational::tests::tds::tdsFilter::testFirstNotNullFunction():Boolean[1]
+{
+   let result = execute(
+      |Person.all()
+         ->project(p|$p.firstName,'firstName')
+         ->filter(p | meta::pure::tds::extensions::firstNotNull([$p.getString('firstName'), 'N/A']) != 'N/A'),
+      simpleRelationalMapping, 
+      testRuntime(), meta::relational::extension::relationalExtensions());
+
+   assertSize($result.values.columns, 1);   
+
+   assertEquals('Peter,John,John,Anthony,Fabrice,Oliver,David',
+      $result.values.rows->map(r|$r.values->makeString('|'))->makeString(','));
+
+   assertEquals('select "root".FIRSTNAME as "firstName" from personTable as "root" where (coalesce("root".FIRSTNAME, \'N/A\') <> \'N/A\' OR coalesce("root".FIRSTNAME, \'N/A\') is null)', 
+      $result->sqlRemoveFormatting());   
+}
diff --git a/...l-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java b/...l-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java
@@ -969,28 +969,44 @@ public Node visitLogicalBinary(SqlBaseParser.LogicalBinaryContext context)
     public Node visitJoinRelation(SqlBaseParser.JoinRelationContext ctx)
     {
         Join join = new Join();
-
         join.left = (Relation) visit(ctx.left);
-        join.right = (Relation) visit(ctx.rightRelation);
-        join.type = getJoinType(ctx.joinType());
 
-        if (ctx.joinCriteria().ON() != null)
+        if (ctx.CROSS() != null)
         {
-            JoinOn joinOn = new JoinOn();
-            joinOn.expression = (Expression) visit(ctx.joinCriteria().booleanExpression());
-            join.criteria = joinOn;
+            join.right = (Relation) visit(ctx.right);
+            join.type = JoinType.CROSS;
+            return join;
         }
-        else if (ctx.joinCriteria().USING() != null)
+
+        if (ctx.NATURAL() != null)
         {
-            JoinUsing joinUsing = new JoinUsing();
-            joinUsing.columns = identsToStrings(ctx.joinCriteria().ident());
-            join.criteria = joinUsing;
+            join.right = (Relation) visit(ctx.right);
+            join.criteria = new NaturalJoin();
         }
         else
         {
-            throw new IllegalArgumentException("Unsupported join criteria");
+            join.right = (Relation) visit(ctx.rightRelation);
+            if (ctx.joinCriteria().ON() != null)
+            {
+                JoinOn joinOn = new JoinOn();
+                joinOn.expression = (Expression) visit(ctx.joinCriteria().booleanExpression());
+                join.criteria = joinOn;
+            }
+            else if (ctx.joinCriteria().USING() != null)
+            {
+                List<String> columns = identsToStrings(ctx.joinCriteria().ident());
+                JoinUsing joinUsing = new JoinUsing();
+                joinUsing.columns = columns;
+                join.criteria = joinUsing;
+            }
+            else
+            {
+                throw new IllegalArgumentException("Unsupported join criteria");
+            }
         }
 
+        join.type = getJoinType(ctx.joinType());
+
         return join;
     }
 

diff --git a/...mar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java b/...mar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java
@@ -53,7 +53,8 @@ public class SQLGrammarComposer
     private final MutableMap<JoinType, String> joins = UnifiedMap.newMapWith(
             Tuples.pair(JoinType.LEFT, "LEFT OUTER"),
             Tuples.pair(JoinType.RIGHT, "RIGHT OUTER"),
-            Tuples.pair(JoinType.INNER, "INNER")
+            Tuples.pair(JoinType.INNER, "INNER"),
+            Tuples.pair(JoinType.CROSS, "CROSS")
     );
 
     private final MutableMap<CurrentTimeType, String> currentTime = UnifiedMap.newMapWith(
@@ -298,7 +299,9 @@ public String visit(Join val)
                 String type = joins.get(val.type);
                 String left = val.left.accept(this);
                 String right = val.right.accept(this);
-                String criteria = val.criteria.accept(new JoinCriteriaVisitor<String>()
+                String natural = val.criteria instanceof NaturalJoin ? "NATURAL " : "";
+
+                String criteria = val.criteria != null ? val.criteria.accept(new JoinCriteriaVisitor<String>()
                 {
                     @Override
                     public String visit(JoinOn val)
@@ -311,9 +314,15 @@ public String visit(JoinUsing val)
                     {
                         return "USING (" + String.join(", ", val.columns) + ")";
                     }
-                });
 
-                return left + " " + type + " JOIN " + right + " " + criteria;
+                    @Override
+                    public String visit(NaturalJoin val)
+                    {
+                        return "";
+                    }
+                }) : "";
+
+                return left + " " + natural + type + " JOIN " + right + " " + criteria;
             }
 
             @Override

diff --git a/...st/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java b/...st/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java
@@ -25,6 +25,7 @@
 
 public class TestSQLRoundTrip
 {
+
     @Test
     public void testEmptyStatement()
     {
@@ -214,6 +215,18 @@ public void testJoinOnQualifiedAlias()
         check("SELECT myTable1.col, myTable2.col FROM myTable AS myTable1 LEFT OUTER JOIN myTableb AS myTable2 ON (myTable1.col = myTable2.col)");
     }
 
+    @Test
+    public void testNaturalJoin()
+    {
+        check("SELECT * FROM myTable NATURAL LEFT OUTER JOIN myTable2");
+    }
+
+    @Test
+    public void testCrossJoin()
+    {
+        check("SELECT * FROM myTable CROSS JOIN myTable1");
+    }
+
     @Test
     public void testUnionAll()
     {

diff --git a/...xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure b/...xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure
@@ -270,6 +270,10 @@ Class meta::external::query::sql::metamodel::JoinCriteria
 {
 }
 
+Class meta::external::query::sql::metamodel::NaturalJoin extends meta::external::query::sql::metamodel::JoinCriteria
+{
+}
+
 Class meta::external::query::sql::metamodel::JoinOn extends meta::external::query::sql::metamodel::JoinCriteria
 {
   expression: meta::external::query::sql::metamodel::Expression[1];
-Original file line number
+Diff line change
@@ Expand Up / @@ -270,6 +270,10 @@ Class meta::external::query::sql::metamodel::JoinCriteria @@
     {
     }
+    Class meta::external::query::sql::metamodel::NaturalJoin extends meta::external::query::sql::metamodel::JoinCriteria
+    {
+    }
     Class meta::external::query::sql::metamodel::JoinOn extends meta::external::query::sql::metamodel::JoinCriteria
     {
       expression: meta::external::query::sql::metamodel::Expression[1];
@@ Expand Down @@