Legend SQL - handle tableToTDS functions (#2259)

* Legend SQL - handle tableToTDS functions * Handle missing rawType case
finos · Sep 12, 2023 · f01564c · f01564c
1 parent cb50108
commit f01564c
Show file tree

Hide file tree

Showing 9 changed files with 76 additions and 10 deletions.
diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml
@@ -517,6 +517,13 @@
         </dependency>
         <!-- GraphQL -->
 
+        <!-- SQL -->
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-sql-grammar-integration</artifactId>
+        </dependency>
+        <!-- SQL -->
+
         <!-- DAML -->
         <dependency>
             <groupId>org.finos.legend.engine</groupId>

diff --git a/...rc/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java b/...rc/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java
@@ -73,6 +73,8 @@
 import org.finos.legend.engine.language.snowflakeApp.compiler.toPureGraph.SnowflakeAppCompilerExtension;
 import org.finos.legend.engine.language.snowflakeApp.grammar.from.SnowflakeAppGrammarParserExtension;
 import org.finos.legend.engine.language.snowflakeApp.grammar.to.SnowflakeAppGrammarComposer;
+import org.finos.legend.engine.language.sql.grammar.integration.SQLGrammarParserExtension;
+import org.finos.legend.engine.language.sql.grammar.integration.SQLPureGrammarComposerExtension;
 import org.finos.legend.engine.language.stores.elasticsearch.v7.from.ElasticsearchGrammarParserExtension;
 import org.finos.legend.engine.protocol.mongodb.schema.metamodel.MongoDBPureProtocolExtension;
 import org.finos.legend.pure.code.core.ElasticsearchPureCoreExtension;
@@ -282,6 +284,7 @@ protected Iterable<? extends Class<? extends PureProtocolExtension>> getExpected
                 .with(org.finos.legend.engine.protocol.pure.v1.AuthenticationProtocolExtension.class)
                 .with(org.finos.legend.engine.protocol.pure.v1.TextProtocolExtension.class)
                 .with(org.finos.legend.engine.language.graphQL.grammar.integration.GraphQLPureProtocolExtension.class)
+                .with(org.finos.legend.engine.language.sql.grammar.integration.SQLPureProtocolExtension.class)
                 .with(org.finos.legend.engine.protocol.store.elasticsearch.v7.ElasticsearchV7ProtocolExtension.class)
                 .with(org.finos.legend.engine.protocol.mongodb.schema.metamodel.MongoDBPureProtocolExtension.class)
                 ;
@@ -318,6 +321,7 @@ protected Iterable<? extends Class<? extends PureGrammarParserExtension>> getExp
                 .with(ServiceParserExtension.class)
                 .with(AuthenticationGrammarParserExtension.class)
                 .with(GraphQLGrammarParserExtension.class)
+                .with(SQLGrammarParserExtension.class)
                 .with(ServiceStoreGrammarParserExtension.class)
                 .with(TextParserExtension.class)
                 .with(ElasticsearchGrammarParserExtension.class)
@@ -349,6 +353,7 @@ protected Iterable<? extends Class<? extends PureGrammarComposerExtension>> getE
                 .with(ServiceGrammarComposerExtension.class)
                 .with(ServiceStoreGrammarComposerExtension.class)
                 .with(GraphQLPureGrammarComposerExtension.class)
+                .with(SQLPureGrammarComposerExtension.class)
                 .with(AuthenticationGrammarComposerExtension.class)
                 .with(TextGrammarComposerExtension.class)
                 .with(ElasticsearchGrammarComposerExtension.class)
@@ -380,6 +385,7 @@ protected Iterable<? extends Class<? extends CompilerExtension>> getExpectedComp
                 .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.RedshiftCompilerExtension.class)
                 .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.DatabricksCompilerExtension.class)
                 .with(org.finos.legend.engine.language.graphQL.grammar.integration.GraphQLCompilerExtension.class)
+                .with(org.finos.legend.engine.language.sql.grammar.integration.SQLCompilerExtension.class)
                 .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.ServiceStoreCompilerExtension.class)
                 .with(org.finos.legend.engine.language.pure.dsl.authentication.compiler.toPureGraph.AuthenticationCompilerExtension.class)
                 .with(org.finos.legend.engine.language.stores.elasticsearch.v7.compiler.ElasticsearchCompilerExtension.class)
@@ -479,6 +485,7 @@ protected Iterable<String> getExpectedCodeRepositories()
                 .with("core_external_format_xml")
                 .with("core_external_query_graphql")
                 .with("core_external_query_graphql_metamodel")
+                .with("core_external_query_sql_metamodel")
                 .with("core_function_activator")
                 .with("core_external_compiler")
                 .with("core_persistence")

diff --git a/...ode/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure b/...ode/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure
@@ -16,7 +16,8 @@ import meta::pure::tds::schema::*;
 
 function meta::pure::tds::schema::resolveSchema(f : FunctionDefinition<Any>[1], extensions:meta::pure::extension::Extension[*]) : TDSColumn[*]
 {
-   assert($f->functionReturnType().rawType == TabularDataSet, 'function must return TabularDataSet');
+   let returnType = $f->functionReturnType().rawType;
+   assert($returnType->isNotEmpty() && $returnType->toOne()->subTypeOf(TabularDataSet), 'function must return TabularDataSet');
    resolveSchemaImpl($f, $extensions).columns;
 }
 
@@ -270,7 +271,7 @@ function <<access.private>> meta::pure::tds::schema::resolveSchemaImpl(fe : Func
                | $tdsSchema.rename($renameMapping->toOneMany())
                );
 
-         }) 
+         })
       ])
       ->concatenate($extensions.tdsSchema_resolveSchemaImpl->map(e|$e->eval($fe, $openVars, $extensions)))
       ->concatenate([

diff --git a/...ationalStore-pure/src/main/resources/core_relational/relational/extensions/extension.pure b/...ationalStore-pure/src/main/resources/core_relational/relational/extensions/extension.pure
@@ -200,6 +200,10 @@ function meta::relational::extension::relationalExtension() : meta::pure::extens
                                                          }
                                                     )
                                                 )
+                                             )->concatenate(
+                                                pair(tableToTDS_Table_1__TableTDS_1_->cast(@Function<Any>), {|
+                                                    createSchemaState($fe->reactivate()->cast(@TabularDataSet).columns);
+                                                })                                              
                                              )
                                     },
       testExtension_testedBy = {allReferenceUses:ReferenceUsage[*], extensions:Extension[*] | {soFarr:TestedByResult[1]| $allReferenceUses.owner->filter(o|$o->instanceOf(Database))->cast(@Database)->fold({db,tbr|$db->testedBy($tbr, $extensions)}, $soFarr)}},
@@ -208,6 +212,8 @@ function meta::relational::extension::relationalExtension() : meta::pure::extens
 }
 
 
+
+
 //Helper Functions
 
 function meta::pure::router::clustering::getResolvedStore(rr: meta::relational::mapping::RootRelationalInstanceSetImplementation[*], mapping: Mapping[1]):Store[*]

diff --git a/...onalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsSchema.pure b/...onalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsSchema.pure
@@ -73,6 +73,20 @@ function <<test.Test>> meta::relational::tds::schema::tests::resolveSchemaTest()
             ->sort(asc('first_name'))
    }); 
 
+   assertSchemaRoundTripEquality(
+      [
+         ^TDSColumn(name = 'ID', offset= 0, type = Integer), 
+         ^TDSColumn(name = 'FIRSTNAME', offset = 1, type = String),
+         ^TDSColumn(name = 'LASTNAME', offset = 2, type = String),
+         ^TDSColumn(name = 'AGE', offset = 3, type = Integer),
+         ^TDSColumn(name = 'ADDRESSID', offset = 4, type = Integer),
+         ^TDSColumn(name = 'FIRMID', offset = 5, type = Integer),
+         ^TDSColumn(name = 'MANAGERID', offset = 6, type = Integer)
+      ], 
+      {|
+      tableToTDS(meta::relational::functions::database::tableReference(meta::relational::tests::db,'default','personTable'))
+   });    
+
   assertSchemaRoundTripEquality({|
          Trade.all()
             ->groupBy([x|$x.date->adjust(0, DurationUnit.DAYS)], 

diff --git a/...ine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure b/...ine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure
@@ -60,6 +60,11 @@ Class meta::external::query::sql::transformation::queryToPure::PlanGenerationRes
   arguments: PlanParameter[*];
 }
 
+function meta::external::query::sql::transformation::queryToPure::emptyMapping():Mapping[1]
+{
+  ^Mapping(package = meta::external::query::sql::transformation::queryToPure, name = 'sqlDummy')
+}
+
 function meta::external::query::sql::transformation::queryToPure::getPlanResultFromSQL(
                                                                           sources: SQLSource[*],
                                                                           query: meta::external::query::sql::metamodel::Node[1],
@@ -96,7 +101,7 @@ function meta::external::query::sql::transformation::queryToPure::getPlan(
                                                                   ):meta::pure::executionPlan::ExecutionPlan[1]
 {
   //NOTE: we should remove this once issues with enum mappings in plan transformation are fixed.
-  let mapping = if ($sources.mapping->isNotEmpty(), | $sources.mapping->removeDuplicates()->toOneMany()->mergeMappings(), | ^Mapping(package = meta::external::query::sql::transformation::queryToPure, name = 'sqlDummy'));
+  let mapping = if ($sources.mapping->isNotEmpty(), | $sources.mapping->removeDuplicates()->toOneMany()->mergeMappings(), | emptyMapping());
   let runtime = if ($sources.runtime->isNotEmpty(), | $sources.runtime->removeDuplicates({t, t2 | meta::pure::runtime::runtimeEquality($t, $t2, $extensions)})->toOneMany()->mergeRuntimes(), | ^Runtime());
   let lambda = $context.lambda()->meta::pure::router::preeval::preval($extensions);
 
@@ -1073,12 +1078,14 @@ let executionContext = if ($source.executionOptions->isNotEmpty(),
                         | $source.executionContext);
 
   if ($context.scopeWithFrom->isFalse(), | $context, |
-    let scopedExpression = if ($source.mapping->isNotEmpty() && $source.runtime->isNotEmpty(),
+    let mapping = if ($source.mapping->isEmpty(), | emptyMapping(), | $source.mapping->toOne());
+
+    let scopedExpression = if ($source.runtime->isNotEmpty(),
       | if ($executionContext->isNotEmpty(),
           | sfe(meta::pure::mapping::from_TabularDataSet_1__Mapping_1__Runtime_1__ExecutionContext_1__TabularDataSet_1_,
-                [$expression, iv($source.mapping->toOne()), iv($source.runtime->toOne()), iv($executionContext)]),
+                [$expression, iv($mapping), iv($source.runtime->toOne()), iv($executionContext)]),
           | sfe(meta::pure::mapping::from_TabularDataSet_1__Mapping_1__Runtime_1__TabularDataSet_1_,
-                [$expression, iv($source.mapping->toOne()), iv($source.runtime->toOne())])),
+                [$expression, iv($mapping->toOne()), iv($source.runtime->toOne())])),
       | $expression);
 
     ^$context(expression = $scopedExpression);

diff --git a/...xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/shared.pure b/...xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/shared.pure
@@ -196,6 +196,11 @@ function meta::external::query::sql::transformation::queryToPure::tests::service
 }
 
 function meta::external::query::sql::transformation::queryToPure::tests::createService(pattern:String[1], f:FunctionDefinition<Any>[1]): Service[1]
+{
+  createService($pattern, $f, dummyMapping, dummyRuntime())
+}
+
+function meta::external::query::sql::transformation::queryToPure::tests::createService(pattern:String[1], f:FunctionDefinition<Any>[1], mapping:Mapping[0..1], runtime:Runtime[0..1]): Service[1]
 {
   ^Service
   (
@@ -206,8 +211,8 @@ function meta::external::query::sql::transformation::queryToPure::tests::createS
     execution = ^PureSingleExecution
     (
       func = $f,
-      mapping = dummyMapping,
-      runtime = dummyRuntime()
+      mapping = $mapping,
+      runtime = $runtime
     ),
     test = ^SingleExecutionTest(
       data = '',

diff --git a/...pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure b/...pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure
@@ -1423,6 +1423,17 @@ function <<test.Test>> meta::external::query::sql::transformation::queryToPure::
   assertLambdaJSONEquals($expected, $sqlTransformContext.lambda());
 }
 
+function <<test.Test>> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingNoMapping():Boolean[1]
+{
+  let sqlString = 'SELECT * FROM service."/service/service1"';
+  let mapping = meta::external::query::sql::transformation::queryToPure::emptyMapping();
+
+  let sqlTransformContext = $sqlString->processQuery(serviceToSource(createService('/service/service1', | FlatInput.all()->project(x | $x.stringIn, 'String'), [], dummyRuntime())), true);
+  let expected = {| FlatInput.all()->project(x | $x.stringIn, 'String')->from($mapping, dummyRuntime())}->meta::pure::router::preeval::preval(sqlExtensions());
+
+  assertLambdaJSONEquals($expected, $sqlTransformContext.lambda());
+}
+
 function <<test.Test>> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingSimpleNested():Boolean[1]
 {
   let sqlString = 'SELECT * FROM (SELECT Integer FROM service."/service/service1")';

diff --git a/...uery/src/main/java/org/finos/legend/engine/query/sql/api/sources/SQLSourceTranslator.java b/...uery/src/main/java/org/finos/legend/engine/query/sql/api/sources/SQLSourceTranslator.java
@@ -44,8 +44,16 @@ private Root_meta_external_query_sql_transformation_queryToPure_SQLSource transl
 
         compiled._type(source.getType());
         compiled._func(HelperValueSpecificationBuilder.buildLambda(source.getFunc(), pureModel.getContext()));
-        compiled._mapping(pureModel.getMapping(source.getMapping()));
-        compiled._runtime(HelperRuntimeBuilder.buildPureRuntime(source.getRuntime(), pureModel.getContext()));
+
+        if (source.getMapping() != null)
+        {
+            compiled._mapping(pureModel.getMapping(source.getMapping()));
+        }
+
+        if (source.getRuntime() != null)
+        {
+            compiled._runtime(HelperRuntimeBuilder.buildPureRuntime(source.getRuntime(), pureModel.getContext()));
+        }
 
         if (source.getExecutionOptions() != null)
         {