diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java index 05e544d6593..cde29dbdd84 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java @@ -101,6 +101,10 @@ public MutableList validate(Identity identity, { MutableList errors = Lists.mutable.empty(); SnowflakeAppContent content = (SnowflakeAppContent)artifact.content; + if (content.applicationName.trim().equals("")) + { + errors.add(new SnowflakeAppError("Application name cannot be empty")); + } if (!content.sqlExpressions.isEmpty()) { int size = content.sqlExpressions.select(e -> !e.toLowerCase().endsWith("to role public;")).size(); diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java index ab7d2c552e8..3912951e69d 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java @@ -166,7 +166,10 @@ public MutableList generateStatements(String catalogName, SnowflakeAppCo else { statements.add(String.format(content.createStatement, catalogName)); - statements.add(String.format(content.grantStatement, catalogName)); + if (content.grantStatement != null) + { + statements.add(String.format(content.grantStatement, catalogName)); + } } return statements; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/pom.xml index 699ba32f5c6..c1cd908c790 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/pom.xml @@ -56,18 +56,18 @@ - - - - + + org.finos.legend.engine + legend-engine-language-pure-grammar + org.finos.legend.engine legend-engine-language-pure-compiler - - - - + + org.finos.legend.engine + legend-engine-identity-core + org.finos.legend.engine legend-engine-pure-code-compiled-core @@ -137,11 +137,6 @@ legend-engine-xt-relationalStore-snowflake-pure - - - - - com.fasterxml.jackson.core jackson-databind @@ -172,6 +167,11 @@ junit test + + org.finos.legend.engine + legend-engine-xt-relationalStore-snowflake-grammar + test + org.finos.legend.engine legend-engine-xt-relationalStore-javaPlatformBinding-pure @@ -196,6 +196,34 @@ org.finos.legend.engine legend-engine-language-pure-dsl-generation + + + org.finos.legend.engine + legend-engine-xt-json-pure + test + + + org.finos.legend.engine + legend-engine-xt-json-model + test + + + + org.finos.legend.engine + legend-engine-xt-json-javaPlatformBinding-pure + test + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-pureExtensions + test + + + org.finos.legend.engine + legend-engine-xt-javaPlatformBinding-pure + test + + diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/test/java/org/finos/legend/engine/language/snowflakeApp/generator/test/TestSnowflakeAppGenerator.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/test/java/org/finos/legend/engine/language/snowflakeApp/generator/test/TestSnowflakeAppGenerator.java new file mode 100644 index 00000000000..d1c6e01ba16 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/test/java/org/finos/legend/engine/language/snowflakeApp/generator/test/TestSnowflakeAppGenerator.java @@ -0,0 +1,102 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.snowflakeApp.generator.test; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.finos.legend.engine.language.pure.compiler.Compiler; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; +import org.finos.legend.engine.language.snowflakeApp.generator.SnowflakeAppGenerator; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppArtifact; +import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppContent; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.finos.legend.pure.generated.platform_pure_essential_meta_graph_pathToElement.Root_meta_pure_functions_meta_pathToElement_String_1__PackageableElement_1_; + +public class TestSnowflakeAppGenerator +{ + private final PureModelContextData contextData; + private final PureModel pureModel; + private final Function> routerExtensions = (PureModel pureModel) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport())); + + public TestSnowflakeAppGenerator() + { + this.contextData = PureGrammarParser.newInstance().parseModel(readModelContentFromResource(this.modelResourcePath())); + this.pureModel = Compiler.compile(contextData, null, Identity.getAnonymousIdentity().getName()); + } + + public String modelResourcePath() + { + return "/org/finos/legend/engine/language/snowflakeApp/generator/snowflakeAppTestModels.pure"; + } + + private String readModelContentFromResource(String resourcePath) + { + try (BufferedReader buffer = new BufferedReader(new InputStreamReader(Objects.requireNonNull(TestSnowflakeAppGenerator.class.getResourceAsStream(resourcePath))))) + { + return buffer.lines().collect(Collectors.joining("\n")); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private SnowflakeAppArtifact generateForActivator(String activatorPath, PureModel pureModel) + { + Root_meta_external_function_activator_snowflakeApp_SnowflakeApp app = (Root_meta_external_function_activator_snowflakeApp_SnowflakeApp) Root_meta_pure_functions_meta_pathToElement_String_1__PackageableElement_1_(activatorPath, pureModel.getExecutionSupport()); + return SnowflakeAppGenerator.generateArtifact(pureModel, app, this.contextData, routerExtensions); + } + + @Test + public void testNoParamActivator() + { + SnowflakeAppArtifact artifact = generateForActivator("demo::activators::snowflakeApp::App1", this.pureModel); + String expected = "CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1_REVISED() RETURNS TABLE (\"APP NAME\" VARCHAR,\"QUERY\" VARCHAR,\"OWNER\" VARCHAR,\"VERSION\" VARCHAR,\"DOC\" VARCHAR) LANGUAGE SQL AS $$ select \"root\".APP_NAME as \"App Name\", \"root\".SQL_FRAGMENT as \"Query\", \"root\".OWNER as \"Owner\", \"root\".VERSION_NUMBER as \"Version\", \"root\".DESCRIPTION as \"Doc\" from LEGEND_GOVERNANCE.BUSINESS_OBJECTS as \"root\" $$;"; + Assert.assertEquals(expected, ((SnowflakeAppContent)artifact.content).createStatement); + Assert.assertNull(((SnowflakeAppContent) artifact.content).grantStatement); + } + + @Test + public void testParamActivator() + { + SnowflakeAppArtifact artifact = generateForActivator("demo::activators::snowflakeApp::UDTFWithParam", this.pureModel); + String expected = "CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.UDTFWITHPARAMETER(\"nameLength\" INTEGER,\"nameStart\" VARCHAR) RETURNS TABLE (\"APP NAME\" VARCHAR,\"QUERY\" VARCHAR,\"OWNER\" VARCHAR,\"VERSION\" VARCHAR,\"DOC\" VARCHAR) LANGUAGE SQL AS $$ select \"root\".APP_NAME as \"App Name\", \"root\".SQL_FRAGMENT as \"Query\", \"root\".OWNER as \"Owner\", \"root\".VERSION_NUMBER as \"Version\", \"root\".DESCRIPTION as \"Doc\" from LEGEND_GOVERNANCE.BUSINESS_OBJECTS as \"root\" where (length(\"root\".APP_NAME) > nameLength and startswith(\"root\".APP_NAME,nameStart)) $$;"; + Assert.assertEquals(expected, ((SnowflakeAppContent)artifact.content).createStatement); + Assert.assertNull(((SnowflakeAppContent) artifact.content).grantStatement); + } + + @Test + public void testGrantGenerated() + { + SnowflakeAppArtifact artifact = generateForActivator("demo::activators::snowflakeApp::App2", this.pureModel); + String expected = "CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1_REVISED() RETURNS TABLE (\"APP NAME\" VARCHAR,\"QUERY\" VARCHAR,\"OWNER\" VARCHAR,\"VERSION\" VARCHAR,\"DOC\" VARCHAR) LANGUAGE SQL AS $$ select \"root\".APP_NAME as \"App Name\", \"root\".SQL_FRAGMENT as \"Query\", \"root\".OWNER as \"Owner\", \"root\".VERSION_NUMBER as \"Version\", \"root\".DESCRIPTION as \"Doc\" from LEGEND_GOVERNANCE.BUSINESS_OBJECTS as \"root\" $$;"; + String expectedGrant = "GRANT USAGE ON FUNCTION %S.LEGEND_NATIVE_APPS.APP1_REVISED() to role PUBLIC;"; + Assert.assertEquals(expected, ((SnowflakeAppContent)artifact.content).createStatement); + Assert.assertEquals(expectedGrant, ((SnowflakeAppContent) artifact.content).grantStatement); + } +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/test/resources/org/finos/legend/engine/language/snowflakeApp/generator/snowflakeAppTestModels.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/test/resources/org/finos/legend/engine/language/snowflakeApp/generator/snowflakeAppTestModels.pure new file mode 100644 index 00000000000..c32afa42d34 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/test/resources/org/finos/legend/engine/language/snowflakeApp/generator/snowflakeAppTestModels.pure @@ -0,0 +1,185 @@ +###Relational + +Database demo::stores::DemoDb +( + Schema LEGEND_GOVERNANCE + ( + Table BUSINESS_OBJECTS + ( + BUSINESS_OBJECT_UDTF_NAME VARCHAR(16777216), + APP_NAME VARCHAR(16777216) PRIMARY KEY, + ALLOY_SERVICE_NAME VARCHAR(16777216), + VERSION_NUMBER VARCHAR(16777216), + OWNER VARCHAR(16777216), + CREATE_DATETIME TIMESTAMP, + MODIFIED_DATETIME TIMESTAMP, + DESCRIPTION VARCHAR(16777216), + TO_BE_UPDATED VARCHAR(16777216), + SQL_FRAGMENT VARCHAR(16777216) + ) + + ) +) + + +###Snowflake +SnowflakeApp demo::activators::snowflakeApp::App1 +{ + applicationName : 'App1_revised'; + function : demo::functions::NativeApp_QueryFunction():TabularDataSet[1]; + ownership : Deployment { identifier: 'id1'}; + description : 'test App'; + activationConfiguration : demo::connections::DeploymentConnection; +} + +SnowflakeApp demo::activators::snowflakeApp::App2 +{ + applicationName : 'App1_revised'; + function : demo::functions::NativeApp_QueryFunction():TabularDataSet[1]; + ownership : Deployment { identifier: 'id1'}; + description : 'test App'; + usageRole: 'PUBLIC'; + activationConfiguration : demo::connections::DeploymentConnection; +} + +SnowflakeApp demo::activators::snowflakeApp::UDTFWithParam +{ + applicationName : 'UDTFWithParameter'; + function : demo::functions::NativeApp_QueryFunction(Integer[1],String[1]):TabularDataSet[1]; + ownership : Deployment { identifier: 'id1'}; + description : 'UDTFWithParameter'; + activationConfiguration : demo::connections::DeploymentConnection; +} + + + +###Pure + +Class demo::models::NativeApp +{ + AppName: String[1]; + Doc: String[1]; + Query: String[1]; + Owner: String[1]; + Version: String[1]; +} + + +function demo::functions::NativeApp_QueryFunction(): meta::pure::tds::TabularDataSet[1] +{ + demo::models::NativeApp.all()->project( + [ + x|$x.AppName, + x|$x.Query, + x|$x.Owner, + x|$x.Version, + x|$x.Doc + ], + [ + 'App Name', + 'Query', + 'Owner', + 'Version', + 'Doc' + ] + )->from( + demo::mappings::DemoMapping, + demo::runtimes::DemoRuntime + ) +} + +function demo::functions::NativeApp_QueryFunction(nameLength: Integer[1], nameStart: String[1]): meta::pure::tds::TabularDataSet[1] +{ + demo::models::NativeApp.all()->filter( + x|($x.AppName->length() > + $nameLength) && + $x.AppName->startsWith( + $nameStart + ) + )->project( + [ + x|$x.AppName, + x|$x.Query, + x|$x.Owner, + x|$x.Version, + x|$x.Doc + ], + [ + 'App Name', + 'Query', + 'Owner', + 'Version', + 'Doc' + ] + )->from( + demo::mappings::DemoMapping, + demo::runtimes::DemoRuntime + ) +} + + + +###Mapping +Mapping demo::mappings::DemoMapping +( + *demo::models::NativeApp: Relational + { + ~primaryKey + ( + [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS.APP_NAME + ) + ~mainTable [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS + AppName: [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS.APP_NAME, + Doc: [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS.DESCRIPTION, + Query: [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS.SQL_FRAGMENT, + Owner: [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS.OWNER, + Version: [demo::stores::DemoDb]LEGEND_GOVERNANCE.BUSINESS_OBJECTS.VERSION_NUMBER + } +) + + +###Connection +RelationalDatabaseConnection demo::connections::DemoSnowflakeConnection +{ + store: demo::stores::DemoDb; + type: Snowflake; + specification: Snowflake + { + name: 'dbName'; + account: 'account'; + warehouse: 'warehouse'; + region: 'region'; + }; + auth: DefaultH2; +} + +RelationalDatabaseConnection demo::connections::DeploymentConnection +{ + store: demo::stores::DemoDb; + type: Snowflake; + specification: Snowflake + { + name: 'dbName'; + account: 'account'; + warehouse: 'warehouse'; + region: 'region'; + }; + auth: DefaultH2; +} + + +###Runtime +Runtime demo::runtimes::DemoRuntime +{ + mappings: + [ + demo::mappings::DemoMapping + ]; + connections: + [ + demo::stores::DemoDb: + [ + connection_1: demo::connections::DemoSnowflakeConnection + ] + ]; +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure index 8f1b0ddcbaa..e7a70b2865b 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure @@ -12,7 +12,7 @@ Class meta::external::function::activator::snowflakeApp::SnowFlakeAppExecutionCo Class meta::external::function::activator::snowflakeApp::generation::Artifact { createQuery:String[1]; - grantStatement:String[1]; + grantStatement:String[0..1]; tables: String[*]; } @@ -32,9 +32,16 @@ function meta::external::function::activator::snowflakeApp::generation::generate let resultStub = generateResultTypeStub($plan.rootExecutionNode.resultType->cast(@TDSResultType), $extensions); let generatedQueryNode = $plan.rootExecutionNode->allNodes($extensions)->filter(n|$n->instanceOf(SQLExecutionNode))->cast(@SQLExecutionNode)->filter(node|$node.sqlQuery->toLower()->startsWith('select'))->last()->toOne('candidate query not found'); let query = 'CREATE OR REPLACE SECURE FUNCTION %S.' + defaultDeploymentSchema() + '.' +$s.applicationName->toUpper() + $inputParamsStub + ' RETURNS TABLE ('+ $resultStub+ ') LANGUAGE SQL AS $$ '+ $generatedQueryNode.sqlQuery +' $$;'; - let grant = 'GRANT USAGE ON FUNCTION %S.' + defaultDeploymentSchema() + '.'+ $s.applicationName->toUpper()+ generateInputParamsStub($s.function, false) + ' to role '+ if($s.usageRole->isNotEmpty(), |$s.usageRole->toOne()->toUpper(),|'PUBLIC')+';' ; - let artifact = ^Artifact(createQuery = $query, grantStatement = $grant, tables = $generatedQueryNode.metadata->filter(m| $m->instanceOf(TableInfo))->cast(@TableInfo)->toOne('No table info found').info->map(i|$i.schema+'.'+$i.table)); - + if($s.usageRole->isNotEmpty(), + | + let grant = 'GRANT USAGE ON FUNCTION %S.' + defaultDeploymentSchema() + '.'+ $s.applicationName->toUpper()+ generateInputParamsStub($s.function, false) + ' to role '+ if($s.usageRole->isNotEmpty(), |$s.usageRole->toOne()->toUpper(),|'PUBLIC')+';' ; + ^Artifact(createQuery = $query, grantStatement = $grant, tables = $generatedQueryNode.metadata->filter(m| $m->instanceOf(TableInfo))->cast(@TableInfo)->toOne('No table info found').info->map(i|$i.schema+'.'+$i.table));, + | + ^Artifact(createQuery = $query, tables = $generatedQueryNode.metadata->filter(m| $m->instanceOf(TableInfo))->cast(@TableInfo)->toOne('No table info found').info->map(i|$i.schema+'.'+$i.table)); + ); + + + } function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1]):String[1]