From 26c96fecbfe89300464af9a94604768109fad2f0 Mon Sep 17 00:00:00 2001 From: An Phi Date: Mon, 2 Dec 2024 22:38:48 -0500 Subject: [PATCH] datacube: update base query model (#3265) * repl: add more information to base query * repl: dim exec timing text * relation: (fix) support for quoted identifiers in relation casting * bump webapp@12.71.0 * bump webapp@12.72.0 --- legend-engine-config/legend-engine-repl/README.md | 2 +- .../legend/engine/repl/core/commands/Execute.java | 2 +- .../repl/dataCube/commands/DataCubeCache.java | 2 +- .../repl/dataCube/commands/DataCubeLoadCSV.java | 2 +- .../repl/dataCube/commands/DataCubeTable.java | 2 +- .../dataCube/commands/DataCubeWalkthrough.java | 6 ++---- .../DataCube__DEV__runDuckDBSelectSQL.java | 2 +- .../repl/dataCube/server/REPLServerHelpers.java | 14 ++++++++++++++ .../engine/repl/relational/commands/Cache.java | 2 +- .../grammar/from/domain/DomainParseTreeWalker.java | 5 ++++- .../HelperValueSpecificationGrammarComposer.java | 5 ++--- .../TestDomainGrammarArgumentsRoundtrip.java | 9 +++++++++ pom.xml | 2 +- 13 files changed, 39 insertions(+), 16 deletions(-) diff --git a/legend-engine-config/legend-engine-repl/README.md b/legend-engine-config/legend-engine-repl/README.md index 436f7bb1dba..1f9e264a026 100644 --- a/legend-engine-config/legend-engine-repl/README.md +++ b/legend-engine-config/legend-engine-repl/README.md @@ -19,7 +19,7 @@ help # to see the list of commands # first load a CSV file into local DuckDB load data.csv local::DuckDuckConnection test1 # then show the data -#>{local::DuckDuckDatabase.test1}#->select()->from(local::DuckDuckRuntime) +#>{local::DuckDuckDatabase.test1}#->from(local::DuckDuckRuntime) # to show the result grid in DataCube show diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java index 1c42a14b087..7ce5b92806a 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Execute.java @@ -54,7 +54,7 @@ public boolean process(String line) throws Exception { long startTime = System.currentTimeMillis(); this.client.println(execute(line)); - this.client.println(printExecutionTime(startTime)); + this.client.printDebug(printExecutionTime(startTime)); return true; } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java index 75aad4faa1f..ee4443c49c6 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java @@ -125,7 +125,7 @@ public boolean process(String line) throws Exception statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().load(tableName, tempFile.getTemporaryPathForFile(), relationalResultColumns)); this.client.println("Cached into table: '" + tableName + "'. Launching DataCube..."); - String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->select()->from(" + DataCube.getLocalRuntimePath() + ")"; + String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->from(" + DataCube.getLocalRuntimePath() + ")"; String functionCode = "###Pure\n" + "function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + functionBodyCode + ";\n}"; PureModelContextData pureModelContextData = client.getModelState().parseWithTransient(functionCode); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeLoadCSV.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeLoadCSV.java index c238a10e90b..cc2efff154b 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeLoadCSV.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeLoadCSV.java @@ -90,7 +90,7 @@ public boolean process(String line) throws Exception this.client.println("Loaded into table: '" + tableName + "'"); } - String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->select()->from(" + DataCube.getLocalRuntimePath() + ")"; + String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->from(" + DataCube.getLocalRuntimePath() + ")"; String functionCode = "###Pure\n" + "function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + functionBodyCode + ";\n}"; PureModelContextData pureModelContextData = client.getModelState().parseWithTransient(functionCode); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java index 2ad50680bce..ae654bf131d 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java @@ -77,7 +77,7 @@ public boolean process(String line) throws Exception } String tableName = tokens[2]; - String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->select()->from(" + DataCube.getLocalRuntimePath() + ")"; + String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->from(" + DataCube.getLocalRuntimePath() + ")"; String functionCode = "###Pure\n" + "function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + functionBodyCode + ";\n}"; PureModelContextData pureModelContextData = client.getModelState().parseWithTransient(functionCode); diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java index 4e34d065cc7..e6335c9b218 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java @@ -136,7 +136,7 @@ public DocumentationHelper.Walkthrough getWalkthrough() public static class DataCubeWalkthrough1 extends DocumentationHelper.Walkthrough { - public static final MutableList SELECT_ALL = Lists.mutable.with("select()"); + public static final MutableList SELECT_ALL = Lists.mutable.empty(); public static final MutableList FILTER = Lists.mutable.with("filter(x|$x.Year == 2010)"); public static final MutableList EXTEND = Lists.mutable.withAll(FILTER).with("extend(~Total: x|$x.Gold + $x.Silver + $x.Bronze)"); public static final MutableList GROUP_BY = Lists.mutable.withAll(EXTEND).with("groupBy(~Country, ~Total: x|$x.Total: x|$x->sum())"); @@ -212,9 +212,7 @@ protected MutableList> getSteps() this.printCommand(this.query(SELECT_ALL, 0), true); this.println(" ^ specifies which database and table to access"); this.printCommand(this.query(SELECT_ALL, 1), true); - this.println(" ^ shows ALL columns"); - this.printCommand(this.query(SELECT_ALL, 2), true); - this.println(" ^ specifies the execution context"); + this.println(" ^ specifies the execution context"); this.printFooter(); return null; }, diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCube__DEV__runDuckDBSelectSQL.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCube__DEV__runDuckDBSelectSQL.java index 6cf2211c234..04d234ef0a7 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCube__DEV__runDuckDBSelectSQL.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCube__DEV__runDuckDBSelectSQL.java @@ -91,7 +91,7 @@ public boolean process(String line) throws Exception } this.client.println("Executed SELECT SQL: '" + expression + "'"); this.client.println(prettyGridPrint(result, columnNames, columnNames, 40, 60)); - this.client.println(printExecutionTime(startTime)); + this.client.printDebug(printExecutionTime(startTime)); } } diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java index 96cf4a10867..7de4ce43707 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java @@ -31,7 +31,9 @@ import org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.SQLExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.Connection; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.runtime.PackageableRuntime; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.ClassInstance; @@ -147,12 +149,24 @@ else if (fn.parameters.size() == 3) fn.parameters.set(0, currentExpression); currentExpression = fn; } + Connection connection = null; + if (runtime != null) + { + String _runtime = runtime; + PackageableRuntime rt = (PackageableRuntime) ListIterate.select(pureModelContextData.getElements(), e -> e.getPath().equals(_runtime)).getFirst(); + if (rt != null && rt.runtimeValue.connections.size() == 1 && rt.runtimeValue.connections.get(0).storeConnections.size() == 1) + { + connection = rt.runtimeValue.connections.get(0).storeConnections.get(0).connection; + } + } Map source = Maps.mutable.empty(); source.put("_type", "repl"); source.put("timestamp", this.startTime); source.put("query", currentExpression.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().build())); source.put("runtime", runtime); source.put("mapping", mapping); + source.put("columns", columns); + source.put("connection", connection); this.source = source; // -------------------- QUERY -------------------- diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java index 61fe2eae5ba..7758366d26a 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/main/java/org/finos/legend/engine/repl/relational/commands/Cache.java @@ -119,7 +119,7 @@ public boolean process(String line) throws Exception { statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().load(tableName, tempFile.getTemporaryPathForFile(), relationalResultColumns)); this.client.println("Cached into table: '" + tableName + "'"); - this.client.println(printExecutionTime(startTime)); + this.client.printDebug(printExecutionTime(startTime)); } } catch (SQLException e) diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/domain/DomainParseTreeWalker.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/domain/DomainParseTreeWalker.java index 7192207934a..9bdd753bbf7 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/domain/DomainParseTreeWalker.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/domain/DomainParseTreeWalker.java @@ -1564,7 +1564,10 @@ else if (ctx.PAREN_OPEN() != null) { type = new RelationType(ListIterate.collect(ctx.columnInfo(), x -> { - Column column = new Column(x.columnName().getText(), x.columnType().getText()); + Column column = new Column( + PureGrammarParserUtility.fromIdentifier(x.columnName().identifier()), + PureGrammarParserUtility.fromIdentifier(x.columnType().identifier()) + ); column.sourceInformation = walkerSourceInformation.getSourceInformation(x); return column; })); diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/HelperValueSpecificationGrammarComposer.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/HelperValueSpecificationGrammarComposer.java index cf4cb60d031..10bc38acb46 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/HelperValueSpecificationGrammarComposer.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/HelperValueSpecificationGrammarComposer.java @@ -94,10 +94,9 @@ public static boolean isPrimitiveValue(ValueSpecification valueSpecification) ); } - public static String printColSpec(ColSpec col, DEPRECATED_PureGrammarComposerCore transformer) { - return PureGrammarComposerUtility.convertIdentifier(col.name) + (col.type != null ? ":" + col.type : "") + (col.function1 != null ? ":" + (transformer.isRenderingPretty() ? " " : "") + col.function1.accept(transformer) : "") + (col.function2 != null ? ":" + col.function2.accept(transformer) : ""); + return PureGrammarComposerUtility.convertIdentifier(col.name) + (col.type != null ? ":" + PureGrammarComposerUtility.convertIdentifier(col.type) : "") + (col.function1 != null ? ":" + (transformer.isRenderingPretty() ? " " : "") + col.function1.accept(transformer) : "") + (col.function2 != null ? ":" + col.function2.accept(transformer) : ""); } public static String printColSpecArray(ColSpecArray colSpecArray, DEPRECATED_PureGrammarComposerCore transformer) @@ -391,7 +390,7 @@ public static String printType(Type type, DEPRECATED_PureGrammarComposerCore tra } else if (type instanceof RelationType) { - return "(" + ListIterate.collect(((RelationType) type).columns, x -> x.name + ":" + x.type).makeString(", ") + ")"; + return "(" + ListIterate.collect(((RelationType) type).columns, x -> PureGrammarComposerUtility.convertIdentifier(x.name) + ":" + PureGrammarComposerUtility.convertIdentifier(x.type)).makeString(", ") + ")"; } throw new RuntimeException(type.getClass().getSimpleName() + ": Not supported"); } diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/roundtrip/TestDomainGrammarArgumentsRoundtrip.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/roundtrip/TestDomainGrammarArgumentsRoundtrip.java index 36d8c7cd6fb..4d46a7c8fd1 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/roundtrip/TestDomainGrammarArgumentsRoundtrip.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/roundtrip/TestDomainGrammarArgumentsRoundtrip.java @@ -52,6 +52,15 @@ public void testCast() "}\n"); } + @Test + public void testCastWithQuotedIdentifiers() + { + test("function withPath::f(s: a::Result[1]): Relation<('2000__a':'2000 Integer')>[0..1]\n" + + "{\n" + + " []->cast(@Relation<('2000__a':'2000 Integer')>)\n" + + "}\n"); + } + @Test public void testFunc() { diff --git a/pom.xml b/pom.xml index 130109e34ab..8e376e72c22 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ 5.25.0 0.25.7 - 12.59.0 + 12.72.0 legend-engine