repl: minor cleanups and improve doc command (finos#3062)

* repl: make sure server use the same plan executor * pure-ide: bump [email protected] * repl: show compatibility in doc
rafaelbey · Sep 3, 2024 · 5107c89 · 5107c89
1 parent 93be98d
commit 5107c89
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 45 deletions.
diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml
@@ -85,6 +85,14 @@
         </dependency>
         <!-- Pure Plan Generation / Execution -->
 
+        <!-- PCT -->
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-javaPlatformBinding-PCT</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <!-- PCT -->
+
         <!-- Extensions (use by Ext) -->
         <dependency>
             <groupId>org.finos.legend.engine</groupId>

diff --git a/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java b/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java
@@ -39,6 +39,7 @@
 import org.finos.legend.pure.m3.pct.aggregate.generation.DocumentationGeneration;
 import org.finos.legend.pure.m3.pct.aggregate.model.Documentation;
 import org.finos.legend.pure.m3.pct.aggregate.model.FunctionDocumentation;
+import org.finos.legend.pure.m3.pct.reports.model.AdapterKey;
 import org.jline.reader.EndOfFileException;
 import org.jline.reader.LineReader;
 import org.jline.reader.LineReaderBuilder;
@@ -328,6 +329,11 @@ private void persistHistory()
         }
     }
 
+    public MutableList<AdapterKey> getDocumentationAdapterKeys()
+    {
+        return Lists.mutable.withAll(this.documentation.adapters);
+    }
+
     public List<String> getDocumentedFunctions()
     {
         return this.functionDocIndex.keysView().toSortedList();

diff --git a/...gend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Doc.java b/...gend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Doc.java
@@ -55,7 +55,7 @@ public boolean process(String line) throws Exception
             FunctionDocumentation functionDocumentation = this.client.getFunctionDocumentation(path);
             if (functionDocumentation != null)
             {
-                client.println(DocumentationHelper.generateANSIFunctionDocumentation(functionDocumentation));
+                client.println(DocumentationHelper.generateANSIFunctionDocumentation(functionDocumentation, client.getDocumentationAdapterKeys()));
             }
             else
             {

diff --git a/...ne-repl-client/src/main/java/org/finos/legend/engine/repl/shared/DocumentationHelper.java b/...ne-repl-client/src/main/java/org/finos/legend/engine/repl/shared/DocumentationHelper.java
@@ -16,14 +16,24 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.collections.api.block.function.Function0;
+import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.api.map.MutableMap;
+import org.eclipse.collections.impl.tuple.Tuples;
+import org.eclipse.collections.impl.utility.ArrayIterate;
+import org.eclipse.collections.impl.utility.ListIterate;
+import org.eclipse.collections.impl.utility.MapIterate;
 import org.finos.legend.engine.repl.client.Client;
+import org.finos.legend.pure.m3.pct.aggregate.model.Documentation;
 import org.finos.legend.pure.m3.pct.aggregate.model.FunctionDocumentation;
-import org.finos.legend.pure.m3.pct.functions.model.Signature;
+import org.finos.legend.pure.m3.pct.functions.model.FunctionDefinition;
+import org.finos.legend.pure.m3.pct.reports.model.Adapter;
+import org.finos.legend.pure.m3.pct.reports.model.AdapterKey;
+import org.finos.legend.pure.m3.pct.reports.model.TestInfo;
 
-import static org.finos.legend.engine.repl.shared.REPLHelper.ansiDim;
-import static org.finos.legend.engine.repl.shared.REPLHelper.ansiGreen;
+import java.util.Objects;
+
+import static org.finos.legend.engine.repl.shared.REPLHelper.*;
 
 public class DocumentationHelper
 {
@@ -37,51 +47,73 @@ public class DocumentationHelper
         MODULE_URLS.put("relation", "https://github.com/finos/legend-engine/tree/master/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources");
     }
 
-    private static final int ANSI_ATTR_WIDTH = 20;
+    private static final int ANSI_ATTR_WIDTH = 8;
 
-    public static String generateANSIFunctionDocumentation(FunctionDocumentation functionDocumentation)
+    public static String generateANSIFunctionDocumentation(FunctionDocumentation functionDocumentation, MutableList<AdapterKey> adapterKeys)
     {
         StringBuilder builder = new StringBuilder();
-        builder.append(ansiAttr("function")).append(ansiGreen(functionDocumentation.reportScope._package + "::" + functionDocumentation.functionDefinition.name + "()")).append("\n");
-        builder.append(ansiAttr("  [src]")).append(getFunctionSourceUrl(functionDocumentation)).append("\n");
-        builder.append("\n");
-        for (int i = 0; i < functionDocumentation.functionDefinition.signatures.size(); i++)
+        FunctionDefinition definition = functionDocumentation.functionDefinition;
+        String name = definition.name;
+        String path = definition._package + "::" + name;
+        String src = MODULE_URLS.get(functionDocumentation.reportScope.module) + definition.sourceId;
+        String grouping = definition.sourceId.substring(functionDocumentation.reportScope.filePath.length(), definition.sourceId.lastIndexOf("/"));
+        // NOTE: make assumption that each function has doc/usage on exactly one of the signatures
+        String syntax = ListIterate.detectOptional(definition.signatures, signature -> signature.grammarCharacter != null).map(s -> s.grammarCharacter).orElse(null);
+        String doc = ListIterate.detectOptional(definition.signatures, signature -> signature.documentation != null).map(s -> s.documentation).orElse(null);
+        String usage = ListIterate.detectOptional(definition.signatures, signature -> signature.grammarDoc != null).map(s -> s.grammarDoc).orElse(null);
+
+        Lists.mutable.with(syntax != null ? ansiGreen(syntax) : null)
+                .with(name)
+                .withAll(ListIterate.collect(definition.signatures, s -> s.simple.substring(definition._package.length() + 2)))
+                .select(Objects::nonNull)
+                .forEachWithIndex((value, idx) -> builder.append(ansiAttr(idx == 0 ? "function" : null)).append(value).append("\n"));
+        builder.append(ansiAttr("path")).append(path).append("\n");
+        builder.append(ansiAttr("grouping")).append("(" + functionDocumentation.reportScope.module + ") " + grouping).append("\n");
+        builder.append(ansiAttr("src")).append(src).append("\n");
+        if (doc != null)
         {
-            Signature signature = functionDocumentation.functionDefinition.signatures.get(i);
-            builder.append(ansiGreen(ansiAttr("#[" + (i + 1) + "]")));
-            if (signature.grammarCharacter != null)
-            {
-                builder.append(ansiGreen(signature.grammarCharacter));
-                builder.append("\n").append(ansiAttr(""));
-            }
-            builder.append(ansiGreen(signature.simple)).append("\n");
-            builder.append(ansiAttr("  [id]")).append(ansiDim(signature.id));
-            if (signature.documentation != null)
-            {
-                builder.append("\n");
-                builder.append(ansiAttr("  [doc]")).append(signature.documentation);
-            }
-            if (signature.grammarDoc != null)
+            builder.append(ansiAttr("doc")).append(ArrayIterate.makeString(wrap(doc).split("\n"), "\n" + ansiAttr(null))).append("\n");
+        }
+        if (usage != null)
+        {
+            builder.append(ansiAttr("usage")).append(usage).append("\n");
+        }
+
+        // compatibility
+        MutableMap<String, String> matrix = MapIterate.collect(functionDocumentation.functionTestResults, (adapterKey, testResults) ->
+        {
+            String key = adapterKey.adapter.group + (adapterKey.adapter.group.isEmpty() ? "" : "/") + adapterKey.adapter.name;
+            MutableList<TestInfo> tests = Lists.mutable.withAll(testResults.tests);
+            String value;
+            if (tests.isEmpty())
             {
-                builder.append("\n");
-                builder.append(ansiAttr("  [usage]")).append(signature.grammarDoc);
+                value = ansiDim("∅");
             }
-            if (i != functionDocumentation.functionDefinition.signatures.size() - 1)
+            else
             {
-                builder.append("\n");
+                int passedCount = tests.select(t -> t.success).size();
+                value = passedCount + "/" + tests.size();
+                value = passedCount == 0 ? ansiRed(value) : passedCount == tests.size() ? ansiGreen(value) : ansiYellow(value);
             }
-        }
+            return Tuples.pair(key, value);
+        });
+        // NOTE: here we sort the adapter naively, and it achieves the desired order anyway,
+        // but we should consider a more methodical/intentional sort: e.g. native goes first, followed by platforms and stores
+        MutableList<String> adapters = adapterKeys.collect(adapterKey -> adapterKey.adapter.group + (adapterKey.adapter.group.isEmpty() ? "" : "/") + adapterKey.adapter.name).toSortedList();
+        int maxKeyLength = adapters.collect(String::length).max();
+
+        builder.append("\n").append(StringUtils.rightPad("compatibility", maxKeyLength + 2)).append(" :").append("\n");
+        builder.append(adapters.collect(adapter -> StringUtils.rightPad("  " + adapter, maxKeyLength + 2) + " : " + matrix.getOrDefault(adapter, "∅")).makeString("\n"));
         return builder.toString();
     }
 
     private static String ansiAttr(String attr)
     {
-        return StringUtils.rightPad(attr, ANSI_ATTR_WIDTH);
-    }
-
-    private static String getFunctionSourceUrl(FunctionDocumentation functionDocumentation)
-    {
-        return MODULE_URLS.get(functionDocumentation.reportScope.module) + functionDocumentation.functionDefinition.sourceId;
+        if (attr == null)
+        {
+            return StringUtils.rightPad("", ANSI_ATTR_WIDTH + 3);
+        }
+        return StringUtils.rightPad(attr, ANSI_ATTR_WIDTH) + " : ";
     }
 
     public abstract static class Walkthrough

diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-data-cube/pom.xml
@@ -28,7 +28,7 @@
 
     <properties>
         <skip.yarn>false</skip.yarn>
-        <repl.web-application.version>12.28.0</repl.web-application.version>
+        <repl.web-application.version>12.31.0</repl.web-application.version>
         <repl.web-application.url>${npm.registry.url}/@finos/legend-application-repl-deployment/-/legend-application-repl-deployment-${repl.web-application.version}.tgz</repl.web-application.url>
     </properties>
 

diff --git a/...ube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java b/...ube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeWalkthrough.java
@@ -302,7 +302,7 @@ protected MutableList<Function0<Void>> getSteps()
                                 "usages for 'filter', use the following command:"));
                         this.printCommand("doc meta::pure::functions::relation::filter");
                         this.println(ansiDim(printRule(null)));
-                        this.println(DocumentationHelper.generateANSIFunctionDocumentation(this.client.getFunctionDocumentation("meta::pure::functions::relation::filter")));
+                        this.println(DocumentationHelper.generateANSIFunctionDocumentation(this.client.getFunctionDocumentation("meta::pure::functions::relation::filter"), this.client.getDocumentationAdapterKeys()));
                         this.client.addCommandToHistory("doc meta::pure::functions::relation::filter");
                         this.println(ansiDim(printRule(null)));
                         this.println("");

diff --git a/...repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServer.java b/...repl-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServer.java
@@ -14,21 +14,18 @@
 
 package org.finos.legend.engine.repl.dataCube.server;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sun.net.httpserver.Filter;
 import com.sun.net.httpserver.HttpContext;
 import com.sun.net.httpserver.HttpServer;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.impl.factory.Maps;
-import org.finos.legend.engine.plan.execution.PlanExecutor;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.repl.client.Client;
 import org.finos.legend.engine.repl.dataCube.server.handler.DataCubeInfrastructure;
 import org.finos.legend.engine.repl.dataCube.server.handler.DataCubeQueryBuilder;
 import org.finos.legend.engine.repl.dataCube.server.handler.DataCubeQueryExecutor;
 import org.finos.legend.engine.repl.shared.ExecutionHelper;
-import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 
 import java.net.InetSocketAddress;
 import java.util.List;
@@ -37,8 +34,6 @@
 
 public class REPLServer
 {
-    private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports();
-    private static final PlanExecutor planExecutor = PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build();
     private final Client client;
     private final REPLServerHelpers.REPLServerState state;
 
@@ -49,7 +44,7 @@ public class REPLServer
     public REPLServer(Client client)
     {
         this.client = client;
-        this.state = new REPLServerHelpers.REPLServerState(client, objectMapper, planExecutor, client.getLegendInterface());
+        this.state = new REPLServerHelpers.REPLServerState(client, client.getObjectMapper(), client.getPlanExecutor(), client.getLegendInterface());
     }
 
     public void initializeStateWithREPLExecutedQuery(ExecutionHelper.ExecuteResultSummary executeResultSummary)

diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml
@@ -85,6 +85,30 @@
         </dependency>
         <!-- Relational -->
 
+        <!-- PCT -->
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-duckdb-PCT</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-h2-PCT</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-postgres-PCT</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-snowflake-PCT</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <!-- PCT -->
+
         <dependency>
             <groupId>com.h2database</groupId>
             <artifactId>h2</artifactId>

diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/pom.xml b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/pom.xml
@@ -28,7 +28,7 @@
 
     <properties>
         <skip.yarn>false</skip.yarn>
-        <pure-ide.web-application.version>9.0.0</pure-ide.web-application.version>
+        <pure-ide.web-application.version>12.31.0</pure-ide.web-application.version>
         <pure-ide.web-application.url>${npm.registry.url}/@finos/legend-application-pure-ide-deployment/-/legend-application-pure-ide-deployment-${pure-ide.web-application.version}.tgz</pure-ide.web-application.url>
     </properties>