REPL bug fixes and cleanup for DataCube (#2945)

* repl: fix a bug with relational DB table completer * repl: fix a bug tabbing when input is empty crashing the repl * repl: make sure SIGINT and SIGTERM not crash the app * repl: support loading table with specific name * repl: support dropping table * repl: add command description to help * repl: make opening brace part of function completer * repl: add README with quickstart and development guide * repl: refactor DataCube API for REPL server * relation: fix bug with relation store access not properly handling Date type * relation: support empty col spec array parsing to be consistent with Pure * relation: bug fixes for selecting column with spaces for duckdb * repl: fix tests
finos · Jul 8, 2024 · 37af507 · 37af507
1 parent c03e69a
commit 37af507
Show file tree

Hide file tree

Showing 54 changed files with 2,137 additions and 667 deletions.
diff --git a/legend-engine-config/legend-engine-repl/README.md b/legend-engine-config/legend-engine-repl/README.md
@@ -0,0 +1,72 @@
+# Legend REPL
+
+## Quick Start
+
+Run the REPL either in IDE (IntelliJ) or assembling a JAR and run the JAR. Within the REPL, to start
+
+> For autocomplete to work properly, it is recommended to run the REPL in a terminal; integrated terminal
+> in IDE often override hotkeys / keybindings that are used by the REPL. _Developers are also recommended
+> to work in a terminal to better test the interactions._
+
+```sh
+help # to see the list of commands
+
+# first load a CSV file into local DuckDB
+load data.csv local::DuckDuckConnection test1
+# then show the data
+#>{local::DuckDuckDatabase.test1}#->sort([])->from(local::DuckDuckRuntime)
+
+# to show the result grid in GUI mode
+show
+# to debug when error occurs, toggle debug mode
+debug
+```
+
+## Configuration
+
+```shell
+java \
+  # Specify ag-grid license key for full enterprise functionalities support
+  -Dlegend.repl.dataCube.gridLicenseKey=YOUR_LICENSE_KEY \
+
+  # [DEVELOPMENT] Specify the base URL for the development instance of the web application
+  # this is needed to bypass CORS
+  -Dlegend.repl.dataCube.devWebAppBaseUrl=http://localhost:9005 \
+
+  # [DEVELOPMENT] By default, the port is randomized, but for development, the port needs
+  # to be fixed to allow the web application to connect to the REPL
+  -Dlegend.repl.dataCube.devPort=9006 \
+
+  -jar legend-engine-repl.jar
+```
+
+## Developer Guide
+
+### REPL Development Setup
+
+To debug the REPL, you can either run it in Debug mode in IntelliJ, which would compromise certain features, such as autocomplete
+or you can run the REPL in a terminal. You can then setup a Remote JVM Debugger from IntelliJ to attach to the REPL instance.
+The exact command to run it can be copied from the command run by IntelliJ (and removing some IntelliJ specifics from the classpath), for example:
+
+```shell
+# NOTE: this command has a very long classpath!
+java -Dfile.encoding=UTF-8 -classpath ... -Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 org.finos.legend.engine.repl.relational.client.RClient
+```
+
+Then, in IntelliJ, start the Remote JVM Debug session with the specified port (5005 like in command above).
+
+![img.png](docs/repl-debug-setup.png)
+
+### DataCube Development Setup
+
+Configure `legend.repl.dataCube.devWebAppBaseUrl` and `legend.repl.dataCube.devPort`. See screenshot below for an example of
+how to configure the REPL in IntelliJ
+
+![img.png](docs/repl-webapp-dev-setup.png)
+
+Or if in a terminal
+
+```shell
+# NOTE: this command has a very long classpath!
+java -Dfile.encoding=UTF-8 -classpath ... -Dlegend.repl.devPort=9006 -Dlegend.repl.devWebAppBaseUrl=http://localhost:9005 org.finos.legend.engine.repl.relational.client.RClient
+```
diff --git a/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png b/legend-engine-config/legend-engine-repl/docs/repl-debug-setup.png
diff --git a/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png b/legend-engine-config/legend-engine-repl/docs/repl-webapp-dev-setup.png
diff --git a/...engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java b/...engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java
@@ -60,6 +60,9 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_QUALIFIED_PATH;
+import static org.finos.legend.engine.repl.core.Helpers.REPL_RUN_FUNCTION_SIGNATURE;
+
 public class Completer
 {
     private final String buildCodeContext;
@@ -82,7 +85,7 @@ public Completer(String buildCodeContext, MutableList<CompleterExtension> extens
                 buildCodeContext +
                         "\n###Pure\n" +
                         "import meta::pure::functions::relation::*;\n" +
-                        "function _pierre::func():Any[*]{\n";
+                        "function " + REPL_RUN_FUNCTION_SIGNATURE + "{\n";
         this.lineOffset = StringUtils.countMatches(header, "\n") + 1;
         this.handlers = Lists.mutable.with(
                 new FilterHandler(),
@@ -158,7 +161,7 @@ else if (topExpression instanceof AppliedFunction)
             if (currentExpression == topExpression)
             {
                 // The top function name is being written, propose candidates
-                return new CompletionResult(getFunctionCandidates(leftCompiledVS, pureModel, null).select(c -> c.startsWith(currentlyTypeFunctionName)).collect(c -> new CompletionItem(c, c)));
+                return new CompletionResult(getFunctionCandidates(leftCompiledVS, pureModel, null).select(c -> c.startsWith(currentlyTypeFunctionName)).collect(c -> new CompletionItem(c, c + "(")));
             }
             else if (handler != null)
             {
@@ -238,7 +241,7 @@ private ValueSpecification parseValueSpecification(String value)
     {
         String code = header + value + "\n" + "\n}";
         PureModelContextData pureModelContextData = PureGrammarParser.newInstance().parseModel(code);
-        Function func = (Function) ListIterate.select(pureModelContextData.getElements(), s -> s.getPath().equals("_pierre::func__Any_MANY_")).getFirst();
+        Function func = (Function) ListIterate.select(pureModelContextData.getElements(), s -> s.getPath().equals(REPL_RUN_FUNCTION_QUALIFIED_PATH)).getFirst();
         return func.body.get(0);
     }
 
@@ -260,14 +263,14 @@ else if (leftType._rawType().getName().equals("String"))
             }
             else
             {
-                return Lists.mutable.with("count");
+                return Lists.mutable.with("count", "joinStrings");
             }
         }
         else if (org.finos.legend.pure.m3.navigation.type.Type.subTypeOf(leftType._rawType(), pureModel.getType(M3Paths.Number), pureModel.getExecutionSupport().getProcessorSupport()))
         {
             if (org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.isToOne(multiplicity))
             {
-                return Lists.mutable.with("sqrt", "pow", "exp");
+                return Lists.mutable.with("abs", "pow", "sqrt", "exp");
             }
             else
             {

diff --git a/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java b/...l/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java
@@ -24,21 +24,22 @@
 import org.finos.legend.engine.repl.client.jline3.JLine3Parser;
 import org.finos.legend.engine.repl.core.Command;
 import org.finos.legend.engine.repl.core.ReplExtension;
-import org.finos.legend.engine.repl.core.commands.Debug;
-import org.finos.legend.engine.repl.core.commands.Execute;
-import org.finos.legend.engine.repl.core.commands.Ext;
-import org.finos.legend.engine.repl.core.commands.Graph;
-import org.finos.legend.engine.repl.core.commands.Help;
+import org.finos.legend.engine.repl.core.commands.*;
 import org.finos.legend.engine.repl.core.legend.LegendInterface;
 import org.finos.legend.engine.repl.core.legend.LocalLegendInterface;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
+import org.jline.reader.EndOfFileException;
 import org.jline.reader.LineReader;
 import org.jline.reader.LineReaderBuilder;
+import org.jline.reader.UserInterruptException;
 import org.jline.terminal.Terminal;
 import org.jline.terminal.TerminalBuilder;
 import org.jline.utils.AttributedStringBuilder;
 import org.jline.utils.AttributedStyle;
 
+import static org.jline.jansi.Ansi.ansi;
+import static org.jline.reader.LineReader.BLINK_MATCHING_PAREN;
+
 public class Client
 {
     private final LegendInterface legendInterface = new LocalLegendInterface();
@@ -50,7 +51,6 @@ public class Client
     private ModelState state;
     private final PlanExecutor planExecutor;
 
-
     public static void main(String[] args) throws Exception
     {
         new Client(Lists.mutable.empty(), Lists.mutable.empty(), PlanExecutor.newPlanExecutorBuilder().withAvailableStoreExecutors().build()).loop();
@@ -61,17 +61,14 @@ public static void main(String[] args) throws Exception
     public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterExtension> completerExtensions, PlanExecutor planExecutor) throws Exception
     {
         this.replExtensions = replExtensions;
-
         this.completerExtensions = completerExtensions;
-
         this.planExecutor = planExecutor;
-
         this.state = new ModelState(this.legendInterface, this.replExtensions);
+        this.terminal = TerminalBuilder.terminal();
 
         replExtensions.forEach(e -> e.initialize(this));
 
-        this.terminal = TerminalBuilder.terminal();
-
+        this.terminal.writer().println(ansi().fgBrightBlack().a("Welcome to the Legend REPL! Press 'Enter' or type 'help' to see the list of available commands.").reset());
         this.terminal.writer().println("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
 
         this.commands = replExtensions
@@ -89,8 +86,17 @@ public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterEx
 
         this.reader = LineReaderBuilder.builder()
                 .terminal(terminal)
+                // Disable cursor jumping to opening brace when typing closing brace
+                // See https://github.com/jline/jline3/issues/216
+                .variable(BLINK_MATCHING_PAREN, false)
+                // Make sure hitting <tab> at the beginning of line will insert a tab instead of triggering a completion
+                // which will cause error since the completer doesn't handle such case
+                // See https://github.com/jline/jline3/wiki/Completion
+                .option(LineReader.Option.INSERT_TAB, true)
+                // Make sure word navigation works properly with Alt + (left/right) arrow key
+                .variable(LineReader.WORDCHARS, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-$")
                 .highlighter(new JLine3Highlighter())
-                .parser(new JLine3Parser())//new DefaultParser().quoteChars(new char[]{'"'}))
+                .parser(new JLine3Parser())
                 .completer(new JLine3Completer(this.commands))
                 .build();
 
@@ -105,16 +111,16 @@ public void loop()
     {
         while (true)
         {
-            String line = this.reader.readLine("> ");
-            if (line == null || line.equalsIgnoreCase("exit"))
+            try
             {
-                break;
-            }
+                String line = this.reader.readLine("> ");
+                if (line == null || line.equalsIgnoreCase("exit"))
+                {
+                    break;
+                }
 
-            this.reader.getHistory().add(line);
+                this.reader.getHistory().add(line);
 
-            try
-            {
                 this.commands.detect(new CheckedPredicate<Command>()
                 {
                     @Override
@@ -126,14 +132,34 @@ public boolean safeAccept(Command c) throws Exception
             }
             catch (EngineException e)
             {
-                printError(e, line);
+                printError(e, this.reader.getBuffer().toString());
+            }
+            // handle Ctrl + C: if the input is not empty, start a new line; otherwise, exit
+            catch (UserInterruptException e)
+            {
+                String lineContent = this.reader.getBuffer().toString();
+                if (lineContent.isEmpty())
+                {
+                    System.exit(0);
+                    break;
+                }
+                else
+                {
+                    this.loop();
+                }
             }
-            catch (Exception ee)
+            // handle Ctrl + D: exit
+            catch (EndOfFileException e)
             {
-                this.terminal.writer().println(ee.getMessage());
+                System.exit(0);
+                break;
+            }
+            catch (Exception e)
+            {
+                this.terminal.writer().println(ansi().fgRed().a(e.getMessage()).reset());
                 if (this.debug)
                 {
-                    ee.printStackTrace();
+                    e.printStackTrace();
                 }
             }
         }
@@ -204,4 +230,9 @@ public MutableList<CompleterExtension> getCompleterExtensions()
     {
         return this.completerExtensions;
     }
+
+    public Execute getExecuteCommand()
+    {
+        return (Execute) this.commands.detect(c -> c instanceof Execute);
+    }
 }
diff --git a/...ne-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Parser.java b/...ne-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Parser.java
@@ -45,7 +45,7 @@ public MyParsedLine(ParserResult result)
         public String word()
         {
             int index = wordIndex();
-            if (result.words.size() > index)
+            if (index >= 0 && result.words.size() > index)
             {
                 return result.words.get(index);
             }

diff --git a/...pl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java b/...pl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Command.java
@@ -25,5 +25,10 @@ public interface Command
 
     public String documentation();
 
+    public default String description()
+    {
+        return "";
+    }
+
     public MutableList<Candidate> complete(String cmd, LineReader lineReader, ParsedLine parsedLine);
 }
diff --git a/...pl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java b/...pl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java
@@ -20,6 +20,9 @@
 
 public class Helpers
 {
+    public static final String REPL_RUN_FUNCTION_QUALIFIED_PATH = "repl::__internal__::run__Any_MANY_";
+    public static final String REPL_RUN_FUNCTION_SIGNATURE = "repl::__internal__::run():Any[*]";
+
     public static Identity resolveIdentityFromLocalSubject(Client client)
     {
         try

diff --git a/...end-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java b/...end-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/ReplExtension.java
@@ -42,8 +42,6 @@ default MutableList<String> typeGroup()
 
     MutableList<Command> getExtraCommands();
 
-//    MutableList<String> getExtraState();
-
     boolean supports(Result res);
 
     String print(Result res);

diff --git a/...nd-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java b/...nd-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java
@@ -36,6 +36,12 @@ public String documentation()
         return "debug (<boolean>)";
     }
 
+    @Override
+    public String description()
+    {
+        return "toggle debug mode";
+    }
+
     @Override
     public boolean process(String line) throws Exception
     {