repl: showing function doc and add walkthrough for datacube (#3042)

* datacube: fix dev shaded jar build

* repl: support showing function doc

* repl: allow clearing screen and unify print styling

* datacube: walkthrough

legend-engine-config/legend-engine-repl/README.md

@@ -2,11 +2,11 @@
 
 ## Quick Start
 
-Run the REPL either in IDE (IntelliJ) or run the script in legend-engine-repl-app-assembly module.
+Run the REPL either in IDE (IntelliJ) or run the script in [legend-engine-repl-app-assembly](./legend-engine-repl-app-assembly/assemble) module.
 
-The legend-engine-repl-app-assembly allow you to pick which REPL to run: default, relational, or datacube.
+The `legend-engine-repl-app-assembly` allow you to pick which REPL to run: default, relational, datacube, etc.
 
-The script also allow developers to enable debug on a given port.  This allows to run the REPL on an OS native terminal, 
+> The script also allow developers to enable debug on a given port. This allows to run the REPL on an OS native terminal, 
 and still be able to debug it through an IDE.
 
 > For autocomplete to work properly, it is recommended to run the REPL in a terminal; integrated terminal
@@ -27,6 +27,8 @@ show
 debug
 ```
 
+> Check out the walkthrough with `datacube walkthrough`
+
 ## Configuration
 
 ```shell

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java

@@ -37,15 +37,7 @@
 import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda;
 import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.relation.ColSpec;
 import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.relation.ColSpecArray;
-import org.finos.legend.engine.repl.autocomplete.handlers.ExtendHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.FilterHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.FromHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.GroupByHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.JoinHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.OverHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.RenameHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.SelectHandler;
-import org.finos.legend.engine.repl.autocomplete.handlers.SortHandler;
+import org.finos.legend.engine.repl.autocomplete.handlers.*;
 import org.finos.legend.engine.repl.autocomplete.parser.ParserFixer;
 import org.finos.legend.engine.repl.core.legend.LegendInterface;
 import org.finos.legend.engine.shared.core.identity.Identity;
@@ -109,6 +101,7 @@ private Completer(String buildCodeContext, MutableList<CompleterExtension> exten
                 new SortHandler(),
                 new JoinHandler(),
                 new SelectHandler(),
+                new DistinctHandler(),
                 new OverHandler()
         ).toMap(FunctionHandler::functionName, x -> x);
     }

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/handlers/DistinctHandler.java

@@ -0,0 +1,42 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.repl.autocomplete.handlers;
+
+import org.eclipse.collections.api.list.MutableList;
+import org.finos.legend.engine.language.pure.compiler.toPureGraph.ProcessingContext;
+import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
+import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification;
+import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction;
+import org.finos.legend.engine.repl.autocomplete.Completer;
+import org.finos.legend.engine.repl.autocomplete.CompletionItem;
+import org.finos.legend.engine.repl.autocomplete.FunctionHandler;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType;
+
+import static org.finos.legend.engine.repl.autocomplete.Completer.proposeColumnNamesForEditColSpec;
+
+public class DistinctHandler extends FunctionHandler
+{
+    @Override
+    public String functionName()
+    {
+        return "distinct";
+    }
+
+    @Override
+    public MutableList<CompletionItem> proposedParameters(AppliedFunction currentFunc, GenericType leftType, PureModel pureModel, Completer completer, ProcessingContext processingContext, ValueSpecification currentVS)
+    {
+        return proposeColumnNamesForEditColSpec(currentFunc, leftType);
+    }
+}

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/Client.java

@@ -18,8 +18,11 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.factory.Maps;
 import org.eclipse.collections.api.list.MutableList;
+import org.eclipse.collections.api.map.MutableMap;
 import org.eclipse.collections.impl.block.predicate.checked.CheckedPredicate;
+import org.eclipse.collections.impl.utility.ListIterate;
 import org.finos.legend.engine.plan.execution.PlanExecutor;
 import org.finos.legend.engine.repl.autocomplete.CompleterExtension;
 import org.finos.legend.engine.repl.client.jline3.JLine3Completer;
@@ -33,6 +36,9 @@
 import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 import org.finos.legend.engine.shared.core.deployment.DeploymentStateAndVersions;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
+import org.finos.legend.pure.m3.pct.aggregate.generation.DocumentationGeneration;
+import org.finos.legend.pure.m3.pct.aggregate.model.Documentation;
+import org.finos.legend.pure.m3.pct.aggregate.model.FunctionDocumentation;
 import org.jline.reader.EndOfFileException;
 import org.jline.reader.LineReader;
 import org.jline.reader.LineReaderBuilder;
@@ -41,11 +47,14 @@
 import org.jline.terminal.TerminalBuilder;
 import org.jline.utils.AttributedStringBuilder;
 import org.jline.utils.AttributedStyle;
+import org.jline.utils.InfoCmp;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 
-import static org.jline.jansi.Ansi.ansi;
+import static org.finos.legend.engine.repl.shared.REPLHelper.ansiDim;
+import static org.finos.legend.engine.repl.shared.REPLHelper.ansiRed;
 import static org.jline.reader.LineReader.BLINK_MATCHING_PAREN;
 
 public class Client
@@ -59,6 +68,8 @@ public class Client
     private final ModelState state;
     private final PlanExecutor planExecutor;
     private final Path homeDirectory;
+    private final Documentation documentation;
+    private final MutableMap<String, FunctionDocumentation> functionDocIndex = Maps.mutable.empty();
 
     private boolean debug = false;
 
@@ -84,16 +95,17 @@ public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterEx
         this.terminal = TerminalBuilder.terminal();
         this.homeDirectory = homeDirectory;
 
+        this.documentation = DocumentationGeneration.buildDocumentation();
         this.initialize();
         replExtensions.forEach(e -> e.initialize(this));
 
-        this.printDebug("Legend REPL v" + DeploymentStateAndVersions.sdlc.buildVersion + " (" + DeploymentStateAndVersions.sdlc.commitIdAbbreviated + ")");
+        this.printDebug("[DEV] Legend REPL v" + DeploymentStateAndVersions.sdlc.buildVersion + " (" + DeploymentStateAndVersions.sdlc.commitIdAbbreviated + ")");
         if (System.getProperty("legend.repl.initializationMessage") != null)
         {
             this.printDebug(StringEscapeUtils.unescapeJava(System.getProperty("legend.repl.initializationMessage")));
         }
         this.printDebug("Press 'Enter' or type 'help' to see the list of available commands.");
-        this.printInfo("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
+        this.println("\n" + Logos.logos.get((int) (Logos.logos.size() * Math.random())) + "\n");
 
         // NOTE: the order here matters, the default command 'help' should always go first
         // and "catch-all" command 'execute' should always go last
@@ -103,6 +115,7 @@ public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterEx
                         Lists.mutable.with(
                                 new Ext(this),
                                 new Debug(this),
+                                new Doc(this),
                                 new Graph(this),
                                 new Execute(this)
                         )
@@ -135,10 +148,33 @@ public Client(MutableList<ReplExtension> replExtensions, MutableList<CompleterEx
                 .completer(new JLine3Completer(this.commands))
                 .build();
 
-        this.printInfo("Warming up...");
+        this.println("Warming up...");
         this.terminal.flush();
         ((Execute) this.commands.getLast()).execute("1+1");
-        this.printInfo("Ready!\n");
+        this.println("Ready!\n");
+    }
+
+    private void initialize()
+    {
+        // Index function documentation by path and remove platform only function
+        this.documentation.functionsDocumentation.forEach(doc ->
+                doc.functionDefinition.signatures = ListIterate.select(doc.functionDefinition.signatures, signature -> !signature.platformOnly));
+        this.documentation.functionsDocumentation = ListIterate.reject(this.documentation.functionsDocumentation, (doc) -> doc.functionDefinition.signatures.isEmpty());
+        this.documentation.functionsDocumentation.forEach(doc ->
+                this.functionDocIndex.put(doc.functionDefinition._package + "::" + doc.functionDefinition.name, doc));
+
+        try
+        {
+            Path homeDir = this.getHomeDir();
+            if (Files.notExists(homeDir))
+            {
+                Files.createDirectories(homeDir);
+            }
+        }
+        catch (Exception e)
+        {
+            this.printError("Failed to create home directory at: " + this.getHomeDir().toString());
+        }
     }
 
     public void loop()
@@ -154,28 +190,33 @@ public void loop()
                     this.forceExit();
                     break;
                 }
-
-                this.reader.getHistory().add(line);
-
-                this.commands.detect(new CheckedPredicate<Command>()
+                else if (line.equalsIgnoreCase("clear"))
+                {
+                    this.clearScreen();
+                }
+                else
                 {
-                    @Override
-                    public boolean safeAccept(Command c) throws Exception
+                    this.reader.getHistory().add(line);
+                    this.commands.detect(new CheckedPredicate<Command>()
                     {
-                        try
+                        @Override
+                        public boolean safeAccept(Command c) throws Exception
                         {
-                            return c.process(line);
+                            try
+                            {
+                                return c.process(line);
+                            }
+                            catch (RuntimeException e)
+                            {
+                                throw e;
+                            }
+                            catch (Exception e)
+                            {
+                                throw new RuntimeException(e);
+                            }
                         }
-                        catch (RuntimeException e)
-                        {
-                            throw e;
-                        }
-                        catch (Exception e)
-                        {
-                            throw new RuntimeException(e);
-                        }
-                    }
-                });
+                    });
+                }
             }
             catch (EngineException e)
             {
@@ -228,16 +269,13 @@ public void printEngineError(EngineException e, String line)
             {
                 String beg = line.substring(0, e_start - 1);
                 String mid = line.substring(e_start - 1, e_end);
-                String end = line.substring(e_end, line.length());
+                String end = line.substring(e_end);
                 AttributedStringBuilder ab = new AttributedStringBuilder();
-                ab.style(new AttributedStyle().underlineOff().foregroundOff());
                 ab.append(beg);
-                ab.style(new AttributedStyle().underline().foreground(AttributedStyle.RED));
-                ab.append(mid);
-                ab.style(new AttributedStyle().underlineOff().foregroundOff());
+                ab.style(new AttributedStyle().underline());
+                ab.append(ansiRed(mid));
                 ab.append(end);
-                this.printInfo("");
-                this.printInfo(ab.toAnsi());
+                this.println(ab.toAnsi());
             }
         }
         catch (Exception ex)
@@ -251,35 +289,31 @@ public void printEngineError(EngineException e, String line)
         }
     }
 
-    public void printDebug(String message)
+    public void clearScreen()
     {
-        this.terminal.writer().println(ansi().fgBrightBlack().a(message).reset());
+        // See https://github.com/jline/jline3/issues/418
+        this.terminal.puts(InfoCmp.Capability.clear_screen);
+        this.terminal.flush();
     }
 
-    public void printInfo(String message)
+    public void print(String message)
+    {
+        this.terminal.writer().print(message);
+    }
+
+    public void println(String message)
     {
         this.terminal.writer().println(message);
     }
 
-    public void printError(String message)
+    public void printDebug(String message)
     {
-        this.terminal.writer().println(ansi().fgRed().a(message).reset());
+        this.terminal.writer().println(ansiDim(message));
     }
 
-    private void initialize()
+    public void printError(String message)
     {
-        try
-        {
-            Path homeDir = this.getHomeDir();
-            if (Files.notExists(homeDir))
-            {
-                Files.createDirectories(homeDir);
-            }
-        }
-        catch (Exception e)
-        {
-            this.printError("Failed to create home directory at: " + this.getHomeDir().toString());
-        }
+        this.terminal.writer().println(ansiRed(message));
     }
 
     private void persistHistory()
@@ -294,6 +328,16 @@ private void persistHistory()
         }
     }
 
+    public List<String> getDocumentedFunctions()
+    {
+        return this.functionDocIndex.keysView().toSortedList();
+    }
+
+    public FunctionDocumentation getFunctionDocumentation(String path)
+    {
+        return this.functionDocIndex.get(path);
+    }
+
     public Path getHomeDir()
     {
         return this.homeDirectory;
@@ -361,6 +405,12 @@ public String getLastCommand(int skip)
         }
     }
 
+    public void addCommandToHistory(String command)
+    {
+        this.reader.getHistory().add(command);
+        this.persistHistory();
+    }
+
     /**
      * Force exit (using System.exit()) will make sure the REPL quit completely
      * when "exit" command is called or terminate signal with Ctrl+D/Ctrl+C in invoked

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/client/jline3/JLine3Highlighter.java

@@ -28,7 +28,8 @@ public class JLine3Highlighter implements Highlighter
     public AttributedString highlight(LineReader lineReader, String s)
     {
         AttributedStringBuilder ab = new AttributedStringBuilder();
-        drawCommand(ab, s);
+        ab.style(new AttributedStyle().foreground(AttributedStyle.GREEN).italic());
+        ab.append(s);
         return ab.toAttributedString();
     }
 
@@ -43,10 +44,4 @@ public void setErrorIndex(int i)
     {
 
     }
-
-    public static void drawCommand(AttributedStringBuilder ab, String command)
-    {
-        ab.style(new AttributedStyle().foreground(AttributedStyle.GREEN).italic());
-        ab.append(command);
-    }
 }

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/commands/Debug.java

@@ -56,7 +56,7 @@ public boolean process(String line) throws Exception
             {
                 this.client.setDebug(Boolean.parseBoolean(cmd[1]));
             }
-            this.client.printInfo("debug: " + this.client.isDebug());
+            this.client.println("debug: " + this.client.isDebug());
             return true;
         }
         return false;