Error recovery for Rascal

rascal-lsp/pom.xml

@@ -65,7 +65,7 @@
     <dependency>
       <groupId>org.rascalmpl</groupId>
       <artifactId>rascal</artifactId>
-      <version>0.40.11</version>
+      <version>0.40.9-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.rascalmpl</groupId>

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/TextDocumentState.java

@@ -30,9 +30,13 @@
 import java.util.function.BiFunction;
 
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.rascalmpl.library.util.ErrorRecovery;
+import org.rascalmpl.values.RascalValueFactory;
+import org.rascalmpl.values.ValueFactoryFactory;
 import org.rascalmpl.values.parsetrees.ITree;
 import org.rascalmpl.vscode.lsp.util.Versioned;
 
+import io.usethesource.vallang.IList;
 import io.usethesource.vallang.ISourceLocation;
 
 /**
@@ -51,7 +55,9 @@ public class TextDocumentState {
     @SuppressWarnings("java:S3077") // we are use volatile correctly
     private volatile Versioned<String> currentContent;
     @SuppressWarnings("java:S3077") // we are use volatile correctly
-    private volatile @MonotonicNonNull Versioned<ITree> lastFullTree;
+    private volatile @MonotonicNonNull Versioned<ITree> lastTree;
+    @SuppressWarnings("java:S3077") // we are use volatile correctly
+    private volatile @MonotonicNonNull Versioned<ITree> lastTreeWithoutErrors;
     @SuppressWarnings("java:S3077") // we are use volatile correctly
     private volatile CompletableFuture<Versioned<ITree>> currentTree;
 
@@ -83,9 +89,14 @@ public CompletableFuture<Versioned<ITree>> update(int version, String content) {
     private CompletableFuture<Versioned<ITree>> newTreeAsync(int version, String content) {
         return parser.apply(file, content)
             .thenApply(t -> new Versioned<ITree>(version, t))
-            .whenComplete((r, t) -> {
-                if (r != null) {
-                    lastFullTree = r;
+            .whenComplete((t, error) -> {
+                if (t != null) {
+                    RascalValueFactory valueFactory = (RascalValueFactory) ValueFactoryFactory.getValueFactory();
+                    IList errors = new ErrorRecovery(valueFactory).findAllErrors(t.get());
+                    if (errors.isEmpty()) {
+                        lastTreeWithoutErrors = t;
+                    }
+                    lastTree = t;
                 }
             });
     }
@@ -94,8 +105,12 @@ public CompletableFuture<Versioned<ITree>> getCurrentTreeAsync() {
         return currentTree;
     }
 
-    public @MonotonicNonNull Versioned<ITree> getMostRecentTree() {
-        return lastFullTree;
+    public @MonotonicNonNull Versioned<ITree> getLastTree() {
+        return lastTree;
+    }
+
+    public @MonotonicNonNull Versioned<ITree> getLastTreeWithoutErrors() {
+        return lastTreeWithoutErrors;
     }
 
     public ISourceLocation getLocation() {

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java

@@ -342,7 +342,7 @@ public CompletableFuture<List<InlayHint>> inlayHint(InlayHintParams params) {
         final TextDocumentState file = getFile(params.getTextDocument());
         final ILanguageContributions contrib = contributions(params.getTextDocument());
         return recoverExceptions(
-                recoverExceptions(file.getCurrentTreeAsync(), file::getMostRecentTree)
+                recoverExceptions(file.getCurrentTreeAsync(), file::getLastTreeWithoutErrors)
                 .thenApply(Versioned::get)
                 .thenApply(contrib::inlayHint)
                 .thenCompose(InterruptibleFuture::get)

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java

@@ -94,7 +94,7 @@ private static Either<IFunction, Exception> loadParser(ParserSpecification spec)
         try {
             logger.debug("Loading parser {} at {}", reifiedType, spec.getParserLocation());
             // this hides all the loading and instantiation details of Rascal-generated parsers
-            var parser = vf.loadParser(reifiedType, spec.getParserLocation(), VF.bool(spec.getAllowAmbiguity()), VF.bool(false), VF.bool(false), vf.set());
+            var parser = vf.loadParser(reifiedType, spec.getParserLocation(), VF.bool(spec.getAllowAmbiguity()), VF.bool(false), VF.bool(false), VF.bool(false), vf.set());
             logger.debug("Got parser: {}", parser);
             return Either.forLeft(parser);
         }

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java

@@ -51,6 +51,7 @@
 import org.eclipse.lsp4j.jsonrpc.messages.ResponseErrorCode;
 import org.rascalmpl.exceptions.Throw;
 import org.rascalmpl.interpreter.Evaluator;
+import org.rascalmpl.library.util.ErrorRecovery;
 import org.rascalmpl.library.util.PathConfig;
 import org.rascalmpl.values.IRascalValueFactory;
 import org.rascalmpl.values.functions.IFunction;
@@ -76,6 +77,7 @@
 
 public class RascalLanguageServices {
     private static final IValueFactory VF = IRascalValueFactory.getInstance();
+    private static final ErrorRecovery RECOVERY = new ErrorRecovery(IRascalValueFactory.getInstance());
 
     private static final Logger logger = LogManager.getLogger(RascalLanguageServices.class);
 
@@ -236,8 +238,13 @@ public List<CodeLensSuggestion> locateCodeLenses(ITree tree) {
         List<CodeLensSuggestion> result = new ArrayList<>(2);
         result.add(new CodeLensSuggestion(module, "Import in new Rascal terminal", "rascalmpl.importModule", moduleName));
 
-        for (IValue topLevel : TreeAdapter
-            .getListASTArgs(TreeAdapter.getArg(TreeAdapter.getArg(tree, "body"), "toplevels"))) {
+        ITree body = TreeAdapter.getArg(tree, "body");
+        ITree toplevels = TreeAdapter.getArg(body, "toplevels");
+        for (IValue topLevel : TreeAdapter.getListASTArgs(toplevels)) {
+            if (RECOVERY.hasErrors((ITree) topLevel)) {
+                continue;
+            }
+
             ITree decl = TreeAdapter.getArg((ITree) topLevel, "declaration");
             if ("function".equals(TreeAdapter.getConstructorName(decl))) {
                 ITree signature = TreeAdapter.getArg(TreeAdapter.getArg(decl, "functionDeclaration"), "signature");

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java

@@ -84,8 +84,11 @@
 import org.eclipse.lsp4j.services.LanguageClient;
 import org.eclipse.lsp4j.services.LanguageClientAware;
 import org.rascalmpl.library.Prelude;
+import org.rascalmpl.library.util.ErrorRecovery;
 import org.rascalmpl.parser.gtd.exception.ParseError;
 import org.rascalmpl.uri.URIResolverRegistry;
+import org.rascalmpl.values.RascalValueFactory;
+import org.rascalmpl.values.ValueFactoryFactory;
 import org.rascalmpl.values.parsetrees.ITree;
 import org.rascalmpl.vscode.lsp.BaseWorkspaceService;
 import org.rascalmpl.vscode.lsp.IBaseLanguageClient;
@@ -105,6 +108,7 @@
 import org.rascalmpl.vscode.lsp.util.locations.LineColumnOffsetMap;
 import org.rascalmpl.vscode.lsp.util.locations.Locations;
 
+import io.usethesource.vallang.IList;
 import io.usethesource.vallang.ISourceLocation;
 import io.usethesource.vallang.IValue;
 
@@ -215,26 +219,35 @@ private TextDocumentState updateContents(VersionedTextDocumentIdentifier doc, St
 
     private void handleParsingErrors(TextDocumentState file, CompletableFuture<Versioned<ITree>> futureTree) {
         futureTree.handle((tree, excp) -> {
-            Diagnostic newParseError = null;
-            if (excp != null && excp instanceof CompletionException) {
+            List<Diagnostic> parseErrors = new ArrayList<>();
+
+            if (excp instanceof CompletionException) {
                 excp = excp.getCause();
             }
+
             if (excp instanceof ParseError) {
-                newParseError = Diagnostics.translateDiagnostic((ParseError)excp, columns);
-            }
-            else if (excp != null) {
+                parseErrors.add(Diagnostics.translateDiagnostic((ParseError)excp, columns));
+            } else if (excp != null) {
                 logger.error("Parsing crashed", excp);
-                newParseError = new Diagnostic(
+                parseErrors.add(new Diagnostic(
                     new Range(new Position(0,0), new Position(0,1)),
                     "Parsing failed: " + excp.getMessage(),
                     DiagnosticSeverity.Error,
-                    "Rascal Parser");
+                    "Rascal Parser"));
+            }
+
+            if (tree != null) {
+                RascalValueFactory valueFactory = (RascalValueFactory) ValueFactoryFactory.getValueFactory();
+                IList errors = new ErrorRecovery(valueFactory).findAllErrors(tree.get());
+                for (IValue error : errors) {
+                    ITree errorTree = (ITree) error;
+                    parseErrors.add(Diagnostics.translateErrorRecoveryDiagnostic(errorTree, columns));
+                }
             }
-            logger.trace("Finished parsing tree, reporting new parse error: {} for: {}", newParseError, file.getLocation());
 
+            logger.trace("Finished parsing tree, reporting new parse errors: {} for: {}", parseErrors, file.getLocation());
             if (facts != null) {
-                facts.reportParseErrors(file.getLocation(),
-                    newParseError == null ? Collections.emptyList() : Collections.singletonList(newParseError));
+                facts.reportParseErrors(file.getLocation(), parseErrors);
             }
             return null;
         });
@@ -267,7 +280,7 @@ public CompletableFuture<Either<List<? extends Location>, List<? extends Locatio
         TextDocumentState file = getFile(params.getTextDocument());
         return file.getCurrentTreeAsync()
             .thenApply(Versioned::get)
-            .handle((t, r) -> (t == null ? (file.getMostRecentTree().get()) : t))
+            .handle((t, r) -> (t == null ? (file.getLastTreeWithoutErrors().get()) : t))
             .thenCompose(tr -> rascalServices.getOutline(tr).get())
             .thenApply(c -> Outline.buildOutline(c, columns.get(file.getLocation())))
             ;
@@ -285,7 +298,7 @@ public CompletableFuture<WorkspaceEdit> rename(RenameParams params) {
 
         return file.getCurrentTreeAsync()
             .thenApply(Versioned::get)
-            .handle((t, r) -> (t == null ? (file.getMostRecentTree().get()) : t))
+            .handle((t, r) -> (t == null ? (file.getLastTreeWithoutErrors().get()) : t))
             .thenCompose(tr -> rascalServices.getRename(tr, params.getPosition(), workspaceFolders, facts::getPathConfig, params.getNewName(), columns).get())
             .thenApply(c -> new WorkspaceEdit(DocumentChanges.translateDocumentChanges(this, c)))
             ;
@@ -395,7 +408,7 @@ public CompletableFuture<List<? extends CodeLens>> codeLens(CodeLensParams param
             .handle((r, e) -> {
                 // fallback to tree if a parsing error occurred.
                 if (r == null) {
-                    r = f.getMostRecentTree();
+                    r = f.getLastTreeWithoutErrors();
                 }
                 if (r == null) {
                     throw new RuntimeException(e);

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/Diagnostics.java

@@ -42,6 +42,8 @@
 import org.eclipse.lsp4j.Range;
 import org.rascalmpl.parser.gtd.exception.ParseError;
 import org.rascalmpl.values.ValueFactoryFactory;
+import org.rascalmpl.values.parsetrees.ITree;
+import org.rascalmpl.values.parsetrees.TreeAdapter;
 import org.rascalmpl.vscode.lsp.IBaseTextDocumentService;
 import org.rascalmpl.vscode.lsp.util.locations.ColumnMaps;
 import org.rascalmpl.vscode.lsp.util.locations.LineColumnOffsetMap;
@@ -74,6 +76,9 @@ public static Diagnostic translateDiagnostic(ParseError e, ColumnMaps cm) {
         return new Diagnostic(toRange(e, cm), e.getMessage(), DiagnosticSeverity.Error, "parser");
     }
 
+    public static Diagnostic translateErrorRecoveryDiagnostic(ITree errorTree, ColumnMaps cm) {
+        return new Diagnostic(toRange(errorTree, cm), "Parse error", DiagnosticSeverity.Error, "parser");
+    }
 
     public static Diagnostic translateRascalParseError(IValue e, ColumnMaps cm) {
         if (e instanceof IConstructor) {
@@ -121,14 +126,27 @@ public static Diagnostic translateDiagnostic(IConstructor d, Range range) {
         return result;
     }
 
+    private static Range toRange(ITree t, ColumnMaps cm) {
+        return toRange(TreeAdapter.getLocation(t), cm);
+    }
+
     private static Range toRange(ParseError pe, ColumnMaps cm) {
-        ISourceLocation loc = pe.getLocation();
+        return toRange(pe.getLocation(), cm);
+    }
+
+    private static Range toRange(ISourceLocation loc, ColumnMaps cm) {
         if (loc.getBeginLine() == loc.getEndLine() && loc.getBeginColumn() == loc.getEndColumn()) {
             // zero width parse error is not something LSP likes, so we make it one char wider
-            loc = ValueFactoryFactory.getValueFactory().sourceLocation(loc,
-                loc.getOffset(), loc.getLength() + 1,
-                loc.getBeginLine(), loc.getBeginColumn(),
-                loc.getEndLine(), loc.getEndColumn() + 1);
+            try {
+                loc = ValueFactoryFactory.getValueFactory().sourceLocation(loc,
+                    loc.getOffset(), loc.getLength() + 1,
+                    loc.getBeginLine(), loc.getBeginColumn(),
+                    loc.getEndLine(), loc.getEndColumn() + 1);
+            } catch (Throwable t) {
+                logger.trace("Cannot extend 0-width location for parse error: " + t.getMessage());
+                loc = ValueFactoryFactory.getValueFactory().sourceLocation(
+                    loc, 0, 1, 1, 1, 0, 1);
+            }
         }
         return Locations.toRange(loc, cm);
     }

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/RascalServices.java

@@ -27,19 +27,39 @@
 package org.rascalmpl.vscode.lsp.util;
 
 import org.rascalmpl.library.lang.rascal.syntax.RascalParser;
+import org.rascalmpl.library.util.ErrorRecovery;
 import org.rascalmpl.parser.Parser;
-import org.rascalmpl.parser.gtd.result.action.IActionExecutor;
 import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener;
+import org.rascalmpl.parser.gtd.util.StackNodeIdDispenser;
 import org.rascalmpl.parser.uptr.UPTRNodeFactory;
 import org.rascalmpl.parser.uptr.action.NoActionExecutor;
+import org.rascalmpl.parser.uptr.recovery.ToTokenRecoverer;
+import org.rascalmpl.values.RascalValueFactory;
+import org.rascalmpl.values.ValueFactoryFactory;
 import org.rascalmpl.values.parsetrees.ITree;
 
+import io.usethesource.vallang.IBool;
 import io.usethesource.vallang.ISourceLocation;
 
 public class RascalServices {
+
+    private static final RascalValueFactory VALUE_FACTORY = (RascalValueFactory) ValueFactoryFactory.getValueFactory();
+    private static final IBool TRUE = VALUE_FACTORY.bool(true);
+
     public static ITree parseRascalModule(ISourceLocation loc, char[] input) {
-        IActionExecutor<ITree> actions = new NoActionExecutor();
-        return new RascalParser().parse(Parser.START_MODULE, loc.getURI(), input, actions,
-            new DefaultNodeFlattener<>(), new UPTRNodeFactory(true));
+        // TODO: Which of these objects are stateless and can be reused?
+
+        // Parse
+        RascalParser parser = new RascalParser();
+        ITree tree = parser.parse(
+            Parser.START_MODULE, loc.getURI(), input,
+            new NoActionExecutor(),
+            new DefaultNodeFlattener<>(),
+            new UPTRNodeFactory(true),
+            new ToTokenRecoverer(loc.getURI(), parser, new StackNodeIdDispenser(parser)));
+
+        // Recover
+        ErrorRecovery recoverer = new ErrorRecovery(VALUE_FACTORY);
+        return (ITree) recoverer.disambiguateErrors(tree, TRUE);
     }
 }