diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ILanguageContributions.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ILanguageContributions.java index 256593af6..6ce0b1d34 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ILanguageContributions.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ILanguageContributions.java @@ -73,6 +73,8 @@ public interface ILanguageContributions { public CompletableFuture hasImplementation(); public CompletableFuture hasCodeAction(); + public CompletableFuture specialCaseHighlighting(); + public CompletableFuture getAnalyzerSummaryConfig(); public CompletableFuture getBuilderSummaryConfig(); public CompletableFuture getOndemandSummaryConfig(); diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/InterpretedLanguageContributions.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/InterpretedLanguageContributions.java index 14cc02637..fcf2a6236 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/InterpretedLanguageContributions.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/InterpretedLanguageContributions.java @@ -100,6 +100,8 @@ public class InterpretedLanguageContributions implements ILanguageContributions private final CompletableFuture hasImplementation; private final CompletableFuture hasCodeAction; + private final CompletableFuture specialCaseHighlighting; + private final CompletableFuture analyzerSummaryConfig; private final CompletableFuture builderSummaryConfig; private final CompletableFuture ondemandSummaryConfig; @@ -151,6 +153,10 @@ public InterpretedLanguageContributions(LanguageParameter lang, IBaseTextDocumen this.hasImplementation = nonNull(this.implementation); this.hasCodeAction = nonNull(this.codeAction); + this.specialCaseHighlighting = getContributionParameter(contributions, + LanguageContributions.PARSING, + LanguageContributions.Parameters.USES_SPECIAL_CASE_HIGHLIGHTING); + this.analyzerSummaryConfig = scheduledSummaryConfig(contributions, LanguageContributions.ANALYSIS); this.builderSummaryConfig = scheduledSummaryConfig(contributions, LanguageContributions.BUILD); this.ondemandSummaryConfig = ondemandSummaryConfig(contributions); @@ -170,10 +176,10 @@ private static CompletableFuture scheduledSummaryConfig(Completab var constructor = getContribution(c, summarizer); if (constructor != null) { return new SummaryConfig( - isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_HOVERS), - isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_DEFINITIONS), - isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_REFERENCES), - isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_IMPLEMENTATIONS)); + isTrue(constructor, LanguageContributions.Parameters.PROVIDES_HOVERS), + isTrue(constructor, LanguageContributions.Parameters.PROVIDES_DEFINITIONS), + isTrue(constructor, LanguageContributions.Parameters.PROVIDES_REFERENCES), + isTrue(constructor, LanguageContributions.Parameters.PROVIDES_IMPLEMENTATIONS)); } else { return SummaryConfig.FALSY; } @@ -202,6 +208,12 @@ private static boolean hasContribution(ISet contributions, String name) { return getContribution(contributions, name) != null; } + private static CompletableFuture getContributionParameter( + CompletableFuture contributions, String name, String parameter) { + + return contributions.thenApply(c -> isTrue(getContribution(c, name), parameter)); + } + private static boolean isTrue(@Nullable IConstructor constructor, String parameter) { if (constructor == null) { return false; @@ -388,6 +400,11 @@ public CompletableFuture hasBuild() { return hasBuild; } + @Override + public CompletableFuture specialCaseHighlighting() { + return specialCaseHighlighting; + } + @Override public CompletableFuture getAnalyzerSummaryConfig() { return analyzerSummaryConfig; diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/LanguageContributionsMultiplexer.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/LanguageContributionsMultiplexer.java index 8fb52e459..7eaa78f47 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/LanguageContributionsMultiplexer.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/LanguageContributionsMultiplexer.java @@ -77,6 +77,8 @@ private static final CompletableFuture failedInitialization() { private volatile CompletableFuture hasImplementation = failedInitialization(); private volatile CompletableFuture hasCodeAction = failedInitialization(); + private volatile CompletableFuture specialCaseHighlighting = failedInitialization(); + private volatile CompletableFuture analyzerSummaryConfig; private volatile CompletableFuture builderSummaryConfig; private volatile CompletableFuture ondemandSummaryConfig; @@ -159,6 +161,11 @@ private synchronized void calculateRouting() { hasReferences = anyTrue(ILanguageContributions::hasReferences); hasImplementation = anyTrue(ILanguageContributions::hasImplementation); + // Always use the special-case highlighting status of *the first* + // contribution (possibly using the default value in the Rascal ADT if + // it's not explicitly set), just as for `parsing` itself + specialCaseHighlighting = firstOrFail().specialCaseHighlighting(); + analyzerSummaryConfig = anyTrue(ILanguageContributions::getAnalyzerSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or); builderSummaryConfig = anyTrue(ILanguageContributions::getBuilderSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or); ondemandSummaryConfig = anyTrue(ILanguageContributions::getOndemandSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or); @@ -343,6 +350,11 @@ public CompletableFuture hasInlayHint() { return hasInlayHint; } + @Override + public CompletableFuture specialCaseHighlighting() { + return specialCaseHighlighting; + } + @Override public CompletableFuture getAnalyzerSummaryConfig() { return analyzerSummaryConfig; diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java index 93bb0e4e6..3c3efa605 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java @@ -462,9 +462,10 @@ public void shutdown() { } private CompletableFuture getSemanticTokens(TextDocumentIdentifier doc) { + var specialCaseHighlighting = contributions(doc).specialCaseHighlighting(); return recoverExceptions(getFile(doc).getCurrentTreeAsync() .thenApply(Versioned::get) - .thenApplyAsync(tokenizer::semanticTokensFull, ownExecuter) + .thenCombineAsync(specialCaseHighlighting, tokenizer::semanticTokensFull, ownExecuter) .whenComplete((r, e) -> logger.trace("Semantic tokens success, reporting {} tokens back", r == null ? 0 : r.getData().size() / 5) ) diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java index 3ba38ca3a..df2a7c3ff 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java @@ -59,6 +59,7 @@ public class ParserOnlyContribution implements ILanguageContributions { private final String name; private final @Nullable Exception loadingParserError; private final @Nullable IFunction parser; + private final CompletableFuture specialCaseHighlighting; public ParserOnlyContribution(String name, ParserSpecification spec) { this.name = name; @@ -67,6 +68,7 @@ public ParserOnlyContribution(String name, ParserSpecification spec) { Either result = loadParser(spec); this.parser = result.getLeft(); this.loadingParserError = result.getRight(); + this.specialCaseHighlighting = CompletableFuture.completedFuture(spec.getSpecialCaseHighlighting()); } @Override @@ -228,6 +230,11 @@ public CompletableFuture hasInlayHint() { return CompletableFuture.completedFuture(false); } + @Override + public CompletableFuture specialCaseHighlighting() { + return specialCaseHighlighting; + } + @Override public CompletableFuture getAnalyzerSummaryConfig() { return CompletableFuture.completedFuture(SummaryConfig.FALSY); diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/model/RascalADTs.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/model/RascalADTs.java index 837a3c2c2..918e1d46a 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/model/RascalADTs.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/model/RascalADTs.java @@ -47,8 +47,10 @@ private LanguageContributions () {} public static final String IMPLEMENTATION = "implementation"; public static final String CODE_ACTION = "codeAction"; - public static class Summarizers { - private Summarizers() {} + public static class Parameters { + private Parameters() {} + + public static final String USES_SPECIAL_CASE_HIGHLIGHTING = "usesSpecialCaseHighlighting"; public static final String PROVIDES_HOVERS = "providesHovers"; public static final String PROVIDES_DEFINITIONS = "providesDefinitions"; diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java index d57c79e11..000dfc0fa 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java @@ -361,9 +361,10 @@ public void shutdown() { } private CompletableFuture getSemanticTokens(TextDocumentIdentifier doc) { + var specialCaseHighlighting = CompletableFuture.completedFuture(false); return getFile(doc).getCurrentTreeAsync() .thenApply(Versioned::get) - .thenApplyAsync(tokenizer::semanticTokensFull, ownExecuter) + .thenCombineAsync(specialCaseHighlighting, tokenizer::semanticTokensFull, ownExecuter) .exceptionally(e -> { logger.error("Tokenization failed", e); return new SemanticTokens(Collections.emptyList()); diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/terminal/ITerminalIDEServer.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/terminal/ITerminalIDEServer.java index 6d8d1ceac..b4eda1be8 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/terminal/ITerminalIDEServer.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/terminal/ITerminalIDEServer.java @@ -528,13 +528,17 @@ public static class ParserSpecification { private final @Nullable Boolean nonTerminalIsStart; /** allowAmbiguity (default is false) */ private final @Nullable Boolean allowAmbiguity; + /** apply the special case for highlighting syntax-in-syntax, default: true */ + private final @Nullable Boolean specialCaseHighlighting; - public ParserSpecification(String parserLocation, String nonTerminalName, @Nullable Boolean nonTerminalIsStart, @Nullable Boolean allowAmbiguity) { + public ParserSpecification(String parserLocation, String nonTerminalName, + @Nullable Boolean nonTerminalIsStart, @Nullable Boolean allowAmbiguity, @Nullable Boolean specialCaseHighlighting) { this.parserLocation = parserLocation; this.nonTerminalName = nonTerminalName; this.nonTerminalIsStart = nonTerminalIsStart; this.allowAmbiguity = allowAmbiguity; + this.specialCaseHighlighting = specialCaseHighlighting; } public ISourceLocation getParserLocation() throws FactTypeUseException { @@ -553,6 +557,10 @@ public boolean getAllowAmbiguity() { return allowAmbiguity != null && allowAmbiguity; } + public boolean getSpecialCaseHighlighting() { + return specialCaseHighlighting == null || specialCaseHighlighting; + } + @Override public String toString() { return "ParserSpecification [parserLocation=" + parserLocation + ", nonTerminalName=" + nonTerminalName diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java index 5d0d7771f..cbd77f6fd 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizer.java @@ -39,15 +39,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.Nullable; -import org.eclipse.lsp4j.Range; import org.eclipse.lsp4j.SemanticTokenTypes; import org.eclipse.lsp4j.SemanticTokens; import org.eclipse.lsp4j.SemanticTokensCapabilities; import org.eclipse.lsp4j.SemanticTokensClientCapabilitiesRequests; -import org.eclipse.lsp4j.SemanticTokensDelta; import org.eclipse.lsp4j.SemanticTokensLegend; import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; -import org.eclipse.lsp4j.jsonrpc.messages.Either; import org.rascalmpl.values.parsetrees.ITree; import org.rascalmpl.values.parsetrees.ProductionAdapter; import org.rascalmpl.values.parsetrees.SymbolAdapter; @@ -60,7 +57,7 @@ import io.usethesource.vallang.IString; import io.usethesource.vallang.IValue; -public class SemanticTokenizer implements ISemanticTokens { +public class SemanticTokenizer { private static final Logger logger = LogManager.getLogger(SemanticTokenizer.class); @@ -74,24 +71,12 @@ public SemanticTokenizer(boolean rascal) { this.patch = rascal ? new RascalCategoryPatch() : new DefaultCategoryPatch(); } - @Override - public SemanticTokens semanticTokensFull(ITree tree) { + public SemanticTokens semanticTokensFull(ITree tree, boolean specialCaseHighlighting) { TokenList tokens = new TokenList(); - new TokenCollector(tokens, patch).collect(tree); + new TokenCollector(tokens, specialCaseHighlighting, patch).collect(tree); return new SemanticTokens(tokens.getTheList()); } - @Override - public Either semanticTokensFullDelta(String previousId, ITree tree) { - return Either.forLeft(semanticTokensFull(tree)); - } - - @Override - public SemanticTokens semanticTokensRange(Range range, ITree tree) { - return semanticTokensFull(tree); - } - - @Override public SemanticTokensWithRegistrationOptions options() { SemanticTokensWithRegistrationOptions result = new SemanticTokensWithRegistrationOptions(); SemanticTokensLegend legend = new SemanticTokensLegend(TokenTypes.getTokenTypes(), TokenTypes.getTokenModifiers()); @@ -102,7 +87,6 @@ public SemanticTokensWithRegistrationOptions options() { return result; } - @Override public SemanticTokensCapabilities capabilities() { SemanticTokensClientCapabilitiesRequests requests = new SemanticTokensClientCapabilitiesRequests(true); SemanticTokensCapabilities cps = new SemanticTokensCapabilities( @@ -394,67 +378,60 @@ public static int tokenTypeForName(@Nullable String category) { private static class TokenCollector { private int line; private int column; - private int startLineCurrentToken; - private int startColumnCurrentToken; - private String currentTokenCategory; private final boolean showAmb = false; - private TokenList tokens; + + private final TokenList tokens; + private final boolean specialCaseHighlighting; private final CategoryPatch patch; - public TokenCollector(TokenList tokens, CategoryPatch patch) { + public TokenCollector(TokenList tokens, boolean specialCaseHighlighting, CategoryPatch patch) { this.tokens = tokens; + this.specialCaseHighlighting = specialCaseHighlighting; this.patch = patch; + line = 0; column = 0; } public void collect(ITree tree) { collect(tree, null); - //check for final token - if (column > startColumnCurrentToken) { - tokens.addToken(startLineCurrentToken, startColumnCurrentToken, column - startColumnCurrentToken, currentTokenCategory); - } } - private void collect(ITree tree, @Nullable String currentCategory) { + private void collect(ITree tree, @Nullable String parentCategory) { if (tree.isAppl()) { - collectAppl(tree, currentCategory); + collectAppl(tree, parentCategory); } else if (tree.isAmb()) { - collectAmb(tree, currentCategory); + collectAmb(tree, parentCategory); } else if (tree.isChar()) { - collectChar(tree, currentCategory); + collectChar(tree, parentCategory); } } - @SuppressWarnings("java:S3776") // parsing tends to be complex - private void collectAppl(ITree arg, @Nullable String currentCategory) { + private void collectAppl(ITree tree, @Nullable String parentCategory) { String category = null; + if (TokenTypes.AMBIGUITY.equals(parentCategory)) { + category = TokenTypes.AMBIGUITY; + } - IValue catParameter = arg.asWithKeywordParameters().getParameter("category"); - - if (catParameter != null) { + IValue catParameter = tree.asWithKeywordParameters().getParameter("category"); + if (category == null && catParameter != null) { category = ((IString) catParameter).getValue(); } - IConstructor prod = TreeAdapter.getProduction(arg); - + IConstructor prod = TreeAdapter.getProduction(tree); if (category == null && ProductionAdapter.isDefault(prod)) { category = ProductionAdapter.getCategory(prod); } - if (category == null && currentCategory == null && (ProductionAdapter.isLiteral(prod) || ProductionAdapter.isCILiteral(prod))) { + if (category == null && parentCategory == null && isKeyword(tree)) { category = SemanticTokenTypes.Keyword; + } - // unless this is an operator - for (IValue child : TreeAdapter.getArgs(arg)) { - int c = TreeAdapter.getCharacter((ITree) child); - if (c != '-' && !Character.isJavaIdentifierPart(c)) { - category = null; - } - } + if (category == null) { + category = parentCategory; } // Apply a patch to the parse tree to dynamically add categories. In @@ -463,79 +440,54 @@ private void collectAppl(ITree arg, @Nullable String currentCategory) { // is empty and does nothing. category = patch.apply(prod, category); - // now we go down in the tree to find more tokens and to advance the counters - for (IValue child : TreeAdapter.getArgs(arg)) { + // Collect tokens from children + for (IValue child : TreeAdapter.getArgs(tree)) { //Propagate current category to child unless currently in a syntax nonterminal //*AND* the current child is a syntax nonterminal too - if (TreeAdapter.isAmb((ITree)child)) { - collectAmb((ITree) child, category != null ? category : currentCategory); - } - else if (!TreeAdapter.isChar((ITree) child) && ProductionAdapter.isSort(prod) && - ProductionAdapter.isSort(TreeAdapter.getProduction((ITree) child))) { - collect((ITree) child, null); - } else { - collect((ITree) child, category != null ? category : currentCategory); - } + var specialCase = specialCaseHighlighting && + !TreeAdapter.isChar((ITree) child) && ProductionAdapter.isSort(prod) && + ProductionAdapter.isSort(TreeAdapter.getProduction((ITree) child)); + + collect((ITree) child, specialCase ? null : category); } } - private void collectAmb(ITree arg, @Nullable String currentCategory) { - if (showAmb) { - startLineCurrentToken = line; - startColumnCurrentToken = column; + private void collectAmb(ITree tree, @Nullable String parentCategory) { + var category = showAmb ? TokenTypes.AMBIGUITY : parentCategory; + var child = (ITree) TreeAdapter.getAlternatives(tree).iterator().next(); + collect(child, category); + } - collect((ITree) TreeAdapter.getAlternatives(arg).iterator().next(), TokenTypes.AMBIGUITY); + private void collectChar(ITree tree, @Nullable String parentCategory) { + var ch = TreeAdapter.getCharacter(tree); + var length = Character.isSupplementaryCodePoint(ch) ? 2 : 1; // LSP counts 16-bit code units instead of Unicode code points + tokens.addToken(line, column, length, parentCategory); - tokens.addToken(startLineCurrentToken, startColumnCurrentToken, column - startColumnCurrentToken, TokenTypes.AMBIGUITY); + if (ch == '\n') { + line++; + column = 0; } else { - collect((ITree) TreeAdapter.getAlternatives(arg).iterator().next(), currentCategory); + column += length; } } - private void collectChar(ITree ch, @Nullable String currentCategory) { - int currentChar = TreeAdapter.getCharacter(ch); - //First check whether the token category has changed - if (currentCategory == null) { - //character has no semantic category - if (column > startColumnCurrentToken) { - //add token and set column offset - tokens.addToken(startLineCurrentToken, startColumnCurrentToken, column - startColumnCurrentToken, currentTokenCategory); - startColumnCurrentToken = column; - //startLineCurrentToken remains unchanged - } - currentTokenCategory = currentCategory; - } - if (currentCategory != null && !currentCategory.equals(currentTokenCategory)) { - //character has a semantic category that doesn't match the running token - if (column > startColumnCurrentToken) { - //add token and set column offset - tokens.addToken(startLineCurrentToken, startColumnCurrentToken, column - startColumnCurrentToken, currentTokenCategory); - startColumnCurrentToken = column; - } - currentTokenCategory = currentCategory; - //startLineCurrentToken remains unchanged - } + private static boolean isKeyword(ITree tree) { + var p = tree.getProduction(); - //Token administration done, advance column/line counters - if (currentChar == '\n') { - line++; + // A keyword must be a (ci)literal + if (!ProductionAdapter.isLiteral(p) && !ProductionAdapter.isCILiteral(p)) { + return false; + } - // this splits multi-line tokens automatically across the lines - if (currentTokenCategory != null) { - if (column > startColumnCurrentToken) { - tokens.addToken(startLineCurrentToken, startColumnCurrentToken, column - startColumnCurrentToken, currentTokenCategory); - } + // A keyword must consist solely of Java identifier chars or '-' + for (IValue child : TreeAdapter.getArgs(tree)) { + int ch = TreeAdapter.getCharacter((ITree) child); + if (ch != '-' && !Character.isJavaIdentifierPart(ch)) { + return false; } - startColumnCurrentToken = 0; - startLineCurrentToken = line; - column = 0; - } - else if (Character.isSupplementaryCodePoint(currentChar)) { - column += 2; // lsp counts 16-bit chars instead of 32bit codepoints - } - else { - column++; } + + return true; } } @@ -544,7 +496,7 @@ else if (Character.isSupplementaryCodePoint(currentChar)) { // (i.e., semantic token types). These categories should eventually be // incorporated directly in the grammar. Additional background: // https://github.com/SWAT-engineering/rascal-textmate/pull/6. - private interface CategoryPatch extends BiFunction {}; + private interface CategoryPatch extends BiFunction {} private static class DefaultCategoryPatch implements CategoryPatch { @Override diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizerTester.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizerTester.java new file mode 100644 index 000000000..7eee18f46 --- /dev/null +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizerTester.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.rascalmpl.vscode.lsp.util; + +import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IBool; +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.IValueFactory; + +public class SemanticTokenizerTester { + + private final IValueFactory values; + + public SemanticTokenizerTester(IValueFactory values) { + this.values = values; + } + + public IValue toTokens(IConstructor tree, IBool useSpecialCaseHighlighting, IBool applyRascalCategoryPatch) { + var tokenizer = new SemanticTokenizer(applyRascalCategoryPatch.getValue()); + var encoded = tokenizer.semanticTokensFull((ITree) tree, useSpecialCaseHighlighting.getValue()).getData(); + + var tokens = values.listWriter(); + var tokenTypes = tokenizer.capabilities().getTokenTypes(); + + for (int i = 0; i < encoded.size(); i += 5) { + var deltaLine = values.integer(encoded.get(i)); + var deltaStart = values.integer(encoded.get(i + 1)); + var length = values.integer(encoded.get(i + 2)); + var tokenType = values.string(tokenTypes.get(encoded.get(i + 3))); + var tokenModifier = values.string(""); // Token modifiers aren't supported yet + tokens.appendTuple(deltaLine, deltaStart, length, tokenType, tokenModifier); + } + + return tokens.done(); + } +} diff --git a/rascal-lsp/src/main/rascal/demo/lang/pico/LanguageServer.rsc b/rascal-lsp/src/main/rascal/demo/lang/pico/LanguageServer.rsc index 5e0eae424..3c48e6598 100644 --- a/rascal-lsp/src/main/rascal/demo/lang/pico/LanguageServer.rsc +++ b/rascal-lsp/src/main/rascal/demo/lang/pico/LanguageServer.rsc @@ -46,7 +46,7 @@ Here we group all services such that the LSP server can link them with the ((Language)) definition later. } set[LanguageService] picoLanguageServer() = { - parsing(parser(#start[Program])), + parsing(parser(#start[Program]), usesSpecialCaseHighlighting = false), documentSymbol(picoDocumentSymbolService), codeLens(picoCodeLenseService), execution(picoExecutionService), @@ -62,7 +62,7 @@ such that quicky loaded features can be made available while slower to load tools come in later. } set[LanguageService] picoLanguageServerSlowSummary() = { - parsing(parser(#start[Program])), + parsing(parser(#start[Program]), usesSpecialCaseHighlighting = false), analysis(picoAnalysisService, providesImplementations = false), build(picoBuildService) }; diff --git a/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/NestedCategories.rsc b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/NestedCategories.rsc new file mode 100644 index 000000000..ad5cdc279 --- /dev/null +++ b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/NestedCategories.rsc @@ -0,0 +1,96 @@ +@license{ +Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +} +module lang::rascal::tests::semanticTokenizer::NestedCategories + +import lang::rascal::tests::semanticTokenizer::Util; + +// ------- +// Grammar + +syntax Type + = TypeParameter + | @category="type" "str" + | @category="type" "map" "[" Type "," Type "]" + ; + +lexical TypeParameter + = @category="typeParameter" "&" Alnum+; + +lexical String + = @category="string" "\"" (Alnum | ("\<" Variable "\>"))* "\""; + +lexical Variable + = @category="variable" Alnum+; + +lexical Interface + = @category="interface" "I" Class; + +lexical InterfaceOuterOverInner + = @categoryNesting="outerOverInner" Interface; + +lexical Class + = @category="class" Upper Alnum*; + +lexical Alnum = [0-9 A-Z a-z]; +lexical Upper = [A-Z]; + +// ----- +// Tests + +test bool testType() = testTokenizer(#Type, + + "map[str,&T0]", + + expectFirst("map[str,", "type"), + expectFirst("&T0", "typeParameter"), // Inner over outer + expectFirst("]", "type") +); + +test bool testString() = testTokenizer(#String, + + "\"foo\\"", + + expectFirst("\"foo\<", "string"), + expectFirst("bar", "variable"), // Inner over outer + expectFirst("\>\"", "string") +); + +test bool testInterface() = testTokenizer(#Interface, + + "IFoo", + + expectFirst("I", "interface"), + expectFirst("Foo", "class"), + expectEachNot("IFoo", "interface") // *Not* outer over inner + + // This test demonstrates that, sometimes, arguably the "natural" way to + // write a grammar (e.g., "An interface name is just any class name, but + // prefixed with an I") requires outer-over-inner semantic tokenization. + // This is currently not supported (i.e., the only "solution" right now is + // to rewrite the grammar). Further reading: + // https://github.com/usethesource/rascal-language-servers/issues/456 +); diff --git a/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Pico.rsc b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Pico.rsc new file mode 100644 index 000000000..7176c80cf --- /dev/null +++ b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Pico.rsc @@ -0,0 +1,55 @@ +@license{ +Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +} +module lang::rascal::tests::semanticTokenizer::Pico + +import lang::rascal::tests::semanticTokenizer::Util; + +// ------- +// Grammar + +import lang::pico::\syntax::Main; + +// ----- +// Tests + +test bool testKeywordLastLine() = testTokenizer(#Program, + + "begin + declare + x: natural, + y: natural; + + x := 5; + y := x + 1 + end", + + expectFirst("begin", "keyword"), + expectFirst("declare", "keyword"), + expectNth(0, "natural", "keyword"), + expectNth(1, "natural", "keyword"), + expectFirst("end", "keyword") // https://github.com/usethesource/rascal-language-servers/issues/90 +); diff --git a/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Rascal.rsc b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Rascal.rsc new file mode 100644 index 000000000..85e41067b --- /dev/null +++ b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Rascal.rsc @@ -0,0 +1,129 @@ +@license{ +Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +} +module lang::rascal::tests::semanticTokenizer::Rascal + +import lang::rascal::tests::semanticTokenizer::Util; + +// ------- +// Grammar + +import lang::rascal::\syntax::Rascal; + +// ----- +// Tests + +test bool testTypesAndValues() = testTokenizer(#Declaration, + + "void f() { + bool b = true; + int i = 3; + real r = 3.14; + str s = \"foo\\"; + loc l = |unknown:///|; + tuple[int, int] \\tuple = \<3, 14\>; + }", + + expectFirst("void", "keyword"), + expectFirst("bool", "keyword"), + expectFirst("int", "keyword"), + expectFirst("real", "keyword"), + expectFirst("loc", "keyword"), + expectFirst("str", "keyword"), + expectFirst("tuple", "keyword"), + + expectFirst("f", "uncategorized"), + expectFirst("true", "keyword"), + expectFirst("3", "number"), // https://github.com/usethesource/rascal-language-servers/issues/456 + expectFirst("3.14", "number"), + expectFirst("foo", "string"), + expectFirst("\<", "string"), + expectFirst("bar", "uncategorized"), + expectFirst("\>", "string"), + expectFirst("|unknown:///|", "string"), // https://github.com/usethesource/rascal-language-servers/issues/456 + expectLast("\<", "uncategorized"), + expectLast("\>", "uncategorized"), + + applyRascalCategoryPatch = true +); + +test bool testComments() = testTokenizer(#Declaration, + + "void f() { + /* Block comment */ + /* Multi-line 1 + Multi-line 2 */ + // Line comment + }", + + expectFirst("Block comment", "comment"), + expectFirst("Multi-line 1", "comment"), + expectFirst("Multi-line 2", "comment"), // https://github.com/usethesource/rascal-language-servers/issues/20 + expectFirst("Line comment", "comment"), + + applyRascalCategoryPatch = true +); + +test bool testTags() = testTokenizer(#Declaration, + + "@synopsis{Foo} + @category=\"bar\" + @memo + int i = 0;", + + expectFirst("@synopsis{Foo}", "comment"), + expectFirst("@category=", "comment"), + expectFirst("\"bar\"", "string"), + expectFirst("@memo", "comment"), + + applyRascalCategoryPatch = true +); + +test bool testUnicode() = testTokenizer(#Declaration, + "void f() { + str s = \"𝄞𝄞𝄞\"; + }", + + expectFirst("str", "keyword"), + expectFirst(" s = ", "uncategorized"), + expectFirst("\"𝄞𝄞𝄞\"", "string"), // https://github.com/usethesource/rascal-language-servers/issues/19 + expectFirst(";", "uncategorized"), + applyRascalCategoryPatch = true +); + +test bool testSpecialCaseHighlighting() = testTokenizer(#Declaration, + + "void f() { + int i = 3; + loc l = |unknown:///|; + }", + + expectFirst("3", "uncategorized"), // Instead of `number` + expectFirst("|unknown:///|", "uncategorized"), // Instead of `string` + + useSpecialCaseHighlighting = true, + applyRascalCategoryPatch = true +); diff --git a/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Util.rsc b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Util.rsc new file mode 100644 index 000000000..6d642802c --- /dev/null +++ b/rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Util.rsc @@ -0,0 +1,175 @@ +@license{ +Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +} +module lang::rascal::tests::semanticTokenizer::Util + +import IO; +import List; +import ParseTree; +import String; + +// This is the main utility function to test the semantic tokenizer. All other +// functions in this module are auxiliary (and private). +bool testTokenizer(type[&T<:Tree] begin, str input, Expect expects..., + bool printActuals = false, + bool useSpecialCaseHighlighting = false, + bool applyRascalCategoryPatch = false) { + + // First, compute the tokens by calling the semantic tokenizer (in Java) + Tree tree = parse(begin, input); + list[SemanticToken] tokens = toTokens(tree, useSpecialCaseHighlighting, applyRascalCategoryPatch); + + // Next, compute the absolute location of each token (i.e., the position of + // each token in `tokens` is represented *relative to* its predecessor) + list[loc] locations = toAbsoluteLocations(input, tokens); + + // Next, compute the string of each token + list[str] strings = [substring(input, l.offset, l.offset + l.length) | l <- locations]; + + // Last, test the expectations + list[Actual] actuals = sort(zip3(tokens, locations, strings), less); + + if (printActuals) { + iprintln(actuals); + } + + return (true | it && check(actuals, expect) | expect <- expects); +} + +private list[loc] toAbsoluteLocations(str input, list[SemanticToken] tokens) { + list[str] lines = split("\n", input); + + // Initialize the "cursor" + int line = 0; + int column = 0; + + return for (token <- tokens) { + column = token.deltaLine > 0 ? 0 : column; // Reset column? + + // Advance the "cursor" + line += token.deltaLine; + column += toLength32(token.deltaStart, substring(lines[line], column)); + + // Compute the absolute location of `token` + int offset = (0 | it + size(lines[i]) + 1 | i <- [0..line]) + column; + int length = toLength32(token.length, substring(lines[line], column)); + append |unknown:///|(offset, length); + } +} + +// +// Semantic tokens +// + +alias SemanticToken = tuple[ + int deltaLine, + int deltaStart, + int length, + str tokenType, + str tokenModifier]; + +@javaClass{org.rascalmpl.vscode.lsp.util.SemanticTokenizerTester} +java list[SemanticToken] toTokens(Tree _, bool _, bool _); + +// +// Length conversion: Characters are counted as 16-bit units in LSP, whereas +// they are counted as 32-bit units in Rascal, so lengths need to be converted. +// The difference manifests only in the presence of surrogate pairs. +// + +alias Length32 = int; +alias Length16 = int; + +private Length32 toLength32(Length16 n16, str input) { + int n32 = 0; + for (c <- chars(input)) { + if (n16 <= 0) break; + n16 -= isSupplementaryCodePoint(c) ? 2 : 1; + n32 += 1; + } + return n32; +} + +private bool isSupplementaryCodePoint(int c) + = MIN_SUPPLEMENTARY_CODEPOINT <= c && c <= MAX_CODE_POINT; + +private int MIN_SUPPLEMENTARY_CODEPOINT = 0x010000; +private int MAX_CODE_POINT = 0x10FFFF; + +// +// Actuals +// + +alias Actual = tuple[SemanticToken token, loc l, str s]; + +private bool less(Actual a1, Actual a2) + = a1.l.offset < a2.l.offset; + +private list[Actual] filterByTokenType(list[Actual] actuals, str tokenType) + = [a | Actual a: <<_, _, _, tokenType, _>, _, _> <- actuals]; + +private list[Actual] filterByString(list[Actual] actuals, str string) + = [a | a <- actuals, contains(a.s, string)]; + +// +// Expects +// + +data Expect // Represents... + // ...the expectation that the `n`-th/first/last occurrence of `string` is + // in a token of type `tokenType` + = expectNth(int n, str string, str tokenType) + | expectFirst(str string, str tokenType) + | expectLast(str string, str tokenType) + + // ...the expectation that each occurrence of `string` is not in a token of + // type `tokenType` + | expectEachNot(str string, str tokenType) + ; + +private bool check(list[Actual] actuals, expectNth(n, string, tokenType)) { + actuals = filterByString(actuals, string); + assert [] != actuals[n..(n + 1)] : "Unexpected string: \"\""; + assert n >= 0 : "Expected `n` to be non-negative. Actual: ``."; + assert <<_, _, _, tokenType, _>, _, _> := actuals[n] : "Expected token type of \"\": . Actual: ."; + return true; +} + +private bool check(list[Actual] actuals, expectFirst(string, tokenType)) { + return check(actuals, expectNth(0, string, tokenType)); +} + +private bool check(list[Actual] actuals, expectLast(string, tokenType)) { + actuals = filterByString(actuals, string); + return check(actuals, expectNth(size(actuals) - 1, string, tokenType)); +} + +private bool check(list[Actual] actuals, expectEachNot(string, tokenType)) { + actuals = filterByString(actuals, string); + actuals = filterByTokenType(actuals, tokenType); + assert [] == actuals : "Not-expected token type of \"\": ``. Actual: ``.";; + return true; +} diff --git a/rascal-lsp/src/main/rascal/util/LanguageServer.rsc b/rascal-lsp/src/main/rascal/util/LanguageServer.rsc index 58bce9065..caa213c9a 100644 --- a/rascal-lsp/src/main/rascal/util/LanguageServer.rsc +++ b/rascal-lsp/src/main/rascal/util/LanguageServer.rsc @@ -170,6 +170,17 @@ particular downstream services make use of the `src` origin fields that the pars * Parsers can be obtained automatically using the ((ParseTree::parser)) or ((ParseTree::parsers)) functions, like so `parser(#start[Program])`. Like this a fast parser is obtained that does not require a global interpreter lock. If you pass in a normal Rascal function, which is fine, the global interpreter lock will make the editor services less responsive. + * Currently, `@category` tags are ignored in the following special case: + * if a parse tree has a `syntax` non-terminal node `n` with a category + (either declared as part of `n`, or inherited from an ancestors), + * and if `n` has a `syntax` non-terminal node `m` as a child, + * then the category of `n` is ignored in the subtree rooted at `m` + (regardless of whether a category is declared as part of `m`). + This special case is deprecated and will be removed in a future release. In + anticipation of the removal, users that rely on this special case for + syntax highlighting can update their grammars and explicitly opt-out of the + special case by passing `usesSpecialCaseHighlighting = false` when + registering the ((parsing)) service. * The ((analysis)) service indexes a file as a ((Summary)), offering precomputed relations for looking up hover documentation, definition with uses, references to declarations, implementations of types and compiler errors and warnings. * ((analysis)) focuses on their own file, but may reuse cached or stored indices from other files. @@ -234,7 +245,8 @@ typical programming language concepts. Since these are all just `rel[loc, loc]` * `providesDocumentation` is deprecated. Use `providesHovers` instead. } data LanguageService - = parsing(Tree (str _input, loc _origin) parsingService) + = parsing(Tree (str _input, loc _origin) parsingService + , bool usesSpecialCaseHighlighting = true) | analysis(Summary (loc _origin, Tree _input) analysisService , bool providesDocumentation = true , bool providesHovers = providesDocumentation diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/ISemanticTokens.java b/rascal-lsp/src/test/java/engineering/swat/rascal/lsp/util/SemanticTokenizerTests.java similarity index 65% rename from rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/ISemanticTokens.java rename to rascal-lsp/src/test/java/engineering/swat/rascal/lsp/util/SemanticTokenizerTests.java index 80cabd81d..03587dcc2 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/ISemanticTokens.java +++ b/rascal-lsp/src/test/java/engineering/swat/rascal/lsp/util/SemanticTokenizerTests.java @@ -24,20 +24,12 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -package org.rascalmpl.vscode.lsp.util; +package engineering.swat.rascal.lsp.util; -import org.eclipse.lsp4j.Range; -import org.eclipse.lsp4j.SemanticTokens; -import org.eclipse.lsp4j.SemanticTokensCapabilities; -import org.eclipse.lsp4j.SemanticTokensDelta; -import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; -import org.eclipse.lsp4j.jsonrpc.messages.Either; -import org.rascalmpl.values.parsetrees.ITree; +import org.junit.runner.RunWith; +import org.rascalmpl.test.infrastructure.RascalJUnitTestPrefix; +import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; -public interface ISemanticTokens { - SemanticTokensCapabilities capabilities(); - SemanticTokensWithRegistrationOptions options(); - SemanticTokens semanticTokensFull(ITree tree); - Either semanticTokensFullDelta(String previousDelta, ITree tree); - SemanticTokens semanticTokensRange(Range range, ITree tree); -} +@RunWith(RascalJUnitTestRunner.class) +@RascalJUnitTestPrefix("lang::rascal::tests::semanticTokenizer") +public class SemanticTokenizerTests {} diff --git a/rascal-vscode-extension/src/lsp/ParameterizedLanguageServer.ts b/rascal-vscode-extension/src/lsp/ParameterizedLanguageServer.ts index 29957d47e..d05d5ec50 100644 --- a/rascal-vscode-extension/src/lsp/ParameterizedLanguageServer.ts +++ b/rascal-vscode-extension/src/lsp/ParameterizedLanguageServer.ts @@ -183,7 +183,11 @@ export interface ParserSpecification { nonTerminalIsStart?: boolean; /** allow ambiguities during parsing, default: false */ allowAmbiguity?: boolean; - + /** apply the special case for highlighting syntax-in-syntax, default: true + Note: This is a temporary property. In a short-term future release, the + default will become `false`. In a mid-term future release, the property + will be removed (and the special case no longer exists). */ + specialCaseHighlighting?: boolean; } function languageKey(lang: LanguageParameter) {