Add semantic tokenizer tests

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/util/SemanticTokenizerTester.java

@@ -0,0 +1,31 @@
+package org.rascalmpl.vscode.lsp.util;
+
+import org.rascalmpl.values.IRascalValueFactory;
+import org.rascalmpl.values.parsetrees.ITree;
+
+import io.usethesource.vallang.IBool;
+import io.usethesource.vallang.IConstructor;
+import io.usethesource.vallang.IValue;
+
+public class SemanticTokenizerTester {
+
+    public IValue toTokens(IConstructor tree, IBool applyRascalCategoryPatch) {
+        var tokenizer = new SemanticTokenizer(applyRascalCategoryPatch.getValue());
+        var encoded = tokenizer.semanticTokensFull((ITree) tree).getData();
+
+        var values = IRascalValueFactory.getInstance();
+        var tokens = new IValue[encoded.size() / 5];
+        var tokenTypes = tokenizer.capabilities().getTokenTypes();
+
+        for (int i = 0; i < encoded.size(); i += 5) {
+            var deltaLine     = values.integer(encoded.get(i));
+            var deltaStart    = values.integer(encoded.get(i + 1));
+            var length        = values.integer(encoded.get(i + 2));
+            var tokenType     = values.string(tokenTypes.get(encoded.get(i + 3)));
+            var tokenModifier = values.string(""); // Token modifiers aren't supported yet
+            tokens[i / 5] = values.tuple(deltaLine, deltaStart, length, tokenType, tokenModifier);
+        }
+
+        return values.list(tokens);
+    }
+}

rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/NestedCategories.rsc

@@ -0,0 +1,93 @@
+@license{
+Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+}
+module lang::rascal::tests::semanticTokenizer::NestedCategories
+
+import lang::rascal::tests::semanticTokenizer::Util;
+
+// -------
+// Grammar
+
+syntax Type
+    = TypeParameter
+    | @category="type" "str"
+    | @category="type" "map" "[" Type "," Type "]"
+    ;
+
+lexical TypeParameter
+    = @category="typeParameter" "&" Alnum+;
+
+lexical String
+    = @category="string" "\"" (Alnum | ("\<" Variable "\>"))* "\"";
+
+lexical Variable
+    = @category="variable" Alnum+;
+
+lexical Interface
+    = @category="interface" "I" Class;
+
+lexical Class
+    = @category="class" Upper Alnum*;
+
+lexical Alnum = [0-9 A-Z a-z];
+lexical Upper = [A-Z];
+
+// -----
+// Tests
+
+test bool testType() = testTokenizer(#Type,
+
+    "map[str,&T0]",
+
+    expectFirst("map[str,", "type"),
+    expectFirst("&T0", "typeParameter"), // Inner over outer
+    expectFirst("]", "type")
+);
+
+test bool testString() = testTokenizer(#String,
+
+    "\"foo\<bar\>\"",
+
+    expectFirst("\"foo\<", "string"),
+    expectFirst("bar", "variable"), // Inner over outer
+    expectFirst("\>\"", "string")
+);
+
+test bool testInterface() = testTokenizer(#Interface,
+
+    "IFoo",
+
+    expectFirst("I", "interface"),
+    expectFirst("Foo", "class"),
+    expectEachNot("IFoo", "interface") // *Not* outer over inner
+
+    // This test demonstrates that, sometimes, arguably the "natural" way to
+    // write a grammar (e.g., "An interface name is just any class name, but
+    // prefixed with an I") requires outer-over-inner semantic tokenization.
+    // This is currently not supported (i.e., the only "solution" right now is
+    // to rewrite the grammar). Further reading:
+    // https://github.com/usethesource/rascal-language-servers/issues/456
+);

rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Pico.rsc

@@ -0,0 +1,51 @@
+@license{
+Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+}
+module lang::rascal::tests::semanticTokenizer::Pico
+
+import lang::rascal::tests::semanticTokenizer::Util;
+
+// -------
+// Grammar
+
+import lang::pico::\syntax::Main;
+
+// -----
+// Tests
+
+test bool testKeywordLastLine() = testTokenizer(#Program,
+
+   "begin
+        declare
+            x: natural,
+            y: natural;
+
+        x := 5;
+        y := x + 1
+    end",
+
+    expectFirst("end", "keyword") // Fixed: https://github.com/usethesource/rascal-language-servers/issues/90
+);

rascal-lsp/src/main/rascal/lang/rascal/tests/semanticTokenizer/Rascal.rsc

@@ -0,0 +1,115 @@
+@license{
+Copyright (c) 2018-2023, NWO-I CWI and Swat.engineering
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+}
+module lang::rascal::tests::semanticTokenizer::Rascal
+
+import lang::rascal::tests::semanticTokenizer::Util;
+
+// -------
+// Grammar
+
+import lang::rascal::\syntax::Rascal;
+
+// -----
+// Tests
+
+test bool testTypesAndValues() = testTokenizer(#Declaration,
+
+   "void f() {
+        bool b = true;
+        int  i = 3;
+        real r = 3.14;
+        str  s = \"foo\<bar\>\";
+        loc  l = |unknown:///|;
+        tuple[int, int] \\tuple = \<3, 14\>;
+    }",
+
+    expectFirst("void", "keyword"),
+    expectFirst("bool", "keyword"),
+    expectFirst("int", "keyword"),
+    expectFirst("real", "keyword"),
+    expectFirst("loc", "keyword"),
+    expectFirst("str", "keyword"),
+    expectFirst("tuple", "keyword"),
+
+    expectFirst("f", "uncategorized"),
+    expectFirst("true", "keyword"),
+    //expectFirst("3", "number"), // https://github.com/usethesource/rascal-language-servers/issues/456
+    expectFirst("3.14", "number"),
+    expectFirst("foo", "string"),
+    expectFirst("\<", "string"),
+    expectFirst("bar", "uncategorized"),
+    expectFirst("\>", "string"),
+    //expectFirst("|unknown:///|", "string") // https://github.com/usethesource/rascal-language-servers/issues/456
+    expectLast("\<", "uncategorized"),
+    expectLast("\>", "uncategorized"),
+
+    applyRascalCategoryPatch = true
+);
+
+test bool testComments() = testTokenizer(#Declaration,
+
+   "void f() {
+        /* Block comment */
+        /* Multi-line 1
+           Multi-line 2 */
+        // Line comment
+    }",
+
+    expectFirst("Block comment", "comment"),
+    expectFirst("Multi-line 1", "comment"),
+    expectFirst("Multi-line 2", "comment"), // Fixed: https://github.com/usethesource/rascal-language-servers/issues/20
+    expectFirst("Line comment", "comment"),
+
+    applyRascalCategoryPatch = true
+);
+
+test bool testTags() = testTokenizer(#Declaration,
+
+   "@synopsis{Foo}
+    @category=\"bar\"
+    @memo
+    int i = 0;",
+
+    expectFirst("@synopsis{Foo}", "comment"),
+    expectFirst("@category=", "comment"),
+    expectFirst("\"bar\"", "string"),
+    expectFirst("@memo", "comment"),
+
+    applyRascalCategoryPatch = true
+);
+
+test bool testUnicode() = testTokenizer(#Declaration,
+   "void f() {
+        str s = \"𝄞𝄞𝄞\";
+    }",
+
+    expectFirst("str", "keyword"),
+    expectFirst(" s = ", "uncategorized"),
+    expectFirst("\"𝄞𝄞𝄞\"", "string"), // Fixed: https://github.com/usethesource/rascal-language-servers/issues/19
+    expectFirst(";", "uncategorized"),
+    applyRascalCategoryPatch = true
+);