kast --debug-tokens

k-distribution/tests/regression-new/issue-3647-debugTokens/a.test.kast

@@ -1 +1,14 @@
-a + 100 + 2
+1 + 2 + aaaaaaaaaaaa
+
+
+
+
+
+
+
+
+
+
++ 10000000
++ "str"
++ "long str that breaks alighnment"

k-distribution/tests/regression-new/issue-3647-debugTokens/a.test.kast.out

@@ -1,9 +1,15 @@
-`Match`    (location),    Terminal
+(location)    "Match"      Terminal
 ------------------------------------
-`a`        (1,1,1,2),   r"[a-z][a-zA-Z0-9]*"
-`+`        (1,3,1,4),   "+"
-`100`      (1,5,1,8),   r"[\\+-]?[0-9]+"
-`+`        (1,9,1,10),  "+"
-`2`        (1,11,1,12), r"[\\+-]?[0-9]+"
-``         (2,1,2,1),   "<eof>"
+(1,1,1,2),    "1"          r"[\\+-]?[0-9]+"
+(1,3,1,4),    "+"          "+"
+(1,5,1,6),    "2"          r"[\\+-]?[0-9]+"
+(1,7,1,8),    "+"          "+"
+(1,9,1,21),   "aaaaaaaaaaaa" r"[a-z][a-zA-Z0-9]*"
+(12,1,12,2),  "+"          "+"
+(12,3,12,11), "10000000"   r"[\\+-]?[0-9]+"
+(13,1,13,2),  "+"          "+"
+(13,3,13,8),  "\"str\""    r"[\\\"](([^\\\"\\n\\r\\\\])|([\\\\][nrtf\\\"\\\\])|([\\\\][x][0-9a-fA-F]{2})|([\\\\][u][0-9a-fA-F]{4})|([\\\\][U][0-9a-fA-F]{8}))*[\\\"]"
+(14,1,14,2),  "+"          "+"
+(14,3,14,36), "\"long str that breaks alighnment\"" r"[\\\"](([^\\\"\\n\\r\\\\])|([\\\\][nrtf\\\"\\\\])|([\\\\][x][0-9a-fA-F]{2})|([\\\\][u][0-9a-fA-F]{4})|([\\\\][U][0-9a-fA-F]{8}))*[\\\"]"
+(15,1,15,1),  ""           "<eof>"
 

k-distribution/tests/regression-new/issue-3647-debugTokens/test.k

@@ -3,7 +3,8 @@
 module TEST-SYNTAX
     imports INT-SYNTAX
     imports ID-SYNTAX
-    syntax Exp ::= Exp "+" Exp [left] | Int | Id
+    imports STRING-SYNTAX
+    syntax Exp ::= Exp "+" Exp [left] | Int | Id | String
 endmodule
 
 module TEST

kernel/src/main/java/org/kframework/parser/inner/ParseInModule.java

@@ -18,6 +18,7 @@
 import org.kframework.parser.inner.kernel.EarleyParser;
 import org.kframework.parser.inner.kernel.Scanner;
 import org.kframework.parser.outer.Outer;
+import org.kframework.utils.StringUtil;
 import org.kframework.utils.errorsystem.KEMException;
 import org.kframework.utils.file.FileUtil;
 import scala.Tuple2;
@@ -165,20 +166,28 @@ public void initialize() {
      */
     public String tokenizeString(String input, Source source) {
         StringBuilder sb = new StringBuilder();
-        sb.append("`Match`    (location),    Terminal\n");
-        sb.append("------------------------------------\n");
         try (Scanner scanner = getScanner()) {
             EarleyParser.ParserMetadata mdata = new EarleyParser.ParserMetadata(input, scanner, source, 1, 1);
             Map<Integer, TerminalLike> kind2Token =
                     scanner.getTokens().entrySet().stream().map(a -> new Tuple2<>(a.getValue()._1, a.getKey()))
                     .collect(Collectors.toMap(Tuple2::_1, Tuple2::_2));
             List<Integer> lines = mdata.getLines();
             List<Integer> columns = mdata.getColumns();
+            int maxLocLen = 0;
+            List<String> locs = new ArrayList<>();
+            List<Scanner.Token> words = mdata.getWords();
             for (Scanner.Token word : mdata.getWords()) {
                 String loc = String.format("(%d,%d,%d,%d),",
                         lines.get(word.startLoc), columns.get(word.startLoc),
                         lines.get(word.endLoc), columns.get(word.endLoc));
-                sb.append(String.format("%-10s %-12s %s\n", '`' + word.value + '`', loc,
+                locs.add(loc);
+                maxLocLen = Math.max(maxLocLen, loc.length());
+            }
+            sb.append(String.format("%-" + maxLocLen + "s \"Match\"      Terminal\n", "(location)"));
+            sb.append("------------------------------------\n");
+            for (int i = 0; i < words.size(); i++) {
+                Scanner.Token word = words.get(i);
+                sb.append(String.format("%-" + maxLocLen + "s %-12s %s\n", locs.get(i), StringUtil.enquoteKString(word.value),
                         kind2Token.getOrDefault(word.kind, Terminal.apply("<eof>"))));
             }
         }

kernel/src/main/java/org/kframework/parser/inner/kernel/EarleyParser.java

@@ -1,6 +1,7 @@
 // Copyright (c) K Team. All Rights Reserved.
 package org.kframework.parser.inner.kernel;
 
+import com.google.common.primitives.Ints;
 import org.apache.commons.codec.binary.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.kframework.attributes.Att;
@@ -587,11 +588,11 @@ public List<Scanner.Token> getWords() {
     }
 
     public List<Integer> getLines() {
-      return Arrays.stream(lines).boxed().collect(Collectors.toList());
+      return Ints.asList(lines);
     }
 
     public List<Integer> getColumns() {
-      return Arrays.stream(columns).boxed().collect(Collectors.toList());
+      return Ints.asList(columns);
     }
   }