feat: Propagate included file paths & ranges (#50)

* feat: Propagate included file paths & ranges

* chore: Fix formatting

src/Visp.Compiler/ParseUtils.fs

@@ -11,9 +11,16 @@ open Visp.Compiler.LexHelpers
 open Visp.Compiler.Syntax.Macros
 open System.Collections.Generic
 
+type TokenWithPos =
+    { token: token
+      startPos: Position
+      endPos: Position }
+
 type NestedTokenizer =
     { mutable index: int
-      tokens: IReadOnlyList<token> }
+      tokens: IReadOnlyList<TokenWithPos>
+      origStartPos: Position
+      origEndPos: Position }
 
     member t.ReadNext() =
         if t.index < t.tokens.Count then
@@ -124,15 +131,20 @@ let rec mkTokenizerWithArgs args =
     let mutable token_list = []
     let mutable include_tokenizers: NestedTokenizer list = []
 
-    let read_next_nested_token () =
+    let read_next_nested_token (buf: LexBuffer<_>) =
         match include_tokenizers with
         | [] -> None
         | it :: rest ->
             match it.ReadNext() with
             | None ->
+                buf.StartPos <- it.origStartPos
+                buf.EndPos <- it.origEndPos
                 include_tokenizers <- rest
                 None
-            | Some(tok) -> Some(tok)
+            | Some(tok) ->
+                buf.StartPos <- tok.startPos
+                buf.EndPos <- tok.endPos
+                Some(tok.token)
 
     let read_next_token_list_token buf =
         match token_list with
@@ -150,10 +162,9 @@ let rec mkTokenizerWithArgs args =
             tok
 
     // Nested TOKENLISTS are not supported.
-    // TODO: Support correct file & ranges for includes
     let tokenizer buf =
         let token =
-            match read_next_nested_token () with
+            match read_next_nested_token buf with
             | Some(tok) -> tok
             | None -> read_next_token_list_token buf
 
@@ -174,15 +185,14 @@ let rec mkTokenizerWithArgs args =
 
                 loop <- next <> RPAREN
                 ()
-            // includes <- List.rev includes
 
             loop <- true
 
             while loop do
                 match includes with
                 | [] -> loop <- false
                 | it :: rest ->
-                    let tokz = getNestedTokens buf.EndPos.FileName it false
+                    let tokz = getNestedTokens buf it false
                     include_tokenizers <- tokz :: include_tokenizers
                     includes <- rest
 
@@ -196,7 +206,8 @@ let rec mkTokenizerWithArgs args =
 
     tokenizer
 
-and private getNestedTokens rootFile filePath dbg =
+and private getNestedTokens origBuf filePath dbg =
+    let rootFile = origBuf.EndPos.FileName
 
     let actualPath =
         System.IO.Path.GetDirectoryName rootFile
@@ -209,15 +220,24 @@ and private getNestedTokens rootFile filePath dbg =
     use _ = reader
 
     let tokenizer = mkTokenizer dbg
-    let arr = ResizeArray<token>()
+    let arr = ResizeArray<TokenWithPos>()
 
     while not lexbuf.IsPastEndOfStream do
         let next = tokenizer lexbuf
+        let sp = lexbuf.StartPos
+        let ep = lexbuf.EndPos
 
         if next <> EOF then
-            arr.Add(next)
-
-    { tokens = arr; index = 0 }
+            arr.Add(
+                { token = next
+                  startPos = sp
+                  endPos = ep }
+            )
+
+    { tokens = arr
+      index = 0
+      origStartPos = origBuf.StartPos
+      origEndPos = origBuf.EndPos }
 
 and mkTokenizer dbg =
     mkTokenizerWithArgs

tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-0.structured output.verified.txt

@@ -6,21 +6,20 @@
              (8,1--8,31)), (8,0--8,32)); Expr (Const (Unit, (10,0--10,32)), (10,0--10,32));
        Expr
          (FunctionCall
-            (Symbol (SynSymbol printfn),
-             [Const (String ("skip-same-dir loading", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32)),
-          (10,31--10,32));
+            (Symbol (SynSymbol printfn), [Const (String ("skip-same-dir loading", Regular, (8,9--8,32)), (8,9--8,32))],
+             (8,1--8,32)), (8,0--8,33));
        Expr
          (FunctionDef
-            (SynSymbol IncludedFunction, None, Args (List [], (10,31--10,32)),
+            (SynSymbol IncludedFunction, None, Args (List [], (10,21--10,23)),
              [FunctionCall
                 (Symbol (SynSymbol printfn),
-                 [Const (String ("Calling IncludedFunction", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32))],
-             None, (10,31--10,32)), (10,31--10,32));
+                 [Const (String ("Calling IncludedFunction", Regular, (11,11--11,37)), (11,11--11,37))], (11,3--11,37))],
+             None, (10,1--11,38)), (10,0--11,39));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn),
-             [Const (String ("skip-same-dir loaded...", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32)),
-          (10,31--10,32));
+             [Const (String ("skip-same-dir loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)),
+          (13,0--13,35));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn), [Const (String ("include-0 loaded", Regular, (12,9--12,27)), (12,9--12,27))],

tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-1.structured output.verified.txt

@@ -6,38 +6,36 @@
              (8,1--8,31)), (8,0--8,32)); Expr (Const (Unit, (10,0--10,39)), (10,0--10,39));
        Expr
          (FunctionCall
-            (Symbol (SynSymbol printfn),
-             [Const (String ("skip-nested-0 loading", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
-          (10,38--10,39));
+            (Symbol (SynSymbol printfn), [Const (String ("skip-nested-0 loading", Regular, (8,9--8,32)), (8,9--8,32))],
+             (8,1--8,32)), (8,0--8,33));
        Expr
          (FunctionDef
-            (SynSymbol Nested0IncludedFunction, None, Args (List [], (10,38--10,39)),
+            (SynSymbol Nested0IncludedFunction, None, Args (List [], (10,28--10,30)),
              [FunctionCall
                 (Symbol (SynSymbol printfn),
-                 [Const (String ("Calling Nested0IncludedFunction", Regular, (10,38--10,39)), (10,38--10,39))],
-                 (10,38--10,39))], None, (10,38--10,39)), (10,38--10,39));
+                 [Const (String ("Calling Nested0IncludedFunction", Regular, (11,11--11,44)), (11,11--11,44))],
+                 (11,3--11,44))], None, (10,1--11,45)), (10,0--11,46));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn),
-             [Const (String ("skip-nested-0 loaded...", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
-          (10,38--10,39)); Expr (Const (Unit, (10,38--10,39)), (10,38--10,39));
+             [Const (String ("skip-nested-0 loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)),
+          (13,0--13,35)); Expr (Const (Unit, (15,0--15,32)), (15,0--15,32));
        Expr
          (FunctionCall
-            (Symbol (SynSymbol printfn),
-             [Const (String ("skip-nested-1 loading", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
-          (10,38--10,39));
+            (Symbol (SynSymbol printfn), [Const (String ("skip-nested-1 loading", Regular, (8,9--8,32)), (8,9--8,32))],
+             (8,1--8,32)), (8,0--8,33));
        Expr
          (FunctionDef
-            (SynSymbol Nested1IncludedFunction, None, Args (List [], (10,38--10,39)),
+            (SynSymbol Nested1IncludedFunction, None, Args (List [], (10,28--10,30)),
              [FunctionCall
                 (Symbol (SynSymbol printfn),
-                 [Const (String ("Calling Nested1IncludedFunction", Regular, (10,38--10,39)), (10,38--10,39))],
-                 (10,38--10,39))], None, (10,38--10,39)), (10,38--10,39));
+                 [Const (String ("Calling Nested1IncludedFunction", Regular, (11,11--11,44)), (11,11--11,44))],
+                 (11,3--11,44))], None, (10,1--11,45)), (10,0--11,46));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn),
-             [Const (String ("skip-nested-1 loaded...", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
-          (10,38--10,39));
+             [Const (String ("skip-nested-1 loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)),
+          (13,0--13,35));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn), [Const (String ("include-1 loaded", Regular, (12,9--12,27)), (12,9--12,27))],

tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-macros-0.structured output.verified.txt

@@ -8,31 +8,31 @@
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn),
-             [Const (String ("include-macros-0-macros.visp loading...", Regular, (10,41--10,42)), (10,41--10,42))],
-             (10,41--10,42)), (10,41--10,42));
+             [Const (String ("include-macros-0-macros.visp loading...", Regular, (8,9--8,50)), (8,9--8,50))],
+             (8,1--8,50)), (8,0--8,51));
        Expr
          (SyntaxMacro
             (SynMacro
                (SynSymbol MyStruct,
                 [SynMacroCase
-                   ([Discard (10,41--10,42); Symbol (SynSymbol typ, (10,41--10,42));
+                   ([Discard (11,4--11,5); Symbol (SynSymbol typ, (11,6--11,9));
                      List
-                       ([Symbol (SynSymbol arg, (10,41--10,42)); Symbol (SynSymbol ctor, (10,41--10,42));
-                         Ellipsis (10,41--10,42)], (10,41--10,42)); Symbol (SynSymbol body, (10,41--10,42));
-                     Ellipsis (10,41--10,42)],
+                       ([Symbol (SynSymbol arg, (11,11--11,14)); Symbol (SynSymbol ctor, (11,15--11,19));
+                         Ellipsis (11,20--11,23)], (11,10--11,24)); Symbol (SynSymbol body, (11,25--11,29));
+                     Ellipsis (11,30--11,33)],
                     List
                       (Paren,
-                       [List (HashBracket, [Symbol (SynSymbol Struct)], (10,41--10,42)); Symbol (SynSymbol type);
+                       [List (HashBracket, [Symbol (SynSymbol Struct)], (12,5--12,14)); Symbol (SynSymbol type);
                         Symbol (SynSymbol typ);
                         List
-                          (Paren, [Symbol (SynSymbol arg); Symbol (SynSymbol ctor); Ellipsis (10,41--10,42)],
-                           (10,41--10,42)); Symbol (SynSymbol body); Ellipsis (10,41--10,42)], (10,41--10,42)),
-                    (10,41--10,42))], (10,41--10,42))), (10,41--10,42));
+                          (Paren, [Symbol (SynSymbol arg); Symbol (SynSymbol ctor); Ellipsis (13,25--13,28)],
+                           (13,15--13,29)); Symbol (SynSymbol body); Ellipsis (14,11--14,14)], (12,4--15,5)),
+                    (11,2--16,3))], (10,1--16,3))), (10,0--16,4));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn),
-             [Const (String ("include-macros-0-macros.visp loaded", Regular, (10,41--10,42)), (10,41--10,42))],
-             (10,41--10,42)), (10,41--10,42));
+             [Const (String ("include-macros-0-macros.visp loaded", Regular, (18,9--18,46)), (18,9--18,46))],
+             (18,1--18,46)), (18,0--18,47));
        Expr
          (FunctionCall
             (Symbol (SynSymbol printfn),

tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-0.can parse.verified.txt

@@ -8,14 +8,14 @@ let state = { Todo = () }
 // line 8 @"include-0.visp"
 printfn "include-0 loading..."
 ()
-// line 10 @"include-0.visp"
+// line 8 @"skip-same-dir.visp"
 printfn "skip-same-dir loading"
-// line 10 @"include-0.visp"
+// line 10 @"skip-same-dir.visp"
 let IncludedFunction () =
-  // line 10 @"include-0.visp"
+  // line 11 @"skip-same-dir.visp"
   printfn "Calling IncludedFunction"
 
-// line 10 @"include-0.visp"
+// line 13 @"skip-same-dir.visp"
 printfn "skip-same-dir loaded..."
 // line 12 @"include-0.visp"
 printfn "include-0 loaded"

tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-1.can parse.verified.txt

@@ -8,24 +8,24 @@ let state = { Todo = () }
 // line 8 @"include-1.visp"
 printfn "include-1 loading..."
 ()
-// line 10 @"include-1.visp"
+// line 8 @"nested/skip-nested-0.visp"
 printfn "skip-nested-0 loading"
-// line 10 @"include-1.visp"
+// line 10 @"nested/skip-nested-0.visp"
 let Nested0IncludedFunction () =
-  // line 10 @"include-1.visp"
+  // line 11 @"nested/skip-nested-0.visp"
   printfn "Calling Nested0IncludedFunction"
 
-// line 10 @"include-1.visp"
+// line 13 @"nested/skip-nested-0.visp"
 printfn "skip-nested-0 loaded..."
 ()
-// line 10 @"include-1.visp"
+// line 8 @"nested/skip-nested-1.visp"
 printfn "skip-nested-1 loading"
-// line 10 @"include-1.visp"
+// line 10 @"nested/skip-nested-1.visp"
 let Nested1IncludedFunction () =
-  // line 10 @"include-1.visp"
+  // line 11 @"nested/skip-nested-1.visp"
   printfn "Calling Nested1IncludedFunction"
 
-// line 10 @"include-1.visp"
+// line 13 @"nested/skip-nested-1.visp"
 printfn "skip-nested-1 loaded..."
 // line 12 @"include-1.visp"
 printfn "include-1 loaded"

tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-macros-0.can parse.verified.txt

@@ -8,11 +8,11 @@ let state = { Todo = () }
 // line 8 @"include-macros-0.visp"
 printfn "include-macros-0.visp loading..."
 ()
-// line 10 @"include-macros-0.visp"
+// line 8 @"include-macros-0-macros.visp"
 printfn "include-macros-0-macros.visp loading..."
-// line 10 @"include-macros-0.visp"
+// line 10 @"include-macros-0-macros.visp"
 let macro_MyStruct = "__MACRO_INIT__"
-// line 10 @"include-macros-0.visp"
+// line 18 @"include-macros-0-macros.visp"
 printfn "include-macros-0-macros.visp loaded"
 // line 12 @"include-macros-0.visp"
 printfn "include-macros-0.visp loaded"