diff --git a/src/Visp.Compiler/ParseUtils.fs b/src/Visp.Compiler/ParseUtils.fs index a0a7df6..a57a98e 100644 --- a/src/Visp.Compiler/ParseUtils.fs +++ b/src/Visp.Compiler/ParseUtils.fs @@ -11,9 +11,16 @@ open Visp.Compiler.LexHelpers open Visp.Compiler.Syntax.Macros open System.Collections.Generic +type TokenWithPos = + { token: token + startPos: Position + endPos: Position } + type NestedTokenizer = { mutable index: int - tokens: IReadOnlyList } + tokens: IReadOnlyList + origStartPos: Position + origEndPos: Position } member t.ReadNext() = if t.index < t.tokens.Count then @@ -124,15 +131,20 @@ let rec mkTokenizerWithArgs args = let mutable token_list = [] let mutable include_tokenizers: NestedTokenizer list = [] - let read_next_nested_token () = + let read_next_nested_token (buf: LexBuffer<_>) = match include_tokenizers with | [] -> None | it :: rest -> match it.ReadNext() with | None -> + buf.StartPos <- it.origStartPos + buf.EndPos <- it.origEndPos include_tokenizers <- rest None - | Some(tok) -> Some(tok) + | Some(tok) -> + buf.StartPos <- tok.startPos + buf.EndPos <- tok.endPos + Some(tok.token) let read_next_token_list_token buf = match token_list with @@ -150,10 +162,9 @@ let rec mkTokenizerWithArgs args = tok // Nested TOKENLISTS are not supported. - // TODO: Support correct file & ranges for includes let tokenizer buf = let token = - match read_next_nested_token () with + match read_next_nested_token buf with | Some(tok) -> tok | None -> read_next_token_list_token buf @@ -174,7 +185,6 @@ let rec mkTokenizerWithArgs args = loop <- next <> RPAREN () - // includes <- List.rev includes loop <- true @@ -182,7 +192,7 @@ let rec mkTokenizerWithArgs args = match includes with | [] -> loop <- false | it :: rest -> - let tokz = getNestedTokens buf.EndPos.FileName it false + let tokz = getNestedTokens buf it false include_tokenizers <- tokz :: include_tokenizers includes <- rest @@ -196,7 +206,8 @@ let rec mkTokenizerWithArgs args = tokenizer -and private getNestedTokens rootFile filePath dbg = +and private getNestedTokens origBuf filePath dbg = + let rootFile = origBuf.EndPos.FileName let actualPath = System.IO.Path.GetDirectoryName rootFile @@ -209,15 +220,24 @@ and private getNestedTokens rootFile filePath dbg = use _ = reader let tokenizer = mkTokenizer dbg - let arr = ResizeArray() + let arr = ResizeArray() while not lexbuf.IsPastEndOfStream do let next = tokenizer lexbuf + let sp = lexbuf.StartPos + let ep = lexbuf.EndPos if next <> EOF then - arr.Add(next) - - { tokens = arr; index = 0 } + arr.Add( + { token = next + startPos = sp + endPos = ep } + ) + + { tokens = arr + index = 0 + origStartPos = origBuf.StartPos + origEndPos = origBuf.EndPos } and mkTokenizer dbg = mkTokenizerWithArgs diff --git a/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-0.structured output.verified.txt b/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-0.structured output.verified.txt index c293b26..99dd334 100644 --- a/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-0.structured output.verified.txt +++ b/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-0.structured output.verified.txt @@ -6,21 +6,20 @@ (8,1--8,31)), (8,0--8,32)); Expr (Const (Unit, (10,0--10,32)), (10,0--10,32)); Expr (FunctionCall - (Symbol (SynSymbol printfn), - [Const (String ("skip-same-dir loading", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32)), - (10,31--10,32)); + (Symbol (SynSymbol printfn), [Const (String ("skip-same-dir loading", Regular, (8,9--8,32)), (8,9--8,32))], + (8,1--8,32)), (8,0--8,33)); Expr (FunctionDef - (SynSymbol IncludedFunction, None, Args (List [], (10,31--10,32)), + (SynSymbol IncludedFunction, None, Args (List [], (10,21--10,23)), [FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("Calling IncludedFunction", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32))], - None, (10,31--10,32)), (10,31--10,32)); + [Const (String ("Calling IncludedFunction", Regular, (11,11--11,37)), (11,11--11,37))], (11,3--11,37))], + None, (10,1--11,38)), (10,0--11,39)); Expr (FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("skip-same-dir loaded...", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32)), - (10,31--10,32)); + [Const (String ("skip-same-dir loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)), + (13,0--13,35)); Expr (FunctionCall (Symbol (SynSymbol printfn), [Const (String ("include-0 loaded", Regular, (12,9--12,27)), (12,9--12,27))], diff --git a/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-1.structured output.verified.txt b/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-1.structured output.verified.txt index 2975771..9571b4b 100644 --- a/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-1.structured output.verified.txt +++ b/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-1.structured output.verified.txt @@ -6,38 +6,36 @@ (8,1--8,31)), (8,0--8,32)); Expr (Const (Unit, (10,0--10,39)), (10,0--10,39)); Expr (FunctionCall - (Symbol (SynSymbol printfn), - [Const (String ("skip-nested-0 loading", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)), - (10,38--10,39)); + (Symbol (SynSymbol printfn), [Const (String ("skip-nested-0 loading", Regular, (8,9--8,32)), (8,9--8,32))], + (8,1--8,32)), (8,0--8,33)); Expr (FunctionDef - (SynSymbol Nested0IncludedFunction, None, Args (List [], (10,38--10,39)), + (SynSymbol Nested0IncludedFunction, None, Args (List [], (10,28--10,30)), [FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("Calling Nested0IncludedFunction", Regular, (10,38--10,39)), (10,38--10,39))], - (10,38--10,39))], None, (10,38--10,39)), (10,38--10,39)); + [Const (String ("Calling Nested0IncludedFunction", Regular, (11,11--11,44)), (11,11--11,44))], + (11,3--11,44))], None, (10,1--11,45)), (10,0--11,46)); Expr (FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("skip-nested-0 loaded...", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)), - (10,38--10,39)); Expr (Const (Unit, (10,38--10,39)), (10,38--10,39)); + [Const (String ("skip-nested-0 loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)), + (13,0--13,35)); Expr (Const (Unit, (15,0--15,32)), (15,0--15,32)); Expr (FunctionCall - (Symbol (SynSymbol printfn), - [Const (String ("skip-nested-1 loading", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)), - (10,38--10,39)); + (Symbol (SynSymbol printfn), [Const (String ("skip-nested-1 loading", Regular, (8,9--8,32)), (8,9--8,32))], + (8,1--8,32)), (8,0--8,33)); Expr (FunctionDef - (SynSymbol Nested1IncludedFunction, None, Args (List [], (10,38--10,39)), + (SynSymbol Nested1IncludedFunction, None, Args (List [], (10,28--10,30)), [FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("Calling Nested1IncludedFunction", Regular, (10,38--10,39)), (10,38--10,39))], - (10,38--10,39))], None, (10,38--10,39)), (10,38--10,39)); + [Const (String ("Calling Nested1IncludedFunction", Regular, (11,11--11,44)), (11,11--11,44))], + (11,3--11,44))], None, (10,1--11,45)), (10,0--11,46)); Expr (FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("skip-nested-1 loaded...", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)), - (10,38--10,39)); + [Const (String ("skip-nested-1 loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)), + (13,0--13,35)); Expr (FunctionCall (Symbol (SynSymbol printfn), [Const (String ("include-1 loaded", Regular, (12,9--12,27)), (12,9--12,27))], diff --git a/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-macros-0.structured output.verified.txt b/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-macros-0.structured output.verified.txt index 8b4f765..72cb4c2 100644 --- a/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-macros-0.structured output.verified.txt +++ b/tests/Visp.Compiler.UnitTests/parsing-snapshots/tests_include_include-macros-0.structured output.verified.txt @@ -8,31 +8,31 @@ Expr (FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("include-macros-0-macros.visp loading...", Regular, (10,41--10,42)), (10,41--10,42))], - (10,41--10,42)), (10,41--10,42)); + [Const (String ("include-macros-0-macros.visp loading...", Regular, (8,9--8,50)), (8,9--8,50))], + (8,1--8,50)), (8,0--8,51)); Expr (SyntaxMacro (SynMacro (SynSymbol MyStruct, [SynMacroCase - ([Discard (10,41--10,42); Symbol (SynSymbol typ, (10,41--10,42)); + ([Discard (11,4--11,5); Symbol (SynSymbol typ, (11,6--11,9)); List - ([Symbol (SynSymbol arg, (10,41--10,42)); Symbol (SynSymbol ctor, (10,41--10,42)); - Ellipsis (10,41--10,42)], (10,41--10,42)); Symbol (SynSymbol body, (10,41--10,42)); - Ellipsis (10,41--10,42)], + ([Symbol (SynSymbol arg, (11,11--11,14)); Symbol (SynSymbol ctor, (11,15--11,19)); + Ellipsis (11,20--11,23)], (11,10--11,24)); Symbol (SynSymbol body, (11,25--11,29)); + Ellipsis (11,30--11,33)], List (Paren, - [List (HashBracket, [Symbol (SynSymbol Struct)], (10,41--10,42)); Symbol (SynSymbol type); + [List (HashBracket, [Symbol (SynSymbol Struct)], (12,5--12,14)); Symbol (SynSymbol type); Symbol (SynSymbol typ); List - (Paren, [Symbol (SynSymbol arg); Symbol (SynSymbol ctor); Ellipsis (10,41--10,42)], - (10,41--10,42)); Symbol (SynSymbol body); Ellipsis (10,41--10,42)], (10,41--10,42)), - (10,41--10,42))], (10,41--10,42))), (10,41--10,42)); + (Paren, [Symbol (SynSymbol arg); Symbol (SynSymbol ctor); Ellipsis (13,25--13,28)], + (13,15--13,29)); Symbol (SynSymbol body); Ellipsis (14,11--14,14)], (12,4--15,5)), + (11,2--16,3))], (10,1--16,3))), (10,0--16,4)); Expr (FunctionCall (Symbol (SynSymbol printfn), - [Const (String ("include-macros-0-macros.visp loaded", Regular, (10,41--10,42)), (10,41--10,42))], - (10,41--10,42)), (10,41--10,42)); + [Const (String ("include-macros-0-macros.visp loaded", Regular, (18,9--18,46)), (18,9--18,46))], + (18,1--18,46)), (18,0--18,47)); Expr (FunctionCall (Symbol (SynSymbol printfn), diff --git a/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-0.can parse.verified.txt b/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-0.can parse.verified.txt index 0788b00..b444b54 100644 --- a/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-0.can parse.verified.txt +++ b/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-0.can parse.verified.txt @@ -8,14 +8,14 @@ let state = { Todo = () } // line 8 @"include-0.visp" printfn "include-0 loading..." () -// line 10 @"include-0.visp" +// line 8 @"skip-same-dir.visp" printfn "skip-same-dir loading" -// line 10 @"include-0.visp" +// line 10 @"skip-same-dir.visp" let IncludedFunction () = - // line 10 @"include-0.visp" + // line 11 @"skip-same-dir.visp" printfn "Calling IncludedFunction" -// line 10 @"include-0.visp" +// line 13 @"skip-same-dir.visp" printfn "skip-same-dir loaded..." // line 12 @"include-0.visp" printfn "include-0 loaded" diff --git a/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-1.can parse.verified.txt b/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-1.can parse.verified.txt index 29ce302..41b24ab 100644 --- a/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-1.can parse.verified.txt +++ b/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-1.can parse.verified.txt @@ -8,24 +8,24 @@ let state = { Todo = () } // line 8 @"include-1.visp" printfn "include-1 loading..." () -// line 10 @"include-1.visp" +// line 8 @"nested/skip-nested-0.visp" printfn "skip-nested-0 loading" -// line 10 @"include-1.visp" +// line 10 @"nested/skip-nested-0.visp" let Nested0IncludedFunction () = - // line 10 @"include-1.visp" + // line 11 @"nested/skip-nested-0.visp" printfn "Calling Nested0IncludedFunction" -// line 10 @"include-1.visp" +// line 13 @"nested/skip-nested-0.visp" printfn "skip-nested-0 loaded..." () -// line 10 @"include-1.visp" +// line 8 @"nested/skip-nested-1.visp" printfn "skip-nested-1 loading" -// line 10 @"include-1.visp" +// line 10 @"nested/skip-nested-1.visp" let Nested1IncludedFunction () = - // line 10 @"include-1.visp" + // line 11 @"nested/skip-nested-1.visp" printfn "Calling Nested1IncludedFunction" -// line 10 @"include-1.visp" +// line 13 @"nested/skip-nested-1.visp" printfn "skip-nested-1 loaded..." // line 12 @"include-1.visp" printfn "include-1 loaded" diff --git a/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-macros-0.can parse.verified.txt b/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-macros-0.can parse.verified.txt index f02d7db..ba32674 100644 --- a/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-macros-0.can parse.verified.txt +++ b/tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-macros-0.can parse.verified.txt @@ -8,11 +8,11 @@ let state = { Todo = () } // line 8 @"include-macros-0.visp" printfn "include-macros-0.visp loading..." () -// line 10 @"include-macros-0.visp" +// line 8 @"include-macros-0-macros.visp" printfn "include-macros-0-macros.visp loading..." -// line 10 @"include-macros-0.visp" +// line 10 @"include-macros-0-macros.visp" let macro_MyStruct = "__MACRO_INIT__" -// line 10 @"include-macros-0.visp" +// line 18 @"include-macros-0-macros.visp" printfn "include-macros-0-macros.visp loaded" // line 12 @"include-macros-0.visp" printfn "include-macros-0.visp loaded"