Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Propagate included file paths & ranges #50

Merged
merged 2 commits into from
Dec 3, 2024
Merged

feat: Propagate included file paths & ranges #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
461e5a1
feat: Propagate included file paths & ranges
vipentti Dec 3, 2024
3a194ed
chore: Fix formatting
vipentti Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 32 additions & 12 deletions src/Visp.Compiler/ParseUtils.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,16 @@ open Visp.Compiler.LexHelpers
open Visp.Compiler.Syntax.Macros
open System.Collections.Generic

type TokenWithPos =
{ token: token
startPos: Position
endPos: Position }

type NestedTokenizer =
{ mutable index: int
tokens: IReadOnlyList<token> }
tokens: IReadOnlyList<TokenWithPos>
origStartPos: Position
origEndPos: Position }

member t.ReadNext() =
if t.index < t.tokens.Count then
Expand Down Expand Up @@ -124,15 +131,20 @@ let rec mkTokenizerWithArgs args =
let mutable token_list = []
let mutable include_tokenizers: NestedTokenizer list = []

let read_next_nested_token () =
let read_next_nested_token (buf: LexBuffer<_>) =
match include_tokenizers with
| [] -> None
| it :: rest ->
match it.ReadNext() with
| None ->
buf.StartPos <- it.origStartPos
buf.EndPos <- it.origEndPos
include_tokenizers <- rest
None
| Some(tok) -> Some(tok)
| Some(tok) ->
buf.StartPos <- tok.startPos
buf.EndPos <- tok.endPos
Some(tok.token)

let read_next_token_list_token buf =
match token_list with
Expand All @@ -150,10 +162,9 @@ let rec mkTokenizerWithArgs args =
tok

// Nested TOKENLISTS are not supported.
// TODO: Support correct file & ranges for includes
let tokenizer buf =
let token =
match read_next_nested_token () with
match read_next_nested_token buf with
| Some(tok) -> tok
| None -> read_next_token_list_token buf

Expand All @@ -174,15 +185,14 @@ let rec mkTokenizerWithArgs args =

loop <- next <> RPAREN
()
// includes <- List.rev includes

loop <- true

while loop do
match includes with
| [] -> loop <- false
| it :: rest ->
let tokz = getNestedTokens buf.EndPos.FileName it false
let tokz = getNestedTokens buf it false
include_tokenizers <- tokz :: include_tokenizers
includes <- rest

Expand All @@ -196,7 +206,8 @@ let rec mkTokenizerWithArgs args =

tokenizer

and private getNestedTokens rootFile filePath dbg =
and private getNestedTokens origBuf filePath dbg =
let rootFile = origBuf.EndPos.FileName

let actualPath =
System.IO.Path.GetDirectoryName rootFile
Expand All @@ -209,15 +220,24 @@ and private getNestedTokens rootFile filePath dbg =
use _ = reader

let tokenizer = mkTokenizer dbg
let arr = ResizeArray<token>()
let arr = ResizeArray<TokenWithPos>()

while not lexbuf.IsPastEndOfStream do
let next = tokenizer lexbuf
let sp = lexbuf.StartPos
let ep = lexbuf.EndPos

if next <> EOF then
arr.Add(next)

{ tokens = arr; index = 0 }
arr.Add(
{ token = next
startPos = sp
endPos = ep }
)

{ tokens = arr
index = 0
origStartPos = origBuf.StartPos
origEndPos = origBuf.EndPos }

and mkTokenizer dbg =
mkTokenizerWithArgs
Expand Down
15 changes: 7 additions & 8 deletions ...mpiler.UnitTests/parsing-snapshots/tests_include_include-0.structured output.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,20 @@
(8,1--8,31)), (8,0--8,32)); Expr (Const (Unit, (10,0--10,32)), (10,0--10,32));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("skip-same-dir loading", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32)),
(10,31--10,32));
(Symbol (SynSymbol printfn), [Const (String ("skip-same-dir loading", Regular, (8,9--8,32)), (8,9--8,32))],
(8,1--8,32)), (8,0--8,33));
Expr
(FunctionDef
(SynSymbol IncludedFunction, None, Args (List [], (10,31--10,32)),
(SynSymbol IncludedFunction, None, Args (List [], (10,21--10,23)),
[FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("Calling IncludedFunction", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32))],
None, (10,31--10,32)), (10,31--10,32));
[Const (String ("Calling IncludedFunction", Regular, (11,11--11,37)), (11,11--11,37))], (11,3--11,37))],
None, (10,1--11,38)), (10,0--11,39));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("skip-same-dir loaded...", Regular, (10,31--10,32)), (10,31--10,32))], (10,31--10,32)),
(10,31--10,32));
[Const (String ("skip-same-dir loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)),
(13,0--13,35));
Expr
(FunctionCall
(Symbol (SynSymbol printfn), [Const (String ("include-0 loaded", Regular, (12,9--12,27)), (12,9--12,27))],
Expand Down
30 changes: 14 additions & 16 deletions ...mpiler.UnitTests/parsing-snapshots/tests_include_include-1.structured output.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,38 +6,36 @@
(8,1--8,31)), (8,0--8,32)); Expr (Const (Unit, (10,0--10,39)), (10,0--10,39));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("skip-nested-0 loading", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
(10,38--10,39));
(Symbol (SynSymbol printfn), [Const (String ("skip-nested-0 loading", Regular, (8,9--8,32)), (8,9--8,32))],
(8,1--8,32)), (8,0--8,33));
Expr
(FunctionDef
(SynSymbol Nested0IncludedFunction, None, Args (List [], (10,38--10,39)),
(SynSymbol Nested0IncludedFunction, None, Args (List [], (10,28--10,30)),
[FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("Calling Nested0IncludedFunction", Regular, (10,38--10,39)), (10,38--10,39))],
(10,38--10,39))], None, (10,38--10,39)), (10,38--10,39));
[Const (String ("Calling Nested0IncludedFunction", Regular, (11,11--11,44)), (11,11--11,44))],
(11,3--11,44))], None, (10,1--11,45)), (10,0--11,46));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("skip-nested-0 loaded...", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
(10,38--10,39)); Expr (Const (Unit, (10,38--10,39)), (10,38--10,39));
[Const (String ("skip-nested-0 loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)),
(13,0--13,35)); Expr (Const (Unit, (15,0--15,32)), (15,0--15,32));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("skip-nested-1 loading", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
(10,38--10,39));
(Symbol (SynSymbol printfn), [Const (String ("skip-nested-1 loading", Regular, (8,9--8,32)), (8,9--8,32))],
(8,1--8,32)), (8,0--8,33));
Expr
(FunctionDef
(SynSymbol Nested1IncludedFunction, None, Args (List [], (10,38--10,39)),
(SynSymbol Nested1IncludedFunction, None, Args (List [], (10,28--10,30)),
[FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("Calling Nested1IncludedFunction", Regular, (10,38--10,39)), (10,38--10,39))],
(10,38--10,39))], None, (10,38--10,39)), (10,38--10,39));
[Const (String ("Calling Nested1IncludedFunction", Regular, (11,11--11,44)), (11,11--11,44))],
(11,3--11,44))], None, (10,1--11,45)), (10,0--11,46));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("skip-nested-1 loaded...", Regular, (10,38--10,39)), (10,38--10,39))], (10,38--10,39)),
(10,38--10,39));
[Const (String ("skip-nested-1 loaded...", Regular, (13,9--13,34)), (13,9--13,34))], (13,1--13,34)),
(13,0--13,35));
Expr
(FunctionCall
(Symbol (SynSymbol printfn), [Const (String ("include-1 loaded", Regular, (12,9--12,27)), (12,9--12,27))],
Expand Down
24 changes: 12 additions & 12 deletions ...UnitTests/parsing-snapshots/tests_include_include-macros-0.structured output.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,31 @@
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("include-macros-0-macros.visp loading...", Regular, (10,41--10,42)), (10,41--10,42))],
(10,41--10,42)), (10,41--10,42));
[Const (String ("include-macros-0-macros.visp loading...", Regular, (8,9--8,50)), (8,9--8,50))],
(8,1--8,50)), (8,0--8,51));
Expr
(SyntaxMacro
(SynMacro
(SynSymbol MyStruct,
[SynMacroCase
([Discard (10,41--10,42); Symbol (SynSymbol typ, (10,41--10,42));
([Discard (11,4--11,5); Symbol (SynSymbol typ, (11,6--11,9));
List
([Symbol (SynSymbol arg, (10,41--10,42)); Symbol (SynSymbol ctor, (10,41--10,42));
Ellipsis (10,41--10,42)], (10,41--10,42)); Symbol (SynSymbol body, (10,41--10,42));
Ellipsis (10,41--10,42)],
([Symbol (SynSymbol arg, (11,11--11,14)); Symbol (SynSymbol ctor, (11,15--11,19));
Ellipsis (11,20--11,23)], (11,10--11,24)); Symbol (SynSymbol body, (11,25--11,29));
Ellipsis (11,30--11,33)],
List
(Paren,
[List (HashBracket, [Symbol (SynSymbol Struct)], (10,41--10,42)); Symbol (SynSymbol type);
[List (HashBracket, [Symbol (SynSymbol Struct)], (12,5--12,14)); Symbol (SynSymbol type);
Symbol (SynSymbol typ);
List
(Paren, [Symbol (SynSymbol arg); Symbol (SynSymbol ctor); Ellipsis (10,41--10,42)],
(10,41--10,42)); Symbol (SynSymbol body); Ellipsis (10,41--10,42)], (10,41--10,42)),
(10,41--10,42))], (10,41--10,42))), (10,41--10,42));
(Paren, [Symbol (SynSymbol arg); Symbol (SynSymbol ctor); Ellipsis (13,25--13,28)],
(13,15--13,29)); Symbol (SynSymbol body); Ellipsis (14,11--14,14)], (12,4--15,5)),
(11,2--16,3))], (10,1--16,3))), (10,0--16,4));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
[Const (String ("include-macros-0-macros.visp loaded", Regular, (10,41--10,42)), (10,41--10,42))],
(10,41--10,42)), (10,41--10,42));
[Const (String ("include-macros-0-macros.visp loaded", Regular, (18,9--18,46)), (18,9--18,46))],
(18,1--18,46)), (18,0--18,47));
Expr
(FunctionCall
(Symbol (SynSymbol printfn),
Expand Down
8 changes: 4 additions & 4 deletions tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-0.can parse.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ let state = { Todo = () }
// line 8 @"include-0.visp"
printfn "include-0 loading..."
()
// line 10 @"include-0.visp"
// line 8 @"skip-same-dir.visp"
printfn "skip-same-dir loading"
// line 10 @"include-0.visp"
// line 10 @"skip-same-dir.visp"
let IncludedFunction () =
// line 10 @"include-0.visp"
// line 11 @"skip-same-dir.visp"
printfn "Calling IncludedFunction"

// line 10 @"include-0.visp"
// line 13 @"skip-same-dir.visp"
printfn "skip-same-dir loaded..."
// line 12 @"include-0.visp"
printfn "include-0 loaded"
Expand Down
16 changes: 8 additions & 8 deletions tests/Visp.Compiler.UnitTests/snapshots/tests_include_include-1.can parse.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,24 @@ let state = { Todo = () }
// line 8 @"include-1.visp"
printfn "include-1 loading..."
()
// line 10 @"include-1.visp"
// line 8 @"nested/skip-nested-0.visp"
printfn "skip-nested-0 loading"
// line 10 @"include-1.visp"
// line 10 @"nested/skip-nested-0.visp"
let Nested0IncludedFunction () =
// line 10 @"include-1.visp"
// line 11 @"nested/skip-nested-0.visp"
printfn "Calling Nested0IncludedFunction"

// line 10 @"include-1.visp"
// line 13 @"nested/skip-nested-0.visp"
printfn "skip-nested-0 loaded..."
()
// line 10 @"include-1.visp"
// line 8 @"nested/skip-nested-1.visp"
printfn "skip-nested-1 loading"
// line 10 @"include-1.visp"
// line 10 @"nested/skip-nested-1.visp"
let Nested1IncludedFunction () =
// line 10 @"include-1.visp"
// line 11 @"nested/skip-nested-1.visp"
printfn "Calling Nested1IncludedFunction"

// line 10 @"include-1.visp"
// line 13 @"nested/skip-nested-1.visp"
printfn "skip-nested-1 loaded..."
// line 12 @"include-1.visp"
printfn "include-1 loaded"
Expand Down
6 changes: 3 additions & 3 deletions ...s/Visp.Compiler.UnitTests/snapshots/tests_include_include-macros-0.can parse.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ let state = { Todo = () }
// line 8 @"include-macros-0.visp"
printfn "include-macros-0.visp loading..."
()
// line 10 @"include-macros-0.visp"
// line 8 @"include-macros-0-macros.visp"
printfn "include-macros-0-macros.visp loading..."
// line 10 @"include-macros-0.visp"
// line 10 @"include-macros-0-macros.visp"
let macro_MyStruct = "__MACRO_INIT__"
// line 10 @"include-macros-0.visp"
// line 18 @"include-macros-0-macros.visp"
printfn "include-macros-0-macros.visp loaded"
// line 12 @"include-macros-0.visp"
printfn "include-macros-0.visp loaded"
Expand Down
Loading