Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor/lexer-improvements #44

Merged
merged 5 commits into from
Dec 26, 2023
Merged

Refactor/lexer-improvements #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
29d8b2e
Refactor LexHelper and support source identifiers
vipentti Dec 26, 2023
5722b79
Simplify lexer rules and use symbolOrKeywordToken
vipentti Dec 26, 2023
fdae4b7
Separate token stream interning from default and fix macro interning
vipentti Dec 26, 2023
4198799
Normalize SOURCE_DIRECTORY in tests and specify custom debugToken for…
vipentti Dec 26, 2023
0afc37d
Do not output full path for SourceIdentifier in tests
vipentti Dec 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 55 additions & 75 deletions src/Visp.Compiler/Lexer.fsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,41 @@ let punct = [

let propShort = letter ident_char*

let keyword_ex = ':' propShort

let tokenSymbol = (
':' ident_char+
| ident
| ident_arrow
)

let tokenStreamSymbol = (
':' ident_char+
| '.' ident_char+
| '+' ident_char+
| '-' ident_char+
| ident_arrow
| "->>"
| "->"
| "+"
| "/"
| "-"
| "*"
| "..."
| ">="
| "<="
| ">>"
| "!="
| '<'
| '>'
| '='
| "&&"
| "||"
| ":>"
| ":?"
| ident
)

rule token (args: LexArgs) (skip: bool) = parse
| whitespace { token args skip lexbuf }
| newline { newline lexbuf; token args skip lexbuf }
Expand All @@ -270,14 +305,13 @@ rule token (args: LexArgs) (skip: bool) = parse

// punct

| ident_arrow { SYMBOL (lexeme lexbuf) }
| ident_letter_chars ">>" { TOKENLIST [SYMBOL (lexemeTrimRight lexbuf 2); OP_GREATER; OP_GREATER] }
| "<<" ident_letter_chars { SYMBOL (lexeme lexbuf) }
| ident_letter_chars ">>" { TOKENLIST [symbolOrKeywordToken args lexbuf (lexemeTrimRight lexbuf 2); OP_GREATER; OP_GREATER] }
| "<<" ident_letter_chars { symbolOrKeywordToken args lexbuf (lexeme lexbuf) }

| infix_operators { INFIX_OP (lexeme lexbuf) }
| '(' infix_operators { TOKENLIST [LPAREN; INFIX_OP (lexemeTrimLeft lexbuf 1)] }
| unary_operators { UNARY_OP (lexeme lexbuf) }
| '(' unary_operators { TOKENLIST [LPAREN; UNARY_OP (lexemeTrimLeft lexbuf 1)] }
| infix_operators { symbolOrKeywordToken args lexbuf (lexeme lexbuf) }
| '(' infix_operators { TOKENLIST [LPAREN; symbolOrKeywordToken args lexbuf (lexemeTrimLeft lexbuf 1)] }
| unary_operators { symbolOrKeywordToken args lexbuf (lexeme lexbuf) }
| '(' unary_operators { TOKENLIST [LPAREN; symbolOrKeywordToken args lexbuf (lexemeTrimLeft lexbuf 1)] }
| '(' { LPAREN }
| ')' { RPAREN }
| '{' { LBRACE }
Expand All @@ -288,12 +322,8 @@ rule token (args: LexArgs) (skip: bool) = parse
| ':' anyspace+ { COLON }
| ',' { COMMA }
| '|' { BAR }

| "..." { SYMBOL (lexeme lexbuf) }


| "#nowarn" { HASH_IDENT ((lexeme lexbuf).TrimStart('#'))}

| "..." { SYMBOL (lexeme lexbuf) }
| "#nowarn" { HASH_IDENT ((lexeme lexbuf).TrimStart('#'))}
| '.' propShort { DOT_METHOD (lexeme lexbuf) }
| '-' propShort { APPLY_METHOD (lexeme lexbuf) }
| '+' propShort { PROP_PLUS (lexeme lexbuf) }
Expand Down Expand Up @@ -457,6 +487,8 @@ rule token (args: LexArgs) (skip: bool) = parse

| '+' { OP_PLUS }
| '-' { OP_MINUS }
| '*' { OP_MULT }
| '/' { OP_DIV }
| '.' { DOT }
| ">=" { GREATER_EQUALS }
| "<=" { LESS_EQUALS }
Expand Down Expand Up @@ -509,14 +541,10 @@ rule token (args: LexArgs) (skip: bool) = parse
// TODO: validate
singleQuoteString (buf, fin, m, LexerStringKind.InterpolatedStringFirst, args) skip lexbuf
}

| '*' { OP_MULT }
| '/' { OP_DIV }
| ':' propShort { KEYWORD (lexeme lexbuf) }
| ':' ident_char+ { SYMBOL (lexeme lexbuf) }
| ident {
| keyword_ex { KEYWORD (lexeme lexbuf) }
| tokenSymbol {
let text = lexeme lexbuf
symbolOrKeyword args.CurrentContext text
symbolOrKeywordToken args lexbuf text
}
| _ { unexpected_char "token" lexbuf }

Expand All @@ -530,10 +558,10 @@ and tokenStream (args: LexArgs) (skip: bool) = parse

// punct

| infix_operators { SYMBOL (lexeme lexbuf) }
| '(' infix_operators { TOKENLIST [LPAREN; SYMBOL (lexemeTrimLeft lexbuf 1)] }
| unary_operators { SYMBOL (lexeme lexbuf) }
| '(' unary_operators { TOKENLIST [LPAREN; SYMBOL (lexemeTrimLeft lexbuf 1)] }
| infix_operators { symbolOrKeywordToken args lexbuf (lexeme lexbuf) }
| '(' infix_operators { TOKENLIST [LPAREN; symbolOrKeywordToken args lexbuf (lexemeTrimLeft lexbuf 1)] }
| unary_operators { symbolOrKeywordToken args lexbuf (lexeme lexbuf) }
| '(' unary_operators { TOKENLIST [LPAREN; symbolOrKeywordToken args lexbuf (lexemeTrimLeft lexbuf 1)] }
| '(' { LPAREN }
| ')' { RPAREN }
| '{' { LBRACE }
Expand All @@ -543,8 +571,6 @@ and tokenStream (args: LexArgs) (skip: bool) = parse
| "::" anyspace+ { COLON_COLON }
| ':' anyspace+ { COLON }
| ',' { COMMA }
| "||" { SYMBOL (lexeme lexbuf) }
| "-||" { SYMBOL (lexeme lexbuf) }
| '|' { BAR }
| '.' anyspace+ { DOT }
| "#(" { HASH_PAREN }
Expand All @@ -555,38 +581,10 @@ and tokenStream (args: LexArgs) (skip: bool) = parse
| "|}" { BAR_BRACE }
| "[|" { BRACKET_BAR }
| "|]" { BAR_BRACKET }

| "(|>" { TOKENLIST [LPAREN; SYMBOL (lexemeTrimLeft lexbuf 1)] }
| "(||)" { TOKENLIST [PAREN_BAR; BAR_PAREN] }
| "(||" { TOKENLIST [LPAREN; SYMBOL (lexemeTrimLeft lexbuf 1)] }
| "(|" { PAREN_BAR }
| "|)" { BAR_PAREN }


| "..." { SYMBOL (lexeme lexbuf) }

// operators
| '-' anyspace+ { SYMBOL "-" }
| '*' anyspace+ { SYMBOL "*" }
| '+' anyspace+ { SYMBOL "+" }
| '/' anyspace+ { SYMBOL "/" }

| ident_arrow { SYMBOL (lexeme lexbuf) }

| "->>" { SYMBOL (lexeme lexbuf) }
| "->" { SYMBOL (lexeme lexbuf) }
| ">=" { SYMBOL (lexeme lexbuf) }
| "<=" { SYMBOL (lexeme lexbuf) }
| ">>" { SYMBOL (lexeme lexbuf) }
| "!=" { SYMBOL (lexeme lexbuf) }
| '<' { SYMBOL (lexeme lexbuf) }
| '>' { SYMBOL (lexeme lexbuf) }
| '=' { SYMBOL (lexeme lexbuf) }
| "&&" { SYMBOL (lexeme lexbuf) }
| "||" { SYMBOL (lexeme lexbuf) }
| ":>" { SYMBOL (lexeme lexbuf) }
| ":?" { SYMBOL (lexeme lexbuf) }

// Constants
| "#()" { UNIT }
| "()" { UNIT }
Expand Down Expand Up @@ -776,28 +774,10 @@ and tokenStream (args: LexArgs) (skip: bool) = parse
}

// Exprs
| ':' propShort { KEYWORD (lexeme lexbuf) }
| ':' ident_char+ { SYMBOL (lexeme lexbuf) }
| '.' ident_char+ {
let text = lexeme lexbuf;
SYMBOL text }
| '+' ident_char+ {
let text = lexeme lexbuf;
SYMBOL text }
| '-' ident_char+ {
let text = lexeme lexbuf;
SYMBOL text }
| ident {
| keyword_ex { KEYWORD (lexeme lexbuf) }
| tokenStreamSymbol {
let text = lexeme lexbuf
if args.mode.IsQuasiquoteMode then
if text = "unquote" then
UNQUOTE_KW
else if text = "splice-unquote" then
SPLICE_UNQUOTE_KW
else
SYMBOL text
else
SYMBOL text }
symbolOrKeywordToken args lexbuf text }
| _ { unexpected_char "tokenStream" lexbuf }

and singleQuoteString (sargs: LexerStringArgs) (skip: bool) = parse
Expand Down
1 change: 1 addition & 0 deletions src/Visp.Compiler/ParseUtils.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ let debugTokenOutput args (lexbuf: LexBuffer<_>) =
function
| STRING(text, kind, cont) ->
Syntax.StringWriterUtils.writeDebugStringType "STRING" text kind cont
| KEYWORD_STRING(lhs, rhs) -> sprintf "KEYWORD_STRING (\"%s\", \"%s\")" lhs rhs
| it -> sprintf "%A" it

seq {
Expand Down
8 changes: 7 additions & 1 deletion src/Visp.Compiler/Parser.fsy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ let mkRecoveryPat s = Syntax.parserRecoveryPat (lhs s)
%token <System.Decimal> DECIMAL
%token <(string * string)> BIGNUM

// __SOURCE_FILE__ etc
%token <(string * string)> KEYWORD_STRING

%token <string> INFIX_OP
%token <string> UNARY_OP
%token <string> HASH_IDENT
Expand Down Expand Up @@ -108,7 +111,7 @@ let mkRecoveryPat s = Syntax.parserRecoveryPat (lhs s)
%nonassoc prec_new_no_exprs

%nonassoc INT8 INT16 INT32 INT64 NATIVEINT UINT8 UINT16 UINT32 UINT64 UNATIVEINT IEEE32 IEEE64 DECIMAL BIGNUM
%nonassoc TRUE FALSE NIL SYMBOL KEYWORD RAWSTRING DOT_METHOD APPLY_METHOD STRING UNIT CHAR
%nonassoc TRUE FALSE NIL SYMBOL KEYWORD RAWSTRING DOT_METHOD APPLY_METHOD STRING KEYWORD_STRING UNIT CHAR
%nonassoc HASH_IDENT INFIX_OP UNARY_OP
%nonassoc prec_symbol
%nonassoc prec_name_list
Expand Down Expand Up @@ -1207,6 +1210,9 @@ rawConstant:
| STRING
{ let (raw, kind, _) = $1
SynConst.String(raw, kind, (lhs parseState))}
| KEYWORD_STRING
{ let (s, c) = $1
SynConst.SourceIdentifier(s, c, lhs parseState)}
| INT8
{ if snd $1 then errorR(mkDiagnosticWithText (SR.lexOutsideEightBitSigned(), lhs parseState))
SynConst.SByte(fst $1) }
Expand Down
Loading