From 1e0b789379746bc7d1b4e943696bc31fa8283342 Mon Sep 17 00:00:00 2001 From: Won-hee Cho <89635107+wcho21@users.noreply.github.com> Date: Thu, 25 Jan 2024 03:09:52 +0900 Subject: [PATCH] better typing for tokens (#49) --- src/lexer/char-buffer/char-reader/index.ts | 56 ++-- src/lexer/index.test.ts | 270 ++++++++++++------ src/lexer/index.ts | 4 +- src/lexer/source-token/base/index.ts | 37 ++- .../source-token/delimiter/index.test.ts | 96 +++++++ src/lexer/source-token/delimiter/index.ts | 47 +-- .../source-token/identifier/index.test.ts | 71 +++++ src/lexer/source-token/identifier/index.ts | 40 +-- src/lexer/source-token/literal/index.test.ts | 96 +++++++ src/lexer/source-token/literal/index.ts | 49 +--- src/lexer/source-token/operator/index.test.ts | 46 +++ src/lexer/source-token/operator/index.ts | 17 +- src/lexer/source-token/special/index.test.ts | 96 +++++++ src/lexer/source-token/special/index.ts | 47 +-- .../source-token/testing/fixtures/index.ts | 1 + src/lexer/token/index.ts | 1 + src/util/position/index.ts | 4 +- 17 files changed, 691 insertions(+), 287 deletions(-) create mode 100644 src/lexer/source-token/delimiter/index.test.ts create mode 100644 src/lexer/source-token/identifier/index.test.ts create mode 100644 src/lexer/source-token/literal/index.test.ts create mode 100644 src/lexer/source-token/operator/index.test.ts create mode 100644 src/lexer/source-token/special/index.test.ts create mode 100644 src/lexer/source-token/testing/fixtures/index.ts diff --git a/src/lexer/char-buffer/char-reader/index.ts b/src/lexer/char-buffer/char-reader/index.ts index f7a3f6c..dc4de63 100644 --- a/src/lexer/char-buffer/char-reader/index.ts +++ b/src/lexer/char-buffer/char-reader/index.ts @@ -20,33 +20,7 @@ export default class CharReader { this.col = 0; } - private peekNewLine(): string | null { - // return if end of input - if (this.index === this.chars.length) { - return null; - } - - // return if no new line - const char = this.chars[this.index]; - if (char !== "\r" && char !== "\n") { - return null; - } - - // return if the last character - if (this.index+1 === this.chars.length) { - return char; - } - - // return if single-character new line - const nextChar = this.chars[this.index+1]; - if (nextChar !== "\r" && nextChar !== "\n") { - return char; - } - - // return two-character new line - return char + nextChar; - } - + /** return character at current position */ readChar(): SourceChar { // return fallback character if end of input if (this.index === this.chars.length) { @@ -85,6 +59,7 @@ export default class CharReader { return sourceChar; } + /** advance to next character */ advance(): void { if (this.index === this.chars.length) { return; @@ -104,6 +79,33 @@ export default class CharReader { ++this.col; } + private peekNewLine(): string | null { + // return if end of input + if (this.index === this.chars.length) { + return null; + } + + // return if no new line + const char = this.chars[this.index]; + if (char !== "\r" && char !== "\n") { + return null; + } + + // return if the last character + if (this.index+1 === this.chars.length) { + return char; + } + + // return if single-character new line + const nextChar = this.chars[this.index+1]; + if (nextChar !== "\r" && nextChar !== "\n") { + return char; + } + + // return two-character new line + return char + nextChar; + } + /** @deprecated Returns current character; if end of input, return fallback character */ read(): string { if (this.index === this.chars.length) { diff --git a/src/lexer/index.test.ts b/src/lexer/index.test.ts index ac451d7..41814cc 100644 --- a/src/lexer/index.test.ts +++ b/src/lexer/index.test.ts @@ -48,8 +48,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "+", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -57,8 +59,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "-", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -66,8 +70,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "*", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -75,8 +81,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "/", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -84,8 +92,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "=", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -93,8 +103,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "==", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, { @@ -102,8 +114,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "!", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -111,8 +125,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "!=", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, { @@ -120,8 +136,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: ">", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -129,8 +147,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: ">=", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, { @@ -138,8 +158,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "<", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -147,8 +169,10 @@ describe("getSourceToken()", () => { expected: { type: "operator", value: "<=", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, ]; @@ -163,8 +187,10 @@ describe("getSourceToken()", () => { expected: { type: "group delimiter", value: "(", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -172,8 +198,10 @@ describe("getSourceToken()", () => { expected: { type: "group delimiter", value: ")", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -181,8 +209,10 @@ describe("getSourceToken()", () => { expected: { type: "block delimiter", value: "{", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -190,8 +220,10 @@ describe("getSourceToken()", () => { expected: { type: "block delimiter", value: "}", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -199,8 +231,10 @@ describe("getSourceToken()", () => { expected: { type: "separator", value: ",", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, ]; @@ -215,8 +249,10 @@ describe("getSourceToken()", () => { expected: { type: "number literal", value: "0", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -224,8 +260,10 @@ describe("getSourceToken()", () => { expected: { type: "number literal", value: "1", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -233,8 +271,10 @@ describe("getSourceToken()", () => { expected: { type: "number literal", value: "1234", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 3 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 3 }, + }, }, }, { @@ -242,8 +282,10 @@ describe("getSourceToken()", () => { expected: { type: "number literal", value: "12.34", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 4 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 4 }, + }, }, }, ]; @@ -258,8 +300,10 @@ describe("getSourceToken()", () => { expected: { type: "boolean literal", value: "참", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -267,8 +311,10 @@ describe("getSourceToken()", () => { expected: { type: "boolean literal", value: "거짓", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, ]; @@ -283,8 +329,10 @@ describe("getSourceToken()", () => { expected: { type: "string literal", value: "foo bar 123 !@# 참", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 18 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 18 }, + }, }, }, { @@ -292,8 +340,10 @@ describe("getSourceToken()", () => { expected: { type: "string literal", value: "", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, ]; @@ -308,8 +358,10 @@ describe("getSourceToken()", () => { expected: { type: "keyword", value: "만약", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, { @@ -317,8 +369,10 @@ describe("getSourceToken()", () => { expected: { type: "keyword", value: "아니면", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 2 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 2 }, + }, }, }, { @@ -326,8 +380,10 @@ describe("getSourceToken()", () => { expected: { type: "keyword", value: "함수", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, { @@ -335,8 +391,10 @@ describe("getSourceToken()", () => { expected: { type: "keyword", value: "결과", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, }, ]; @@ -351,8 +409,10 @@ describe("getSourceToken()", () => { expected: { type: "identifier", value: "Foo이름123_", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 8 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 8 }, + }, }, }, { @@ -360,8 +420,10 @@ describe("getSourceToken()", () => { expected: { type: "identifier", value: "이름Foo123_", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 8 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 8 }, + }, }, }, { @@ -369,8 +431,10 @@ describe("getSourceToken()", () => { expected: { type: "identifier", value: "_이름Foo123", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 8 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 8 }, + }, }, }, ]; @@ -385,8 +449,10 @@ describe("getSourceToken()", () => { expected: { type: "illegal", value: "$", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, { @@ -394,8 +460,10 @@ describe("getSourceToken()", () => { expected: { type: "illegal string", value: "foo", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 4 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 4 }, + }, }, }, { @@ -403,8 +471,10 @@ describe("getSourceToken()", () => { expected: { type: "end", value: "$end", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 0 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 0 }, + }, }, }, ]; @@ -421,26 +491,34 @@ describe("getSourceToken()", () => { { type: "number literal", value: "12", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, { type: "operator", value: "+", - posBegin: { row: 0, col: 3 }, - posEnd: { row: 0, col: 3 }, + range: { + begin: { row: 0, col: 3 }, + end: { row: 0, col: 3 }, + }, }, { type: "number literal", value: "34", - posBegin: { row: 0, col: 5 }, - posEnd: { row: 0, col: 6 }, + range: { + begin: { row: 0, col: 5 }, + end: { row: 0, col: 6 }, + }, }, { type: "end", value: "$end", - posBegin: { row: 0, col: 7 }, - posEnd: { row: 0, col: 7 }, + range: { + begin: { row: 0, col: 7 }, + end: { row: 0, col: 7 }, + }, }, ] }, @@ -450,38 +528,50 @@ describe("getSourceToken()", () => { { type: "keyword", value: "만약", - posBegin: { row: 0, col: 0 }, - posEnd: { row: 0, col: 1 }, + range: { + begin: { row: 0, col: 0 }, + end: { row: 0, col: 1 }, + }, }, { type: "boolean literal", value: "참", - posBegin: { row: 0, col: 3 }, - posEnd: { row: 0, col: 3 }, + range: { + begin: { row: 0, col: 3 }, + end: { row: 0, col: 3 }, + }, }, { type: "block delimiter", value: "{", - posBegin: { row: 0, col: 5 }, - posEnd: { row: 0, col: 5 }, + range: { + begin: { row: 0, col: 5 }, + end: { row: 0, col: 5 }, + }, }, { type: "number literal", value: "12", - posBegin: { row: 1, col: 2 }, - posEnd: { row: 1, col: 3 }, + range: { + begin: { row: 1, col: 2 }, + end: { row: 1, col: 3 }, + }, }, { type: "block delimiter", value: "}", - posBegin: { row: 2, col: 0 }, - posEnd: { row: 2, col: 0 }, + range: { + begin: { row: 2, col: 0 }, + end: { row: 2, col: 0 }, + }, }, { type: "end", value: "$end", - posBegin: { row: 2, col: 1 }, - posEnd: { row: 2, col: 1 }, + range: { + begin: { row: 2, col: 1 }, + end: { row: 2, col: 1 }, + }, }, ] }, diff --git a/src/lexer/index.ts b/src/lexer/index.ts index a60f30a..c315684 100644 --- a/src/lexer/index.ts +++ b/src/lexer/index.ts @@ -123,8 +123,7 @@ export default class Lexer { { const { position: pos } = this.charBuffer.popChar(); - // TODO - const token = createEndToken(pos, pos); + const token = createEndToken("$end", pos, pos); return token; } @@ -564,3 +563,4 @@ export default class Lexer { } export type { TokenType } from "./token"; +export type { SourceToken } from "./source-token"; diff --git a/src/lexer/source-token/base/index.ts b/src/lexer/source-token/base/index.ts index 445461e..92512f9 100644 --- a/src/lexer/source-token/base/index.ts +++ b/src/lexer/source-token/base/index.ts @@ -1,8 +1,35 @@ import type Position from "../../../util/position"; -export interface SourceTokenBase { - type: string, - value: string, - posBegin: Position, - posEnd: Position, +export interface Range { + readonly begin: Position, + readonly end: Position, }; + +export interface SourceTokenBase { + readonly type: T, + readonly value: V, + readonly range: Range, +}; + +/** returns overloaded token creator function */ +export function createTokenCreator(type: T["type"]) { + // explicitly specify the return type since the overloaded function cannot infer it + type Token = { type: T["type"], value: T["value"], range: Range }; + + function createToken(value: T["value"], range: Range): Token; + function createToken(value: T["value"], pos1: Position, pos2: Position): Token; + function createToken(value: T["value"], arg1: Position | Range, pos2?: Position): Token { + if (pos2 !== undefined) { + const range = { begin: arg1 as Position, end: pos2 as Position }; + return { type, value, range }; + } + + return { type, value, range: arg1 as { begin: Position, end: Position } }; + }; + + return createToken; +}; + +declare function createToken(value: T["value"], range: Range): T; +declare function createToken(value: T["value"], pos1: Position, pos2: Position): T; +export type CreateToken = typeof createToken; diff --git a/src/lexer/source-token/delimiter/index.test.ts b/src/lexer/source-token/delimiter/index.test.ts new file mode 100644 index 0000000..0aeff2c --- /dev/null +++ b/src/lexer/source-token/delimiter/index.test.ts @@ -0,0 +1,96 @@ +import { + createGroupDelimiterToken, + createBlockDelimiterToken, + createSeparatorToken, +} from "./"; +import { fakePos } from "../testing/fixtures"; + +describe("create token with begin and end position", () => { + const cases = [ + { + name: "group delimiter", + token: createGroupDelimiterToken("(", fakePos, fakePos), + expected: { + type: "group delimiter", + value: "(", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "block delimiter", + token: createBlockDelimiterToken("{", fakePos, fakePos), + expected: { + type: "block delimiter", + value: "{", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "separator", + token: createSeparatorToken(",", fakePos, fakePos), + expected: { + type: "separator", + value: ",", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); + +describe("create token with range", () => { + const cases = [ + { + name: "group delimiter", + token: createGroupDelimiterToken("(", { begin: fakePos, end: fakePos }), + expected: { + type: "group delimiter", + value: "(", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "block delimiter", + token: createBlockDelimiterToken("{", { begin: fakePos, end: fakePos }), + expected: { + type: "block delimiter", + value: "{", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "separator", + token: createSeparatorToken(",", { begin: fakePos, end: fakePos }), + expected: { + type: "separator", + value: ",", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); diff --git a/src/lexer/source-token/delimiter/index.ts b/src/lexer/source-token/delimiter/index.ts index 54e23e3..79109b3 100644 --- a/src/lexer/source-token/delimiter/index.ts +++ b/src/lexer/source-token/delimiter/index.ts @@ -1,45 +1,14 @@ -import type Position from "../../../util/position"; -import type { SourceTokenBase } from "./../base"; +import type { SourceTokenBase, CreateToken } from "./../base"; +import { createTokenCreator } from "../base"; export type GroupDelimiterValue = "(" | ")"; export type BlockDelimiterValue = "{" | "}"; export type SeparatorValue = ","; -export interface GroupDelimiterToken extends SourceTokenBase { - type: "group delimiter"; - value: GroupDelimiterValue; -} +export type GroupDelimiterToken = SourceTokenBase<"group delimiter", GroupDelimiterValue>; +export type BlockDelimiterToken = SourceTokenBase<"block delimiter", BlockDelimiterValue>; +export type SeparatorToken = SourceTokenBase<"separator", SeparatorValue>; -export interface BlockDelimiterToken extends SourceTokenBase { - type: "block delimiter"; - value: BlockDelimiterValue; -} - -export interface SeparatorToken extends SourceTokenBase { - type: "separator", - value: SeparatorValue; -} - -type CreateGroupDelimiterToken = (value: GroupDelimiterValue, posBegin: Position, posEnd: Position) => GroupDelimiterToken; -export const createGroupDelimiterToken: CreateGroupDelimiterToken = (value, posBegin, posEnd) => ({ - type: "group delimiter", - value, - posBegin, - posEnd, -}); - -type CreateBlockDelimiterToken = (value: BlockDelimiterValue, posBegin: Position, posEnd: Position) => BlockDelimiterToken; -export const createBlockDelimiterToken: CreateBlockDelimiterToken = (value, posBegin, posEnd) => ({ - type: "block delimiter", - value, - posBegin, - posEnd, -}); - -type CreateSeparatorToken = (value: SeparatorValue, posBegin: Position, posEnd: Position) => SeparatorToken; -export const createSeparatorToken: CreateSeparatorToken = (value, posBegin, posEnd) => ({ - type: "separator", - value, - posBegin, - posEnd, -}); +export const createGroupDelimiterToken: CreateToken = createTokenCreator("group delimiter"); +export const createBlockDelimiterToken: CreateToken = createTokenCreator("block delimiter"); +export const createSeparatorToken: CreateToken = createTokenCreator("separator"); diff --git a/src/lexer/source-token/identifier/index.test.ts b/src/lexer/source-token/identifier/index.test.ts new file mode 100644 index 0000000..e86d0a3 --- /dev/null +++ b/src/lexer/source-token/identifier/index.test.ts @@ -0,0 +1,71 @@ +import { + createIdentifierToken, + createKeywordToken, +} from "./"; +import { fakePos } from "../testing/fixtures"; + +describe("create token with begin and end position", () => { + const cases = [ + { + name: "identifier", + token: createIdentifierToken("foo", fakePos, fakePos), + expected: { + type: "identifier", + value: "foo", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "keyword", + token: createKeywordToken("만약", fakePos, fakePos), + expected: { + type: "keyword", + value: "만약", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); + +describe("create token with range", () => { + const cases = [ + { + name: "identifier", + token: createIdentifierToken("foo", { begin: fakePos, end: fakePos }), + expected: { + type: "identifier", + value: "foo", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "keyword", + token: createKeywordToken("만약", { begin: fakePos, end: fakePos }), + expected: { + type: "keyword", + value: "만약", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); diff --git a/src/lexer/source-token/identifier/index.ts b/src/lexer/source-token/identifier/index.ts index 32948ae..2fb4943 100644 --- a/src/lexer/source-token/identifier/index.ts +++ b/src/lexer/source-token/identifier/index.ts @@ -1,33 +1,13 @@ -import type Position from "../../../util/position"; -import type { SourceTokenBase } from "./../base"; +import type { SourceTokenBase, CreateToken } from "./../base"; +import { createTokenCreator } from "../base"; -type KeywordValue = BranchKeywordValue | FunctionKeywordValue | ReturnKeywordValue; -type BranchKeywordValue = "만약" | "아니면"; -type FunctionKeywordValue = "함수"; -type ReturnKeywordValue = "결과"; +export type KeywordValue = BranchKeywordValue | FunctionKeywordValue | ReturnKeywordValue; +export type BranchKeywordValue = "만약" | "아니면"; +export type FunctionKeywordValue = "함수"; +export type ReturnKeywordValue = "결과"; -export interface IdentifierToken extends SourceTokenBase { - type: "identifier"; - value: string; -} +export type IdentifierToken = SourceTokenBase<"identifier", string>; +export type KeywordToken = SourceTokenBase<"keyword", KeywordValue>; -export interface KeywordToken extends SourceTokenBase { - type: "keyword"; - value: KeywordValue; -} - -type CreateIdentifierToken = (value: string, posBegin: Position, posEnd: Position) => IdentifierToken; -export const createIdentifierToken: CreateIdentifierToken = (value, posBegin, posEnd) => ({ - type: "identifier", - value, - posBegin, - posEnd, -}); - -type CreateKeywordToken = (value: KeywordValue, posBegin: Position, posEnd: Position) => KeywordToken; -export const createKeywordToken: CreateKeywordToken = (value, posBegin, posEnd) => ({ - type: "keyword", - value, - posBegin, - posEnd, -}); +export const createIdentifierToken: CreateToken = createTokenCreator("identifier"); +export const createKeywordToken: CreateToken = createTokenCreator("keyword"); diff --git a/src/lexer/source-token/literal/index.test.ts b/src/lexer/source-token/literal/index.test.ts new file mode 100644 index 0000000..43509ec --- /dev/null +++ b/src/lexer/source-token/literal/index.test.ts @@ -0,0 +1,96 @@ +import { + createNumberLiteralToken, + createBooleanLiteralToken, + createStringLiteralToken, +} from "./"; +import { fakePos } from "../testing/fixtures"; + +describe("create token with begin and end position", () => { + const cases = [ + { + name: "number literal", + token: createNumberLiteralToken("0", fakePos, fakePos), + expected: { + type: "number literal", + value: "0", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "boolean literal", + token: createBooleanLiteralToken("참", fakePos, fakePos), + expected: { + type: "boolean literal", + value: "참", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "string literal", + token: createStringLiteralToken("foo", fakePos, fakePos), + expected: { + type: "string literal", + value: "foo", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); + +describe("create token with range", () => { + const cases = [ + { + name: "number literal", + token: createNumberLiteralToken("0", { begin: fakePos, end: fakePos }), + expected: { + type: "number literal", + value: "0", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "boolean literal", + token: createBooleanLiteralToken("참", { begin: fakePos, end: fakePos }), + expected: { + type: "boolean literal", + value: "참", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "string literal", + token: createStringLiteralToken("foo", { begin: fakePos, end: fakePos }), + expected: { + type: "string literal", + value: "foo", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); diff --git a/src/lexer/source-token/literal/index.ts b/src/lexer/source-token/literal/index.ts index 05905d1..4377e42 100644 --- a/src/lexer/source-token/literal/index.ts +++ b/src/lexer/source-token/literal/index.ts @@ -1,43 +1,12 @@ -import type Position from "../../../util/position"; -import type { SourceTokenBase } from "./../base"; +import type { SourceTokenBase, CreateToken } from "./../base"; +import { createTokenCreator } from "../base"; -type BooleanLiteralValue = "참" | "거짓"; +export type BooleanLiteralValue = "참" | "거짓"; -export interface NumberLiteralToken extends SourceTokenBase { - type: "number literal"; - value: string; -} +export type NumberLiteralToken = SourceTokenBase<"number literal", string>; +export type BooleanLiteralToken = SourceTokenBase<"boolean literal", BooleanLiteralValue>; +export type StringLiteralToken = SourceTokenBase<"string literal", string>; -export interface BooleanLiteralToken extends SourceTokenBase { - type: "boolean literal"; - value: BooleanLiteralValue; -} - -export interface StringLiteralToken extends SourceTokenBase { - type: "string literal"; - value: string; -} - -type CreateNumberLiteralToken = (value: string, posBegin: Position, posEnd: Position) => NumberLiteralToken; -export const createNumberLiteralToken: CreateNumberLiteralToken = (value, posBegin, posEnd) => ({ - type: "number literal", - value, - posBegin, - posEnd, -}); - -type CreateBooleanLiteralToken = (value: BooleanLiteralValue, posBegin: Position, posEnd: Position) => BooleanLiteralToken; -export const createBooleanLiteralToken: CreateBooleanLiteralToken = (value, posBegin, posEnd) => ({ - type: "boolean literal", - value, - posBegin, - posEnd, -}); - -type CreateStringLiteralToken = (value: string, posBegin: Position, posEnd: Position) => StringLiteralToken; -export const createStringLiteralToken: CreateStringLiteralToken = (value, posBegin, posEnd) => ({ - type: "string literal", - value, - posBegin, - posEnd, -}); +export const createNumberLiteralToken: CreateToken = createTokenCreator("number literal"); +export const createBooleanLiteralToken: CreateToken = createTokenCreator("boolean literal"); +export const createStringLiteralToken: CreateToken = createTokenCreator("string literal"); diff --git a/src/lexer/source-token/operator/index.test.ts b/src/lexer/source-token/operator/index.test.ts new file mode 100644 index 0000000..33a68d9 --- /dev/null +++ b/src/lexer/source-token/operator/index.test.ts @@ -0,0 +1,46 @@ +import { + createOperatorToken, +} from "./"; +import { fakePos } from "../testing/fixtures"; + +describe("create token with begin and end position", () => { + const cases = [ + { + name: "operator", + token: createOperatorToken("+", fakePos, fakePos), + expected: { + type: "operator", + value: "+", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); + +describe("create token with range", () => { + const cases = [ + { + name: "operator", + token: createOperatorToken("+", { begin: fakePos, end: fakePos }), + expected: { + type: "operator", + value: "+", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); diff --git a/src/lexer/source-token/operator/index.ts b/src/lexer/source-token/operator/index.ts index 693ed16..97d02ff 100644 --- a/src/lexer/source-token/operator/index.ts +++ b/src/lexer/source-token/operator/index.ts @@ -1,20 +1,11 @@ -import type Position from "../../../util/position"; -import type { SourceTokenBase } from "./../base"; +import type { SourceTokenBase, CreateToken } from "./../base"; +import { createTokenCreator } from "../base"; export type OperatorValue = ArithmeticOperatorValue | AssignmentOperatorValue | LogicalOperatorValue; export type ArithmeticOperatorValue = "+" | "-" | "*" | "/"; export type AssignmentOperatorValue = "="; export type LogicalOperatorValue = "!" | "!=" | "==" | ">" | "<" | ">=" | "<="; -export interface OperatorToken extends SourceTokenBase { - type: "operator", - value: OperatorValue, -} +export type OperatorToken = SourceTokenBase<"operator", OperatorValue>; -type CreateOperatorToken = (value: OperatorValue, posBegin: Position, posEnd: Position) => OperatorToken; -export const createOperatorToken: CreateOperatorToken = (value, posBegin, posEnd) => ({ - type: "operator", - value, - posBegin, - posEnd, -}); +export const createOperatorToken: CreateToken = createTokenCreator("operator"); diff --git a/src/lexer/source-token/special/index.test.ts b/src/lexer/source-token/special/index.test.ts new file mode 100644 index 0000000..6aa7a1b --- /dev/null +++ b/src/lexer/source-token/special/index.test.ts @@ -0,0 +1,96 @@ +import { + createIllegalToken, + createIllegalStringLiteralToken, + createEndToken, +} from "./"; +import { fakePos } from "../testing/fixtures"; + +describe("create token with begin and end position", () => { + const cases = [ + { + name: "illegal", + token: createIllegalToken("$", fakePos, fakePos), + expected: { + type: "illegal", + value: "$", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "illegal string", + token: createIllegalStringLiteralToken("foo", fakePos, fakePos), + expected: { + type: "illegal string", + value: "foo", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "end", + token: createEndToken("$end", fakePos, fakePos), + expected: { + type: "end", + value: "$end", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); + +describe("create token with range", () => { + const cases = [ + { + name: "illegal", + token: createIllegalToken("$end", { begin: fakePos, end: fakePos }), + expected: { + type: "illegal", + value: "$end", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "illegal string", + token: createIllegalStringLiteralToken("foo", { begin: fakePos, end: fakePos }), + expected: { + type: "illegal string", + value: "foo", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + { + name: "end", + token: createEndToken("$end", { begin: fakePos, end: fakePos }), + expected: { + type: "end", + value: "$end", + range: { + begin: fakePos, + end: fakePos, + }, + }, + }, + ]; + + it.each(cases)("create $name token", ({ token, expected }) => { + expect(token).toEqual(expected); + }); +}); diff --git a/src/lexer/source-token/special/index.ts b/src/lexer/source-token/special/index.ts index 3f6f116..6e50e57 100644 --- a/src/lexer/source-token/special/index.ts +++ b/src/lexer/source-token/special/index.ts @@ -1,44 +1,13 @@ -import type Position from "../../../util/position"; -import type { SourceTokenBase } from "./../base"; +import type { SourceTokenBase, CreateToken } from "./../base"; +import { createTokenCreator } from "../base"; export const END_VALUE = "$end"; // unreadable character '$' used to avoid other token values type EndValue = typeof END_VALUE; -export interface IllegalToken extends SourceTokenBase { - type: "illegal"; - value: string; -} +export type IllegalToken = SourceTokenBase<"illegal", string>; +export type IllegalStringLiteralToken = SourceTokenBase<"illegal string", string>; +export type EndToken = SourceTokenBase<"end", EndValue>; -export interface IllegalStringLiteralToken extends SourceTokenBase { - type: "illegal string"; - value: string; -} - -export interface EndToken extends SourceTokenBase { - type: "end"; - value: EndValue -} - -type CreateIllegalToken = (value: string, posBegin: Position, posEnd: Position) => IllegalToken; -export const createIllegalToken: CreateIllegalToken = (value, posBegin, posEnd) => ({ - type: "illegal", - value, - posBegin, - posEnd, -}); - -type CreateIllegalStringLiteralToken = (value: string, posBegin: Position, posEnd: Position) => IllegalStringLiteralToken; -export const createIllegalStringLiteralToken: CreateIllegalStringLiteralToken = (value, posBegin, posEnd) => ({ - type: "illegal string", - value, - posBegin, - posEnd, -}); - -type CreateEndToken = (posBegin: Position, posEnd: Position) => EndToken; -export const createEndToken: CreateEndToken = (posBegin, posEnd) => ({ - type: "end", - value: END_VALUE, - posBegin, - posEnd, -}); +export const createIllegalToken: CreateToken = createTokenCreator("illegal"); +export const createIllegalStringLiteralToken: CreateToken = createTokenCreator("illegal string"); +export const createEndToken: CreateToken = createTokenCreator("end"); diff --git a/src/lexer/source-token/testing/fixtures/index.ts b/src/lexer/source-token/testing/fixtures/index.ts new file mode 100644 index 0000000..441a3d0 --- /dev/null +++ b/src/lexer/source-token/testing/fixtures/index.ts @@ -0,0 +1 @@ +export const fakePos = { row: 0, col: 0 }; diff --git a/src/lexer/token/index.ts b/src/lexer/token/index.ts index cb8d1de..cced492 100644 --- a/src/lexer/token/index.ts +++ b/src/lexer/token/index.ts @@ -1,3 +1,4 @@ +/** @deprecated */ export type TokenType = Operator | Identifier | diff --git a/src/util/position/index.ts b/src/util/position/index.ts index b2c5565..c37a918 100644 --- a/src/util/position/index.ts +++ b/src/util/position/index.ts @@ -1,4 +1,4 @@ export default interface Position { - readonly row: number; - readonly col: number; + readonly row: number, + readonly col: number, }