From d9d092797c12593af8c2f7498cb6e125044f8b63 Mon Sep 17 00:00:00 2001 From: "W. Cho" Date: Thu, 25 Jan 2024 03:32:40 +0900 Subject: [PATCH 1/3] make a source token reader with position context --- src/parser/source-token-reader/index.test.ts | 76 ++++++++++++++++++++ src/parser/source-token-reader/index.ts | 24 +++++++ 2 files changed, 100 insertions(+) create mode 100644 src/parser/source-token-reader/index.test.ts create mode 100644 src/parser/source-token-reader/index.ts diff --git a/src/parser/source-token-reader/index.test.ts b/src/parser/source-token-reader/index.test.ts new file mode 100644 index 0000000..7e222fd --- /dev/null +++ b/src/parser/source-token-reader/index.test.ts @@ -0,0 +1,76 @@ +import Lexer from "../../lexer"; +import SourceTokenReader from "./"; + +const createReader = (input: string) => { + const lexer = new Lexer(input); + + return new SourceTokenReader(lexer); +}; + +describe("read()", () => { + it("read a token", () => { + const input = "42"; + const reader = createReader(input); + const expected = { + type: "number literal", + value: "42", + range: { + begin: { col: 0, row: 0 }, + end: { col: 1, row: 0 }, + }, + }; + + expect(reader.read()).toEqual(expected); + }); + + it("read the end token if nothing to read", () => { + const input = ""; + const reader = createReader(input); + const expected = { + type: "end", + value: "$end", + range: { + begin: { col: 0, row: 0 }, + end: { col: 0, row: 0 }, + }, + }; + + expect(reader.read()).toEqual(expected); + }); +}); + +describe("advance()", () => { + it("advance to next token", () => { + const input = "42 99"; + const reader = createReader(input); + const expected = { + type: "number literal", + value: "99", + range: { + begin: { col: 3, row: 0 }, + end: { col: 4, row: 0 }, + }, + }; + + reader.advance(); + expect(reader.read()).toEqual(expected); + }); +}); + +describe("isEnd()", () => { + it("return true if end", () => { + const input = ""; + const reader = createReader(input); + const expected = true; + + expect(reader.isEnd()).toEqual(expected); + }); + + it("return false if not end", () => { + const input = "42"; + const reader = createReader(input); + const expected = false; + + expect(reader.isEnd()).toEqual(expected); + }); +}); diff --git a/src/parser/source-token-reader/index.ts b/src/parser/source-token-reader/index.ts new file mode 100644 index 0000000..341117b --- /dev/null +++ b/src/parser/source-token-reader/index.ts @@ -0,0 +1,24 @@ +import Lexer from "../../lexer"; +import type { SourceToken } from "../../lexer"; + +export default class SourceTokenReader { + private readonly lexer: Lexer; + private token: SourceToken; + + constructor(lexer: Lexer) { + this.lexer = lexer; + this.token = lexer.getSourceToken(); + } + + read(): SourceToken { + return this.token; + } + + advance(): void { + this.token = this.lexer.getSourceToken(); + } + + isEnd(): boolean { + return this.token.type === "end"; + } +} From 2b027e798d6edae05a234de087eece9c32703d76 Mon Sep 17 00:00:00 2001 From: "W. Cho" Date: Thu, 25 Jan 2024 03:33:15 +0900 Subject: [PATCH 2/3] mark as deprecated since it contains no position context --- src/parser/token-reader/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/parser/token-reader/index.ts b/src/parser/token-reader/index.ts index f1025a4..015b0c0 100644 --- a/src/parser/token-reader/index.ts +++ b/src/parser/token-reader/index.ts @@ -1,6 +1,7 @@ import Lexer from "../../lexer"; import type { TokenType } from "../../lexer"; +/** @deprecated */ export default class TokenReader { private readonly lexer: Lexer; private token: TokenType; @@ -10,14 +11,17 @@ export default class TokenReader { this.token = lexer.getToken(); } + /** @deprecated */ isEnd(): boolean { return this.token.type === "end"; } + /** @deprecated */ read(): TokenType { return this.token; } + /** @deprecated */ next(): void { this.token = this.lexer.getToken(); } From a2caab44e962a47615189a82f89f81bb90bff856 Mon Sep 17 00:00:00 2001 From: "W. Cho" Date: Thu, 25 Jan 2024 03:33:53 +0900 Subject: [PATCH 3/3] add syntax node with position context --- src/parser/syntax-node/base/index.ts | 25 ++++ .../syntax-node/expression/index.test.ts | 118 ++++++++++++++++++ src/parser/syntax-node/expression/index.ts | 35 ++++++ src/parser/syntax-node/group/index.test.ts | 33 +++++ src/parser/syntax-node/group/index.ts | 12 ++ src/parser/syntax-node/index.ts | 12 ++ .../syntax-node/statement/index.test.ts | 65 ++++++++++ src/parser/syntax-node/statement/index.ts | 15 +++ .../syntax-node/testing/fixtures/index.ts | 1 + 9 files changed, 316 insertions(+) create mode 100644 src/parser/syntax-node/base/index.ts create mode 100644 src/parser/syntax-node/expression/index.test.ts create mode 100644 src/parser/syntax-node/expression/index.ts create mode 100644 src/parser/syntax-node/group/index.test.ts create mode 100644 src/parser/syntax-node/group/index.ts create mode 100644 src/parser/syntax-node/index.ts create mode 100644 src/parser/syntax-node/statement/index.test.ts create mode 100644 src/parser/syntax-node/statement/index.ts create mode 100644 src/parser/syntax-node/testing/fixtures/index.ts diff --git a/src/parser/syntax-node/base/index.ts b/src/parser/syntax-node/base/index.ts new file mode 100644 index 0000000..68e77c5 --- /dev/null +++ b/src/parser/syntax-node/base/index.ts @@ -0,0 +1,25 @@ +import type Position from "../../../util/position"; + +export interface SyntaxNodeBase { + type: T, + range: { + begin: Position, + end: Position, + }, + fields: F, +}; + +export function createNodeCreator(type: N["type"]) { + function createNode(fields: N["fields"], rangeBegin: Position, rangeEnd: Position) { + const range = { + begin: rangeBegin, + end: rangeEnd, + }; + + return { type, range, fields }; + }; + + return createNode; +}; + +export type CreateNode = (fields: N["fields"], rangeBegin: Position, rangeEnd: Position) => N; diff --git a/src/parser/syntax-node/expression/index.test.ts b/src/parser/syntax-node/expression/index.test.ts new file mode 100644 index 0000000..3f40b1c --- /dev/null +++ b/src/parser/syntax-node/expression/index.test.ts @@ -0,0 +1,118 @@ +import { + createIdentifierNode, + createNumberNode, + createStringNode, + createPrefixNode, + createInfixNode, + createFunctionNode, + createCallNode, + createAssignmentNode, +} from "./"; +import type { + IdentifierNode, + ExpressionNode, +} from "./"; +import type { + BlockNode, +} from "../group"; +import { fakePos } from "../testing/fixtures"; + +const cases = [ + { + name: "identifier", + node: createIdentifierNode({ value: "foo" }, fakePos, fakePos), + expected: { + type: "identifier", + fields: { + value: "foo", + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "number", + node: createNumberNode({ value: 42 }, fakePos, fakePos), + expected: { + type: "number", + fields: { + value: 42, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "string", + node: createStringNode({ value: "foo" }, fakePos, fakePos), + expected: { + type: "string", + fields: { + value: "foo", + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "prefix", + node: createPrefixNode({ prefix: "+", right: {} as ExpressionNode }, fakePos, fakePos), + expected: { + type: "prefix", + fields: { + prefix: "+", + right: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "infix", + node: createInfixNode({ infix: "+", left: {} as ExpressionNode, right: {} as ExpressionNode }, fakePos, fakePos), + expected: { + type: "infix", + fields: { + infix: "+", + left: {}, + right: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "function", + node: createFunctionNode({ parameters: [] as IdentifierNode[], body: {} as BlockNode }, fakePos, fakePos), + expected: { + type: "function", + fields: { + parameters: [], + body: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "call", + node: createCallNode({ func: {} as IdentifierNode, args: [] as ExpressionNode[] }, fakePos, fakePos), + expected: { + type: "call", + fields: { + func: {}, + args: [], + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "assignment", + node: createAssignmentNode({ left: {} as IdentifierNode, right: {} as ExpressionNode }, fakePos, fakePos), + expected: { + type: "assignment", + fields: { + left: {}, + right: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, +]; +it.each(cases)("create $name node", ({ node, expected }) => { + expect(node).toEqual(expected); +}); diff --git a/src/parser/syntax-node/expression/index.ts b/src/parser/syntax-node/expression/index.ts new file mode 100644 index 0000000..6250ee7 --- /dev/null +++ b/src/parser/syntax-node/expression/index.ts @@ -0,0 +1,35 @@ +import type { SyntaxNodeBase, CreateNode } from "../base"; +import { createNodeCreator } from "../base"; +import type { BlockNode } from "../group"; + +export type Prefix = "+" | "-" | "!"; +export type Infix = "+" | "-" | "*" | "/" | "=" | "==" | "!=" | ">" | "<" | ">=" | "<="; + +export type ExpressionNode = IdentifierNode + | NumberNode + | BooleanNode + | StringNode + | PrefixNode + | InfixNode + | FunctionNode + | CallNode + | AssignmentNode; + +export interface IdentifierNode extends SyntaxNodeBase<"identifier", { value: string }> {}; +export interface NumberNode extends SyntaxNodeBase<"number", { value: number }> {}; +export interface BooleanNode extends SyntaxNodeBase<"boolean", { value: boolean }> {}; +export interface StringNode extends SyntaxNodeBase<"string", { value: string }> {}; +export interface PrefixNode extends SyntaxNodeBase<"prefix", { prefix: Prefix, right: ExpressionNode }> {}; +export interface InfixNode extends SyntaxNodeBase<"infix", { infix: Infix, left: ExpressionNode, right: ExpressionNode }> {}; +export interface FunctionNode extends SyntaxNodeBase<"function", { parameters: IdentifierNode[], body: BlockNode }> {}; +export interface CallNode extends SyntaxNodeBase<"call", { func: IdentifierNode | FunctionNode, args: ExpressionNode[] }> {}; +export interface AssignmentNode extends SyntaxNodeBase<"assignment", { left: IdentifierNode, right: ExpressionNode }> {}; + +export const createIdentifierNode: CreateNode = createNodeCreator("identifier"); +export const createNumberNode: CreateNode = createNodeCreator("number"); +export const createStringNode: CreateNode = createNodeCreator("string"); +export const createPrefixNode: CreateNode = createNodeCreator("prefix"); +export const createInfixNode: CreateNode = createNodeCreator("infix"); +export const createFunctionNode: CreateNode = createNodeCreator("function"); +export const createCallNode: CreateNode = createNodeCreator("call"); +export const createAssignmentNode: CreateNode = createNodeCreator("assignment"); diff --git a/src/parser/syntax-node/group/index.test.ts b/src/parser/syntax-node/group/index.test.ts new file mode 100644 index 0000000..0187f3c --- /dev/null +++ b/src/parser/syntax-node/group/index.test.ts @@ -0,0 +1,33 @@ +import { + createProgramNode, + createBlockNode, +} from "./"; +import { fakePos } from "../testing/fixtures"; + +const cases = [ + { + name: "program", + node: createProgramNode({ statements: [] }, fakePos, fakePos), + expected: { + type: "program", + fields: { + statements: [], + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "block", + node: createBlockNode({ statements: [] }, fakePos, fakePos), + expected: { + type: "block", + fields: { + statements: [], + }, + range: { begin: fakePos, end: fakePos }, + }, + }, +]; +it.each(cases)("create $name node", ({ node, expected }) => { + expect(node).toEqual(expected); +}); diff --git a/src/parser/syntax-node/group/index.ts b/src/parser/syntax-node/group/index.ts new file mode 100644 index 0000000..77a24ef --- /dev/null +++ b/src/parser/syntax-node/group/index.ts @@ -0,0 +1,12 @@ +import type { SyntaxNodeBase, CreateNode } from "../base"; +import { createNodeCreator } from "../base"; + +export type GroupNode = ProgramNode | BlockNode; + +/** a root node for a syntax tree of a program */ +export interface ProgramNode extends SyntaxNodeBase<"program", { statements: any[] }> {}; +/** a group of statements */ +export interface BlockNode extends SyntaxNodeBase<"block", { statements: any[] }> {}; + +export const createProgramNode: CreateNode = createNodeCreator("program"); +export const createBlockNode: CreateNode = createNodeCreator("block"); diff --git a/src/parser/syntax-node/index.ts b/src/parser/syntax-node/index.ts new file mode 100644 index 0000000..cdf0e22 --- /dev/null +++ b/src/parser/syntax-node/index.ts @@ -0,0 +1,12 @@ +import type { GroupNode } from "./group"; +import type { StatementNode } from "./statement"; +import type { ExpressionNode } from "./expression"; + +export type SyntaxNode = GroupNode | StatementNode | ExpressionNode; + +export * from "./group"; +export type * from "./group"; +export * from "./statement"; +export type * from "./statement"; +export * from "./expression"; +export type * from "./expression"; diff --git a/src/parser/syntax-node/statement/index.test.ts b/src/parser/syntax-node/statement/index.test.ts new file mode 100644 index 0000000..9e873d4 --- /dev/null +++ b/src/parser/syntax-node/statement/index.test.ts @@ -0,0 +1,65 @@ +import { + createBranchNode, + createReturnNode, + createExpressionStatementNode, +} from "./"; +import type { + ExpressionNode, +} from "../expression"; +import type { + BlockNode, +} from "../group"; +import { fakePos } from "../testing/fixtures"; + +const cases = [ + { + name: "branch", + node: createBranchNode({ predicate: {} as ExpressionNode, consequence: {} as BlockNode }, fakePos, fakePos), + expected: { + type: "branch", + fields: { + predicate: {}, + consequence: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "branch with alternative", + node: createBranchNode({ predicate: {} as ExpressionNode, consequence: {} as BlockNode, alternative: {} as BlockNode }, fakePos, fakePos), + expected: { + type: "branch", + fields: { + predicate: {}, + consequence: {}, + alternative: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "return", + node: createReturnNode({ expression: {} as ExpressionNode }, fakePos, fakePos), + expected: { + type: "return", + fields: { + expression: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, + { + name: "expression statement", + node: createExpressionStatementNode({ expression: {} as ExpressionNode }, fakePos, fakePos), + expected: { + type: "expression statement", + fields: { + expression: {}, + }, + range: { begin: fakePos, end: fakePos }, + }, + }, +]; +it.each(cases)("create $name node", ({ node, expected }) => { + expect(node).toEqual(expected); +}); diff --git a/src/parser/syntax-node/statement/index.ts b/src/parser/syntax-node/statement/index.ts new file mode 100644 index 0000000..88b0fd5 --- /dev/null +++ b/src/parser/syntax-node/statement/index.ts @@ -0,0 +1,15 @@ +import type { SyntaxNodeBase, CreateNode } from "../base"; +import { createNodeCreator } from "../base"; +import type { BlockNode } from "../group"; +import type { ExpressionNode } from "../expression"; + +export type StatementNode = BranchNode | ReturnNode | ExpressionStatementNode; + +export interface BranchNode extends SyntaxNodeBase<"branch", { predicate: ExpressionNode, consequence: BlockNode, alternative?: BlockNode }> {}; +export interface ReturnNode extends SyntaxNodeBase<"return", { expression: ExpressionNode }> {}; +/** A wrapper type to treat a single expression as a statement. */ +export interface ExpressionStatementNode extends SyntaxNodeBase<"expression statement", { expression: ExpressionNode }> {}; + +export const createBranchNode: CreateNode = createNodeCreator("branch"); +export const createReturnNode: CreateNode = createNodeCreator("return"); +export const createExpressionStatementNode: CreateNode = createNodeCreator("expression statement"); diff --git a/src/parser/syntax-node/testing/fixtures/index.ts b/src/parser/syntax-node/testing/fixtures/index.ts new file mode 100644 index 0000000..e6928c3 --- /dev/null +++ b/src/parser/syntax-node/testing/fixtures/index.ts @@ -0,0 +1 @@ +export const fakePos = { col: 0, row: 0 };