Skip to content

Commit

Permalink
Improved logging
Browse files Browse the repository at this point in the history 
Fixed incorrect node positions when ignoring tokens
Added comment and line terminator ignoring to assembly test
  • Loading branch information
@james-pre
james-pre committed Dec 14, 2024
1 parent 6471b2e commit 3b3a6e8
Show file tree
Hide file tree
Showing 6 changed files with 60 additions and 45 deletions.
14 changes: 11 additions & 3 deletions scripts/bnf.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const {
colors: { short: 'c', type: 'boolean' },
tokens: { short: 'T', type: 'string' },
parser: { short: 'P', type: 'string' },
verbose: { short: 'V', type: 'string', multiple: true },
verbose: { short: 'V', type: 'boolean', multiple: true },
help: { short: 'h', type: 'boolean', default: false },
},
allowPositionals: true,
Expand Down Expand Up @@ -45,6 +45,14 @@ if ((options.tokens == 'only' || options.parser == 'only') && (options.format ||
process.exit(1);
}

function logger(outputLevel) {
return function (level, message, depth) {
if (level > outputLevel) return;

console.log(' '.repeat(4 * depth) + (level > 1 ? '[debug] ' : '') + message);
};
}

const verbose = options.verbose?.filter(Boolean)?.length ?? 0;

const tokens = bnf.tokenize(readFileSync(input, 'utf8'));
Expand All @@ -56,7 +64,7 @@ if (options.tokens) {
if (options.tokens == 'only') process.exit(0);
}

const ast = bnf.parse(tokens, parseInt(options.ast));
const ast = bnf.parse(tokens, logger(parseInt(options.ast)));

if (options.ast) {
for (const node of ast) {
Expand All @@ -65,7 +73,7 @@ if (options.ast) {
if (options.parser == 'only') process.exit(0);
}

const config = bnf.convert(ast[0], verbose);
const config = bnf.convert(ast[0], logger(verbose));

const write = data => (options.output ? writeFileSync(options.output, data) : console.log);

Expand Down
52 changes: 22 additions & 30 deletions src/bnf.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import type { DefinitionPart, Node, NodeDefinition } from './parser.js';
import type { DefinitionPart, Logger, Node, NodeDefinition } from './parser.js';
import { parse } from './parser.js';
import type { Token, TokenDefinition } from './tokens.js';
import { tokenize } from './tokens.js';

export const literals = [
export const literals: TokenDefinition[] = [
{ name: 'identifier', pattern: /^[a-zA-Z_]\w*/ }, // Matches rule names like `identifier`, `register`, etc.
{ name: 'string', pattern: /^"[^"]*"/ }, // Quoted literals like `"0x"`, `"%"`, etc.
{ name: 'equal', pattern: /^=/ }, // Equals sign (`=`)
Expand All @@ -30,7 +30,7 @@ function tokenizeBnf(source: string): Token[] {

export { tokenizeBnf as tokenize };

export const definitions = [
export const definitions: NodeDefinition[] = [
// BNF Rule Definition: identifier = expression ;
{
name: 'rule',
Expand Down Expand Up @@ -100,27 +100,25 @@ export const definitions = [
{ kind: 'rule_list_continue', type: 'repeated' },
],
},
] as const satisfies NodeDefinition[];
];

export function parseSource(source: string, verbose: number = 0): Node[] {
export function parseSource(source: string, log?: Logger): Node[] {
return parse({
ignoreLiterals: ['whitespace', 'comment'],
definitions,
rootNode: 'rule_list',
debug: verbose > 1 ? console.debug : undefined,
verbose: verbose > 2 ? console.debug : undefined,
log,
source,
literals,
});
}

function parseBnf(tokens: Token[], verbose: number = 0): Node[] {
function parseBnf(tokens: Token[], log?: Logger): Node[] {
return parse({
ignoreLiterals: ['whitespace', 'comment'],
definitions,
rootNode: 'rule_list',
debug: verbose > 1 ? console.debug : undefined,
verbose: verbose > 2 ? console.debug : undefined,
log,
tokens,
literals: literals.map(t => t.name),
});
Expand All @@ -134,13 +132,7 @@ const typeForGroup = {
left_paren: 'required',
} as const;

export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefinition[]; literals: TokenDefinition[] } {
function _log(level: number, depth: number, message: string) {
if (level <= verbose) {
console.debug(' '.repeat(depth), message);
}
}

export function convert(ast: Node, log: Logger = () => {}): { definitions: NodeDefinition[]; literals: TokenDefinition[] } {
const definitions: NodeDefinition[] = [];
const literals: TokenDefinition[] = [];

Expand All @@ -149,8 +141,8 @@ export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefi
groups = 0;

function processNode(node: Node, depth: number = 0) {
const log = (level: number, text: string) => _log(level, depth, text);
log(3, `Processing ${node.kind} at ${node.line}:${node.column}`);
const _log = (level: number, text: string) => log(level, text, depth);
_log(3, `Processing ${node.kind} at ${node.line}:${node.column}`);
if (node.kind != 'rule') {
// Recursively process child nodes
for (const child of node.children || []) {
Expand All @@ -163,9 +155,9 @@ export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefi
const name = node.children?.find(child => child.kind === 'identifier')?.text;
const expression = node.children?.find(child => child.kind === 'expression');

log(2, `Found rule "${name}" at ${node.line}:${node.column}`);
_log(2, `Found rule "${name}" at ${node.line}:${node.column}`);
if (!name || !expression) {
log(1, 'Rule is missing name or expression');
_log(1, 'Rule is missing name or expression');
return;
}

Expand Down Expand Up @@ -204,31 +196,31 @@ export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefi

function processExpression(expression: Node, depth: number = 0): DefinitionPart[] {
isOneOf = false;
const log = (level: number, text: string) => _log(level, depth, text);
const _log = (level: number, text: string) => log(level, text, depth);
const pattern: DefinitionPart[] = [];

for (const term of expression.children || []) {
if (term.kind == 'pipe') {
isOneOf = true;
log(2, 'Found pipe in expression');
_log(2, 'Found pipe in expression');
continue;
}

if (term.kind == 'expression_continue') {
log(2, 'Found expression_continue');
_log(2, 'Found expression_continue');
pattern.push(...processExpression(term, depth + 1));
isOneOf = true;
continue;
}

if (term.kind != 'term' && term.kind != 'term_continue') {
log(2, 'Invalid expression child');
_log(2, 'Invalid expression child');
continue;
}

log(2, `Parsing term at ${term.line}:${term.column}`);
_log(2, `Parsing term at ${term.line}:${term.column}`);
if (!term.children?.length) {
log(2, 'Term has no children');
_log(2, 'Term has no children');
continue;
}
for (let factor of term.children) {
Expand All @@ -237,7 +229,7 @@ export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefi
}
const node = factor.children?.[0] ?? factor;

log(2, `Parsing ${node.kind} "${node.text}" at ${node.line}:${node.column}`);
_log(2, `Parsing ${node.kind} "${node.text}" at ${node.line}:${node.column}`);
switch (node.kind) {
case 'string': {
const text = node.text.replace(/^"|"$/g, ''); // Strip quotes
Expand All @@ -253,7 +245,7 @@ export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefi
case 'left_paren': {
const inner = factor.children?.find(({ kind }) => kind == 'expression');
if (!inner) {
log(1, 'Missing inner expression');
_log(1, 'Missing inner expression');
break;
}

Expand Down Expand Up @@ -282,7 +274,7 @@ export function convert(ast: Node, verbose: number = 0): { definitions: NodeDefi
break;
}
default:
log(1, `Unexpected kind "${node.kind}" of factor child`);
_log(1, `Unexpected kind "${node.kind}" of factor child`);
break;
}
}
Expand Down
18 changes: 13 additions & 5 deletions src/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ export function stringifyNode(node: Node, depth = 0): string {
);
}

export type Logger = (verbosity: number, message: string, depth: number) => void;

export interface ParseOptionsShared {
definitions: NodeDefinition[];
rootNode: string;
ignoreLiterals: string[];
log?: Logger;
debug?(message: string): void;
verbose?(message: string): void;
}
Expand All @@ -50,8 +53,8 @@ export function parse(options: ParseOptions): Node[] {
const literals = 'tokens' in options ? options.literals : [...options.literals].map(literal => literal.name);

function parseNode(kind: string, depth = 0): Node | null {
const debug = (message: string): void => options?.debug?.(' '.repeat(depth) + message);
const verbose = (message: string): void => options?.verbose?.(' '.repeat(depth + 1) + 'verbose: ' + message);
const debug = (message: string): void => options.log?.(1, message, depth);
const verbose = (message: string): void => options.log?.(2, message, depth);
if (literals.includes(kind)) {
while (options.ignoreLiterals.includes(tokens[position]?.kind)) {
position++;
Expand Down Expand Up @@ -96,7 +99,13 @@ export function parse(options: ParseOptions): Node[] {
}
case 'sequence': {
const children: Node[] = [];

while (options.ignoreLiterals.includes(tokens[position]?.kind)) {
position++;
}

const start = position;
const token = tokens[position];

debug(`Parsing sequence "${kind}"`);
for (const part of pattern) {
Expand All @@ -120,10 +129,9 @@ export function parse(options: ParseOptions): Node[] {
}
}

const token = tokens[start];
debug(`"${kind}" at ${token.line}:${token.column}`);
debug(`"${token.kind}" at ${token.line}:${token.column}`);
return {
kind: definition.name,
kind,
text: token.text,
position: token.position,
line: token.line,
Expand Down
2 changes: 1 addition & 1 deletion test/asm.bnf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ operand_list = operand, {",", operand};

instruction = identifier, [operand_list];

instruction_list = (instruction | comment), {line_terminator, instruction};
instruction_list = (instruction | comment), {line_terminator, instruction | comment};
16 changes: 11 additions & 5 deletions test/asm.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
import { readFileSync, writeFileSync } from 'node:fs';
import { parseArgs } from 'node:util';
import { parse, stringifyNode } from '../src/parser';
import type { NodeDefinition } from '../src/parser';
import type { TokenDefinition } from '../src/tokens';
import { parse, stringifyNode } from '../src/parser';
import { tokenize } from '../src/tokens';

const {
positionals: [input],
values: options,
} = parseArgs({
options: {
verbose: { type: 'boolean', short: 'w', default: false },
verbose: { short: 'V', type: 'boolean', multiple: true },
json: { type: 'string', short: 'j' },
},
allowPositionals: true,
Expand All @@ -20,6 +20,8 @@ if (!input) {
process.exit(1);
}

const verbosity = options.verbose?.filter(Boolean)?.length ?? 0;

let literals = [
{ name: 'register', pattern: /^%\w+/ },
{ name: 'immediate', pattern: /^\$(0x)?\d+/ },
Expand Down Expand Up @@ -65,10 +67,14 @@ try {
const ast = parse({
source: readFileSync(input, 'utf8'),
literals,
ignoreLiterals: ['whitespace'],
ignoreLiterals: ['whitespace', 'line_terminator', 'comment'],
definitions,
rootNode: 'instruction_list',
debug: options.verbose ? console.debug : undefined,
log(level: number, message: string, depth: number) {
if (level > verbosity) return;

console.log(' '.repeat(4 * depth) + (level > 1 ? '[debug] ' : '') + message);
},
});

console.log('AST:\n');
Expand Down
3 changes: 2 additions & 1 deletion test/test.S
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Test assembly code
mov $100, %eax
mul $2, %eax
mov %eax,0x1000
nop
nop # do nothing

0 comments on commit 3b3a6e8

Please sign in to comment.