Add directives

package.json

@@ -7,7 +7,8 @@
 	"types": "dist/index.d.ts",
 	"keywords": [],
 	"bin": {
-		"xcompile-bnf": "scripts/bnf.js"
+		"xcompile-bnf": "scripts/bnf.js",
+		"xcompile-dump-ast": "scripts/ast.js"
 	},
 	"files": [
 		"dist",

scripts/ast.js

@@ -1,6 +1,8 @@
+#!/usr/bin/env node
 import { readFileSync } from 'node:fs';
 import { parseArgs } from 'node:util';
 import { parse, stringifyNode } from '../dist/parser.js';
+import { parseJSON as parseJSONConfig } from '../dist/config.js';
 
 const {
 	values: options,
@@ -15,7 +17,7 @@ const {
 });
 
 if (options.help || !input) {
-	console.log(`Usage: ${process.argv[0]} [options] <file>
+	console.log(`Usage: xcompile-dump-ast [options] <file>
 Options:
     --config,-c <path>  Specify the config file 
     --verbose,-V        Show verbose output. If passed multiple times, increases the verbosity level
@@ -25,14 +27,10 @@ Options:
 
 const verbosity = options.verbose?.filter(Boolean)?.length ?? 0;
 
-let config = {};
+let config;
 
 try {
-	const json = JSON.parse(readFileSync(options.json, 'utf-8'));
-
-	literals = json.literals.map(({ name, pattern }) => ({ name, pattern: new RegExp('^' + pattern) }));
-
-	definitions = json.definitions;
+	config = parseJSONConfig(JSON.parse(readFileSync(options.config, 'utf-8')));
 } catch (e) {
 	if ('errno' in e) console.error(e);
 	else console.error('Failed to resolve config:', e);
@@ -43,10 +41,6 @@ try {
 const ast = parse({
 	...config,
 	source: readFileSync(input, 'utf8'),
-	literals,
-	ignoreLiterals: ['whitespace', 'line_terminator', 'comment'],
-	definitions,
-	rootNode: 'instruction_list',
 	log(level, message, depth) {
 		if (verbosity < level) return;
 

scripts/bnf.js

@@ -21,7 +21,7 @@ const {
 });
 
 if (options.help || !input) {
-	console.log(`Usage: ${process.argv[0]} [options] <file>\n
+	console.log(`Usage: xcompile-bnf [options] <file>\n
 Output options:
     --format,-f <format=js>  Output format (js, json)
     --output,-o <path>       Path to an output file
@@ -54,7 +54,15 @@ function logger(outputLevel) {
 
 const verbose = options.verbose?.filter(Boolean)?.length ?? 0;
 
-const tokens = bnf.tokenize(readFileSync(input, 'utf8'));
+let contents;
+try {
+	contents = readFileSync(input, 'utf8');
+} catch (e) {
+	console.error(e.message);
+	process.exit(1);
+}
+
+const tokens = bnf.tokenize(contents);
 
 if (options.tokens) {
 	for (const token of tokens) {
@@ -72,7 +80,7 @@ if (options.ast) {
 	if (options.parser == 'only') process.exit(0);
 }
 
-const config = bnf.ast_to_config(ast[0], logger(verbose));
+const config = bnf.ast_to_config(ast, logger(verbose));
 
 const write = data => (options.output ? writeFileSync(options.output, data) : console.log);
 

src/bnf.bnf

@@ -3,9 +3,11 @@ string          = '("(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\')';
 identifier      = '[a-zA-Z_]\w*';
 whitespace      = '[ \t]+';
 line_terminator	= "[\n;]+";
-comment         = "#.*";
+comment         = "#[^\\n]*";
 directive       = "##\w+ .*";
 
+##ignore whitespace comment
+
 parenthesized = '\(', expression, '\)';
 optional = '\[', expression, '\]';
 repetition = '\{', expression, '\}';
@@ -19,3 +21,5 @@ expression = term, {('\|' | ','), term};
 rule = identifier, '=', expression, ';';
 
 rule_list = (rule | comment | directive), {line_terminator, rule | comment | directive};
+
+##root rule_list

src/bnf.json

@@ -14,8 +14,8 @@
 		{ "name": "right_bracket", "pattern": "\\]" },
 		{ "name": "whitespace", "pattern": "[ \\t]+" },
 		{ "name": "line_terminator", "pattern": "[\\n;]+" },
-		{ "name": "directive", "pattern": "##[^\\n]+" },
-		{ "name": "comment", "pattern": "#[^\\n]+" }
+		{ "name": "directive", "pattern": "##\\w+ [^\\n]*" },
+		{ "name": "comment", "pattern": "#[^\\n]*" }
 	],
 	"ignoreLiterals": ["whitespace", "comment"],
 	"definitions": [

src/bnf.ts

@@ -32,17 +32,38 @@ const typeForGroup = {
 	left_paren: 'required',
 } as const;
 
-export function ast_to_config(ast: Node, log: Logger = () => {}): { definitions: NodeDefinition[]; literals: TokenDefinition[] } {
-	const definitions: NodeDefinition[] = [];
-	const literals: TokenDefinition[] = [];
+export function ast_to_config(ast: Node[], log: Logger = () => {}): config.Config {
+	const definitions: NodeDefinition[] = [],
+		literals: TokenDefinition[] = [],
+		ignoreLiterals: string[] = [];
 
 	let isOneOf = false,
 		currentNode: string,
+		rootNode: string | undefined,
 		groups = 0;
 
 	function processNode(node: Node, depth: number = 0) {
 		const _log = (level: number, text: string) => log(level, text, depth);
 		_log(3, `Processing ${node.kind} at ${node.line}:${node.column}`);
+
+		if (node.kind == 'directive') {
+			const [, directive, contents] = node.text.match(/##(\w+) (.*)/i)!;
+
+			switch (directive) {
+				case 'root':
+					if (rootNode) _log(0, `Warning: overwriting root node ("${rootNode}" -> "${contents}")`);
+					rootNode = contents;
+					break;
+				case 'ignore':
+					ignoreLiterals.push(...contents.split(/[ ,;]/));
+					break;
+				default:
+					_log(0, 'Warning: unsupported directive: ' + directive);
+			}
+
+			return;
+		}
+
 		if (node.kind != 'rule') {
 			// Recursively process child nodes
 			for (const child of node.children || []) {
@@ -184,7 +205,13 @@ export function ast_to_config(ast: Node, log: Logger = () => {}): { definitions:
 	}
 
 	// Start processing from the root node
-	processNode(ast);
+	for (const node of ast) {
+		processNode(node);
+	}
+
+	if (!rootNode) {
+		throw 'Missing root node';
+	}
 
-	return { definitions, literals };
+	return { definitions, literals, rootNode, ignoreLiterals };
 }

src/config.ts

@@ -8,7 +8,7 @@ export interface Json {
 	ignoreLiterals: string[];
 }
 
-export interface Parsed {
+export interface Config {
 	literals: TokenDefinition[];
 	definitions: NodeDefinition[];
 	rootNode: string;
@@ -18,7 +18,7 @@ export interface Parsed {
 /**
  * Parses a JSON configuration into a normal configuration
  */
-export function parseJSON(config: Json): Parsed {
+export function parseJSON(config: Json): Config {
 	return {
 		...config,
 		literals: config.literals.map(literal => ({ name: literal.name, pattern: new RegExp('^' + literal.pattern) })),

test/asm.bnf

@@ -8,6 +8,8 @@ whitespace		= "[ \t]+";
 line_terminator	= "[\n;]+";
 comment			= "#.*";
 
+##ignore whitespace line_terminator comment
+
 operand = register | immediate | number;
 
 operand_list = operand, {",", operand};
@@ -16,5 +18,4 @@ instruction	= identifier, [operand_list];
 
 instruction_list = (instruction | comment), {line_terminator, instruction | comment};
 
-#ignore whitespace line_terminator comment
-#root instruction_list
+##root instruction_list