BNF grammar now ignores new lines

Fixed BNF strings and comments
Fixed parser crashing when ending on an ignored token

test/asm.bnf → grammars/asm.bnf

@@ -10,7 +10,8 @@ comment         = "#.*";
 
 ##ignore whitespace line_terminator comment
 
-operand = register | immediate | number;
+operand = register
+| immediate | number;
 
 operand_list = operand, {",", operand};
 

src/bnf.bnf

@@ -1,9 +1,9 @@
 
-string          = '"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'';
+string          = '("(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\')';
 identifier      = '[a-zA-Z_]\w*';
 whitespace      = '[ \t]+';
-line_terminator	= "[\n;]+";
-comment         = "#[^\n]*";
+line_terminator	= "\n+";
+comment         = "#(?!#)([^\n]*)";
 directive       = "##\w+ [^\n]*";
 
 # Simplifies internals
@@ -16,7 +16,7 @@ right_bracket = '\]';
 left_brace = '\{';
 right_brace = '\}';
 
-##ignore whitespace comment
+##ignore whitespace comment line_terminator
 
 parenthesized = left_param, expression, right_param;
 optional = left_bracket, expression, right_bracket;
@@ -32,6 +32,6 @@ expression = term, {',', term};
 
 rule = identifier, '=', expression, ';';
 
-rule_list = (rule | comment | directive), {line_terminator, rule | comment | directive};
+rule_list = (rule | directive), {rule | directive};
 
-##root rule_list
+##root rule_list

src/bnf.json

@@ -92,17 +92,14 @@
 			"type": "oneof",
 			"pattern": [
 				{ "kind": "rule", "type": "required" },
-				{ "kind": "comment", "type": "required" },
 				{ "kind": "directive", "type": "required" }
 			]
 		},
 		{
 			"name": "rule_list#1",
 			"type": "oneof",
 			"pattern": [
-				{ "kind": "line_terminator", "type": "required" },
 				{ "kind": "rule", "type": "required" },
-				{ "kind": "comment", "type": "required" },
 				{ "kind": "directive", "type": "required" }
 			]
 		},
@@ -116,11 +113,11 @@
 		}
 	],
 	"literals": [
-		{ "name": "string", "pattern": "\"(?:\\\\.|[^\"\\\\])*\"|'(?:\\\\.|[^'\\\\])*'" },
+		{ "name": "string", "pattern": "(\"(?:\\\\.|[^\"\\\\])*\"|'(?:\\\\.|[^'\\\\])*')" },
 		{ "name": "identifier", "pattern": "[a-zA-Z_]\\w*" },
 		{ "name": "whitespace", "pattern": "[ \\t]+" },
-		{ "name": "line_terminator", "pattern": "[\\n;]+" },
-		{ "name": "comment", "pattern": "#[^\\n]*" },
+		{ "name": "line_terminator", "pattern": "\\n+" },
+		{ "name": "comment", "pattern": "#(?!#)([^\\n]*)" },
 		{ "name": "directive", "pattern": "##\\w+ [^\\n]*" },
 		{ "name": "pipe", "pattern": "\\|" },
 		{ "name": "left_param", "pattern": "\\(" },
@@ -134,5 +131,5 @@
 		{ "name": ";", "pattern": ";" }
 	],
 	"rootNode": "rule_list",
-	"ignoreLiterals": ["whitespace", "comment"]
+	"ignoreLiterals": ["whitespace", "comment", "line_terminator"]
 }

src/parser.ts

@@ -147,6 +147,7 @@ export function parse(options: ParseOptions): Node[] {
 	while (position < tokens.length) {
 		const node = parseNode(options.rootNode);
 		if (!node) {
+			if (position >= tokens.length && options.ignoreLiterals.includes(tokens.at(-1)!.kind)) break;
 			const token = tokens[dirtyPosition || position];
 			throw new Error(`Unexpected token "${token.text}" at line ${token.line}, column ${token.column}`);
 		}