From f7d7afd57b88750d8085e006a0c389f8041aad8f Mon Sep 17 00:00:00 2001 From: Tommy Yu Date: Sun, 28 Jul 2019 11:52:00 -0700 Subject: [PATCH] Initial implementation of TemplateLiteral rules - The parser can now parse template literals from tokens produced by the lexer and produce the ast with the new asttypes for templates. - Initial set of test cases included, however there are certain limitations which have been found that needed correction in the lexer itself. --- src/calmjs/parse/asttypes.py | 37 ++++ src/calmjs/parse/parsers/es2015.py | 58 +++++- src/calmjs/parse/tests/parser.py | 185 +++++++++++++++++++ src/calmjs/parse/tests/test_es2015_parser.py | 8 + 4 files changed, 287 insertions(+), 1 deletion(-) diff --git a/src/calmjs/parse/asttypes.py b/src/calmjs/parse/asttypes.py index 5cf27e0..ae76025 100644 --- a/src/calmjs/parse/asttypes.py +++ b/src/calmjs/parse/asttypes.py @@ -153,6 +153,43 @@ def __init__(self, value): self.value = value +class Template(Node): + """ + All template subclasses. + """ + + +class TemplateLiteral(Template): + """ + The top level template literal object + """ + + +class TemplateFragment(Template): + """ + All template fragments + """ + + def __init__(self, value): + self.value = value + + +class TemplateNoSub(TemplateFragment): + pass + + +class TemplateHead(TemplateFragment): + pass + + +class TemplateMiddle(TemplateFragment): + pass + + +class TemplateTail(TemplateFragment): + pass + + class Regex(Node): def __init__(self, value): self.value = value diff --git a/src/calmjs/parse/parsers/es2015.py b/src/calmjs/parse/parsers/es2015.py index ad514c1..3503090 100644 --- a/src/calmjs/parse/parsers/es2015.py +++ b/src/calmjs/parse/parsers/es2015.py @@ -167,7 +167,6 @@ def p_literal(self, p): | boolean_literal | numeric_literal | string_literal - | regex_literal """ p[0] = p[1] @@ -198,6 +197,61 @@ def p_regex_literal(self, p): p[0] = self.asttypes.Regex(p[1]) p[0].setpos(p) + # 11.8.6 Template + def p_template_nosub(self, p): + """template_nosub : TEMPLATE_NOSUB""" + # no_sub_template is called as such here for consistency + p[0] = self.asttypes.TemplateNoSub(p[1]) + p[0].setpos(p) + + def p_template_head(self, p): + """template_head : TEMPLATE_HEAD""" + p[0] = self.asttypes.TemplateHead(p[1]) + p[0].setpos(p) + + def p_template_middle(self, p): + """template_middle : TEMPLATE_MIDDLE""" + p[0] = self.asttypes.TemplateMiddle(p[1]) + p[0].setpos(p) + + def p_template_tail(self, p): + """template_tail : TEMPLATE_TAIL""" + p[0] = self.asttypes.TemplateTail(p[1]) + p[0].setpos(p) + + def p_template_literal(self, p): + """template_literal : template_nosub + | template_head expr template_spans + """ + literals = [p[1]] + if len(p) > 2: + # append the expression and extend with template spans + literals.append(p[2]) + literals.extend(p[3]) + p[0] = self.asttypes.TemplateLiteral(literals) + p[0].setpos(p) + + def p_template_spans(self, p): + """template_spans : template_tail + | template_middle_list template_tail + """ + if len(p) == 2: + p[0] = [p[1]] + else: + p[1].append(p[2]) + p[0] = p[1] + + def p_template_middle_list(self, p): + """template_middle_list : template_middle expr + | template_middle_list template_middle \ + expr + """ + if len(p) == 3: + p[0] = [p[1], p[2]] + else: + p[1].extend([p[2], p[3]]) + p[0] = p[1] + def p_identifier(self, p): """identifier : ID""" p[0] = self.asttypes.Identifier(p[1]) @@ -276,6 +330,8 @@ def p_primary_expr_no_brace_2(self, p): def p_primary_expr_no_brace_3(self, p): """primary_expr_no_brace : literal | array_literal + | regex_literal + | template_literal """ p[0] = p[1] diff --git a/src/calmjs/parse/tests/parser.py b/src/calmjs/parse/tests/parser.py index 577b717..fa93c2a 100644 --- a/src/calmjs/parse/tests/parser.py +++ b/src/calmjs/parse/tests/parser.py @@ -2589,6 +2589,175 @@ def regenerate(value): )])) +def build_es2015_node_repr_test_cases(clsname, parse, program_type): + + def parse_to_repr(value): + return repr_walker.walk(parse(value), pos=True) + + return build_equality_testcase(clsname, parse_to_repr, (( + label, + textwrap.dedent(argument).strip(), + singleline(result).replace(', + initializer= + ]> + > + ]> + ]> + """, + ), ( + 'template_literal_sub', + """ + var t = `some_template${value}tail` + """, + """ + , + initializer=, + , + + ]> + > + ]> + ]> + """, + ), ( + 'template_literal_sub_once', + """ + var t = `some_template${value}middle${value}tail` + """, + """ + , + initializer=, + , + , + , + + ]> + > + ]> + ]> + """, + ), ( + 'template_literal_sub_multiple', + """ + var t = `some_template${value}middle${value}another${tail}tail` + """, + """ + , + initializer=, + , + , + , + , + , + + ]> + > + ]> + ]> + """, + ), ( + 'template_multiline_between_expressions', + """ + t = `tt${ + s + s + }ttttt` + """, + """ + , + op='=', + right=, + , + op='+', right=>, + + ]> + >> + ]> + """, + ), ( + 'template_with_regex', + """ + value = `regex is ${/wat/}` + """, + """ + , + op='=', + right=, + , + + ]> + >> + ]> + """, + ), ( + 'template_with_string', + """ + value = `string is ${'wat'}` + """, + """ + , + op='=', + right=, + , + + ]> + >> + ]> + """, + ), ( + 'template_in_template', + """ + value = `template embed ${`another${`template`}`} inside` + """, + """ + , + op='=', + right=, + , + + ]>, + + ]>, + + ]> + >> + ]> + """, + )])) + + def build_syntax_error_test_cases(clsname, parse): return build_exception_testcase(clsname, parse, (( label, @@ -2639,6 +2808,22 @@ def build_syntax_error_test_cases(clsname, parse): )]), ECMASyntaxError) +def build_es2015_syntax_error_test_cases(clsname, parse): + return build_exception_testcase(clsname, parse, (( + label, + textwrap.dedent(argument).strip(), + msg, + ) for label, argument, msg in [( + 'unexpected_if_in_template', + '`${if (wat)}`', + "Unexpected 'if' at 1:4 between '`${' at 1:1 and '(' at 1:7", + ), ( + 'empty_expression_in_template', + '`head${}tail`', + "Unexpected '}tail`' at 1:8 after '`head${' at 1:1", + )]), ECMASyntaxError) + + def build_regex_syntax_error_test_cases(clsname, parse): return build_exception_testcase(clsname, parse, (( label, diff --git a/src/calmjs/parse/tests/test_es2015_parser.py b/src/calmjs/parse/tests/test_es2015_parser.py index 003158d..d6f9c36 100644 --- a/src/calmjs/parse/tests/test_es2015_parser.py +++ b/src/calmjs/parse/tests/test_es2015_parser.py @@ -15,8 +15,10 @@ from calmjs.parse.tests.parser import ( ParserCaseMixin, build_node_repr_test_cases, + build_es2015_node_repr_test_cases, # build_asi_test_cases, build_syntax_error_test_cases, + build_es2015_syntax_error_test_cases, build_regex_syntax_error_test_cases, ) @@ -60,6 +62,9 @@ def test_read(self): ParsedNodeTypeTestCase = build_node_repr_test_cases( 'ParsedNodeTypeTestCase', parse, 'ES2015Program') +ParsedES2015NodeTypeTestCase = build_es2015_node_repr_test_cases( + 'ParsedES2015NodeTypeTestCase', parse, 'ES2015Program') + # ASI - Automatic Semicolon Insertion # ParserToECMAASITestCase = build_asi_test_cases( # 'ParserToECMAASITestCase', parse, pretty_print) @@ -67,5 +72,8 @@ def test_read(self): ECMASyntaxErrorsTestCase = build_syntax_error_test_cases( 'ECMASyntaxErrorsTestCase', parse) +ECMA2015SyntaxErrorsTestCase = build_es2015_syntax_error_test_cases( + 'ECMA2015SyntaxErrorsTestCase', parse) + ECMARegexSyntaxErrorsTestCase = build_regex_syntax_error_test_cases( 'ECMARegexSyntaxErrorsTestCase', parse)