Initial implementation of TemplateLiteral rules

- The parser can now parse template literals from tokens produced by the lexer and produce the ast with the new asttypes for templates. - Initial set of test cases included, however there are certain limitations which have been found that needed correction in the lexer itself.
calmjs · Jul 28, 2019 · f7d7afd · f7d7afd
1 parent 58f26eb
commit f7d7afd
Show file tree

Hide file tree

Showing 4 changed files with 287 additions and 1 deletion.
diff --git a/src/calmjs/parse/asttypes.py b/src/calmjs/parse/asttypes.py
@@ -153,6 +153,43 @@ def __init__(self, value):
         self.value = value
 
 
+class Template(Node):
+    """
+    All template subclasses.
+    """
+
+
+class TemplateLiteral(Template):
+    """
+    The top level template literal object
+    """
+
+
+class TemplateFragment(Template):
+    """
+    All template fragments
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+
+class TemplateNoSub(TemplateFragment):
+    pass
+
+
+class TemplateHead(TemplateFragment):
+    pass
+
+
+class TemplateMiddle(TemplateFragment):
+    pass
+
+
+class TemplateTail(TemplateFragment):
+    pass
+
+
 class Regex(Node):
     def __init__(self, value):
         self.value = value

diff --git a/src/calmjs/parse/parsers/es2015.py b/src/calmjs/parse/parsers/es2015.py
@@ -167,7 +167,6 @@ def p_literal(self, p):
                    | boolean_literal
                    | numeric_literal
                    | string_literal
-                   | regex_literal
         """
         p[0] = p[1]
 
@@ -198,6 +197,61 @@ def p_regex_literal(self, p):
         p[0] = self.asttypes.Regex(p[1])
         p[0].setpos(p)
 
+    # 11.8.6 Template
+    def p_template_nosub(self, p):
+        """template_nosub : TEMPLATE_NOSUB"""
+        # no_sub_template is called as such here for consistency
+        p[0] = self.asttypes.TemplateNoSub(p[1])
+        p[0].setpos(p)
+
+    def p_template_head(self, p):
+        """template_head : TEMPLATE_HEAD"""
+        p[0] = self.asttypes.TemplateHead(p[1])
+        p[0].setpos(p)
+
+    def p_template_middle(self, p):
+        """template_middle : TEMPLATE_MIDDLE"""
+        p[0] = self.asttypes.TemplateMiddle(p[1])
+        p[0].setpos(p)
+
+    def p_template_tail(self, p):
+        """template_tail : TEMPLATE_TAIL"""
+        p[0] = self.asttypes.TemplateTail(p[1])
+        p[0].setpos(p)
+
+    def p_template_literal(self, p):
+        """template_literal : template_nosub
+                            | template_head expr template_spans
+        """
+        literals = [p[1]]
+        if len(p) > 2:
+            # append the expression and extend with template spans
+            literals.append(p[2])
+            literals.extend(p[3])
+        p[0] = self.asttypes.TemplateLiteral(literals)
+        p[0].setpos(p)
+
+    def p_template_spans(self, p):
+        """template_spans : template_tail
+                          | template_middle_list template_tail
+        """
+        if len(p) == 2:
+            p[0] = [p[1]]
+        else:
+            p[1].append(p[2])
+            p[0] = p[1]
+
+    def p_template_middle_list(self, p):
+        """template_middle_list : template_middle expr
+                                | template_middle_list template_middle \
+                                  expr
+        """
+        if len(p) == 3:
+            p[0] = [p[1], p[2]]
+        else:
+            p[1].extend([p[2], p[3]])
+            p[0] = p[1]
+
     def p_identifier(self, p):
         """identifier : ID"""
         p[0] = self.asttypes.Identifier(p[1])
@@ -276,6 +330,8 @@ def p_primary_expr_no_brace_2(self, p):
     def p_primary_expr_no_brace_3(self, p):
         """primary_expr_no_brace : literal
                                  | array_literal
+                                 | regex_literal
+                                 | template_literal
         """
         p[0] = p[1]
 

diff --git a/src/calmjs/parse/tests/parser.py b/src/calmjs/parse/tests/parser.py
@@ -2589,6 +2589,175 @@ def regenerate(value):
     )]))
 
 
+def build_es2015_node_repr_test_cases(clsname, parse, program_type):
+
+    def parse_to_repr(value):
+        return repr_walker.walk(parse(value), pos=True)
+
+    return build_equality_testcase(clsname, parse_to_repr, ((
+        label,
+        textwrap.dedent(argument).strip(),
+        singleline(result).replace('<Program', '<' + program_type),
+    ) for label, argument, result in [(
+        'template_literal_nosub',
+        """
+        var t = `some_template`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <VarStatement @1:1 ?children=[
+            <VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
+              initializer=<TemplateLiteral @1:9 ?children=[
+                <TemplateNoSub @1:9 value='`some_template`'>
+              ]>
+            >
+          ]>
+        ]>
+        """,
+    ), (
+        'template_literal_sub',
+        """
+        var t = `some_template${value}tail`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <VarStatement @1:1 ?children=[
+            <VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
+              initializer=<TemplateLiteral @1:9 ?children=[
+                <TemplateHead @1:9 value='`some_template${'>,
+                <Identifier @1:25 value='value'>,
+                <TemplateTail @1:30 value='}tail`'>
+              ]>
+            >
+          ]>
+        ]>
+        """,
+    ), (
+        'template_literal_sub_once',
+        """
+        var t = `some_template${value}middle${value}tail`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <VarStatement @1:1 ?children=[
+            <VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
+              initializer=<TemplateLiteral @1:9 ?children=[
+                <TemplateHead @1:9 value='`some_template${'>,
+                <Identifier @1:25 value='value'>,
+                <TemplateMiddle @1:30 value='}middle${'>,
+                <Identifier @1:39 value='value'>,
+                <TemplateTail @1:44 value='}tail`'>
+              ]>
+            >
+          ]>
+        ]>
+        """,
+    ), (
+        'template_literal_sub_multiple',
+        """
+        var t = `some_template${value}middle${value}another${tail}tail`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <VarStatement @1:1 ?children=[
+            <VarDecl @1:5 identifier=<Identifier @1:5 value='t'>,
+              initializer=<TemplateLiteral @1:9 ?children=[
+                <TemplateHead @1:9 value='`some_template${'>,
+                <Identifier @1:25 value='value'>,
+                <TemplateMiddle @1:30 value='}middle${'>,
+                <Identifier @1:39 value='value'>,
+                <TemplateMiddle @1:44 value='}another${'>,
+                <Identifier @1:54 value='tail'>,
+                <TemplateTail @1:58 value='}tail`'>
+              ]>
+            >
+          ]>
+        ]>
+        """,
+    ), (
+        'template_multiline_between_expressions',
+        """
+        t = `tt${
+        s + s
+        }ttttt`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <ExprStatement @1:1 expr=<Assign @1:3 left=<
+              Identifier @1:1 value='t'>,
+            op='=',
+            right=<TemplateLiteral @1:5 ?children=[
+              <TemplateHead @1:5 value='`tt${'>,
+              <BinOp @2:3 left=<Identifier @2:1 value='s'>,
+                op='+', right=<Identifier @2:5 value='s'>>,
+              <TemplateTail @3:1 value='}ttttt`'>
+            ]>
+          >>
+        ]>
+        """,
+    ), (
+        'template_with_regex',
+        """
+        value = `regex is ${/wat/}`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <ExprStatement @1:1 expr=<Assign @1:7 left=<
+              Identifier @1:1 value='value'>,
+            op='=',
+            right=<TemplateLiteral @1:9 ?children=[
+              <TemplateHead @1:9 value='`regex is ${'>,
+              <Regex @1:21 value='/wat/'>,
+              <TemplateTail @1:26 value='}`'>
+            ]>
+          >>
+        ]>
+        """,
+    ), (
+        'template_with_string',
+        """
+        value = `string is ${'wat'}`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <ExprStatement @1:1 expr=<Assign @1:7 left=<
+              Identifier @1:1 value='value'>,
+            op='=',
+            right=<TemplateLiteral @1:9 ?children=[
+              <TemplateHead @1:9 value='`string is ${'>,
+              <String @1:22 value="'wat'">,
+              <TemplateTail @1:27 value='}`'>
+            ]>
+          >>
+        ]>
+        """,
+    ), (
+        'template_in_template',
+        """
+        value = `template embed ${`another${`template`}`} inside`
+        """,
+        """
+        <Program @1:1 ?children=[
+          <ExprStatement @1:1 expr=<Assign @1:7 left=<
+              Identifier @1:1 value='value'>,
+            op='=',
+            right=<TemplateLiteral @1:9 ?children=[
+              <TemplateHead @1:9 value='`template embed ${'>,
+              <TemplateLiteral @1:27 ?children=[
+                <TemplateHead @1:27 value='`another${'>,
+                <TemplateLiteral @1:37 ?children=[
+                  <TemplateNoSub @1:37 value='`template`'>
+                ]>,
+                <TemplateTail @1:47 value='}`'>
+              ]>,
+              <TemplateTail @1:49 value='} inside`'>
+            ]>
+          >>
+        ]>
+        """,
+    )]))
+
+
 def build_syntax_error_test_cases(clsname, parse):
     return build_exception_testcase(clsname, parse, ((
         label,
@@ -2639,6 +2808,22 @@ def build_syntax_error_test_cases(clsname, parse):
     )]), ECMASyntaxError)
 
 
+def build_es2015_syntax_error_test_cases(clsname, parse):
+    return build_exception_testcase(clsname, parse, ((
+        label,
+        textwrap.dedent(argument).strip(),
+        msg,
+    ) for label, argument, msg in [(
+        'unexpected_if_in_template',
+        '`${if (wat)}`',
+        "Unexpected 'if' at 1:4 between '`${' at 1:1 and '(' at 1:7",
+    ), (
+        'empty_expression_in_template',
+        '`head${}tail`',
+        "Unexpected '}tail`' at 1:8 after '`head${' at 1:1",
+    )]), ECMASyntaxError)
+
+
 def build_regex_syntax_error_test_cases(clsname, parse):
     return build_exception_testcase(clsname, parse, ((
         label,

diff --git a/src/calmjs/parse/tests/test_es2015_parser.py b/src/calmjs/parse/tests/test_es2015_parser.py
@@ -15,8 +15,10 @@
 from calmjs.parse.tests.parser import (
     ParserCaseMixin,
     build_node_repr_test_cases,
+    build_es2015_node_repr_test_cases,
     # build_asi_test_cases,
     build_syntax_error_test_cases,
+    build_es2015_syntax_error_test_cases,
     build_regex_syntax_error_test_cases,
 )
 
@@ -60,12 +62,18 @@ def test_read(self):
 ParsedNodeTypeTestCase = build_node_repr_test_cases(
     'ParsedNodeTypeTestCase', parse, 'ES2015Program')
 
+ParsedES2015NodeTypeTestCase = build_es2015_node_repr_test_cases(
+    'ParsedES2015NodeTypeTestCase', parse, 'ES2015Program')
+
 # ASI - Automatic Semicolon Insertion
 # ParserToECMAASITestCase = build_asi_test_cases(
 #     'ParserToECMAASITestCase', parse, pretty_print)
 
 ECMASyntaxErrorsTestCase = build_syntax_error_test_cases(
     'ECMASyntaxErrorsTestCase', parse)
 
+ECMA2015SyntaxErrorsTestCase = build_es2015_syntax_error_test_cases(
+    'ECMA2015SyntaxErrorsTestCase', parse)
+
 ECMARegexSyntaxErrorsTestCase = build_regex_syntax_error_test_cases(
     'ECMARegexSyntaxErrorsTestCase', parse)