fix trailing IdentifierPart in grammar (NomicFoundation#641)

- Moved numeric literals to use `notFollowedBy: IdentifierStart` instead of `IdentifierPart`. - Removed `notFollowedBy: IdentifierPart` from string literals, to match the behavior of `solc`.
Xanewok · Nov 6, 2023 · a59a464 · a59a464
1 parent 36be89c
commit a59a464
Show file tree

Hide file tree

Showing 8 changed files with 161 additions and 184 deletions.
diff --git a/crates/solidity/inputs/language/definition/05-expressions/04-numbers/productions.yml b/crates/solidity/inputs/language/definition/05-expressions/04-numbers/productions.yml
@@ -37,7 +37,7 @@
                   - oneOrMore:
                       reference: "HexCharacter"
         notFollowedBy:
-          reference: "IdentifierPart"
+          reference: "IdentifierStart"
     0.5.0:
       # removed uppercase "0X"
       trailingContext:
@@ -52,7 +52,7 @@
                   - oneOrMore:
                       reference: "HexCharacter"
         notFollowedBy:
-          reference: "IdentifierPart"
+          reference: "IdentifierStart"
 
 - name: "DecimalLiteral"
   kind: "Scanner"
@@ -75,7 +75,7 @@
             - optional:
                 reference: "DecimalExponent"
         notFollowedBy:
-          reference: "IdentifierPart"
+          reference: "IdentifierStart"
     0.5.0:
       # Second "DecimalDigits" is no longer "optional"
       trailingContext:
@@ -94,7 +94,7 @@
             - optional:
                 reference: "DecimalExponent"
         notFollowedBy:
-          reference: "IdentifierPart"
+          reference: "IdentifierStart"
 
 - name: "DecimalDigits"
   kind: "Scanner"

diff --git a/crates/solidity/inputs/language/definition/05-expressions/05-strings/productions.yml b/crates/solidity/inputs/language/definition/05-expressions/05-strings/productions.yml
@@ -24,13 +24,9 @@
 - name: "HexStringLiteral"
   kind: "Scanner"
   unversioned:
-    trailingContext:
-      scanner:
-        choice:
-          - reference: "SingleQuotedHexStringLiteral"
-          - reference: "DoubleQuotedHexStringLiteral"
-      notFollowedBy:
-        reference: "IdentifierPart"
+    choice:
+      - reference: "SingleQuotedHexStringLiteral"
+      - reference: "DoubleQuotedHexStringLiteral"
 
 - name: "SingleQuotedHexStringLiteral"
   kind: "Scanner"
@@ -92,13 +88,9 @@
 - name: "AsciiStringLiteral"
   kind: "Scanner"
   unversioned:
-    trailingContext:
-      scanner:
-        choice:
-          - reference: "SingleQuotedAsciiStringLiteral"
-          - reference: "DoubleQuotedAsciiStringLiteral"
-      notFollowedBy:
-        reference: "IdentifierPart"
+    choice:
+      - reference: "SingleQuotedAsciiStringLiteral"
+      - reference: "DoubleQuotedAsciiStringLiteral"
 
 - name: "SingleQuotedAsciiStringLiteral"
   kind: "Scanner"
@@ -151,13 +143,9 @@
   kind: "Scanner"
   versioned:
     0.7.0:
-      trailingContext:
-        scanner:
-          choice:
-            - reference: "SingleQuotedUnicodeStringLiteral"
-            - reference: "DoubleQuotedUnicodeStringLiteral"
-        notFollowedBy:
-          reference: "IdentifierPart"
+      choice:
+        - reference: "SingleQuotedUnicodeStringLiteral"
+        - reference: "DoubleQuotedUnicodeStringLiteral"
 
 - name: "SingleQuotedUnicodeStringLiteral"
   kind: "Scanner"

diff --git a/crates/solidity/inputs/language/definition/06-yul/03-yul-expressions/productions.yml b/crates/solidity/inputs/language/definition/06-yul/03-yul-expressions/productions.yml
@@ -135,7 +135,7 @@
           - oneOrMore:
               reference: "HexCharacter"
       notFollowedBy:
-        reference: "IdentifierPart"
+        reference: "IdentifierStart"
 
 - name: "YulDecimalLiteral"
   kind: "Scanner"
@@ -153,4 +153,4 @@
                     from: "0"
                     to: "9"
       notFollowedBy:
-        reference: "IdentifierPart"
+        reference: "IdentifierStart"
diff --git a/crates/solidity/inputs/language/src/definition.rs b/crates/solidity/inputs/language/src/definition.rs
diff --git a/crates/solidity/inputs/language/src/dsl.rs b/crates/solidity/inputs/language/src/dsl.rs
@@ -796,30 +796,21 @@ slang_grammar! {
 
     // Ascii String Literals
 
-    scanner AsciiStringLiteral = (
-        (SingleQuotedAsciiStringLiteral | DoubleQuotedAsciiStringLiteral)
-        not followed by IdentifierStart
-    ) ;
+    scanner AsciiStringLiteral = (SingleQuotedAsciiStringLiteral | DoubleQuotedAsciiStringLiteral) ;
     scanner DoubleQuotedAsciiStringLiteral = ("\"" ((EscapeSequence | AsciiCharacterWithoutDoubleQuoteOrBackslash) *) "\"") ;
     scanner SingleQuotedAsciiStringLiteral = ("\'" ((EscapeSequence | AsciiCharacterWithoutSingleQuoteOrBackslash) *) "\'") ;
 
     // Hex String Literals
 
-    scanner HexStringLiteral = (
-        (SingleQuotedHexStringLiteral | DoubleQuotedHexStringLiteral)
-        not followed by IdentifierStart
-    ) ;
+    scanner HexStringLiteral = (SingleQuotedHexStringLiteral | DoubleQuotedHexStringLiteral) ;
     scanner DoubleQuotedHexStringLiteral = ("hex\"" (HexStringContents ?) "\"") ;
     scanner SingleQuotedHexStringLiteral = ("hex\'" (HexStringContents ?) "\'") ;
     scanner HexStringContents = (HexCharacter HexCharacter ((('_' ?) HexCharacter HexCharacter) *)) ;
 
     // Unicode String Literals
 
     scanner UnicodeStringLiteral = {
-        introduced in "0.7.0" (
-            (SingleQuotedUnicodeStringLiteral | DoubleQuotedUnicodeStringLiteral)
-            not followed by IdentifierStart
-        )
+        introduced in "0.7.0" (SingleQuotedUnicodeStringLiteral | DoubleQuotedUnicodeStringLiteral)
     } ;
     scanner DoubleQuotedUnicodeStringLiteral = { introduced in "0.7.0" ("unicode\"" ((EscapeSequence | (! "\n\r\"\\")) *) "\"") } ;
     scanner SingleQuotedUnicodeStringLiteral = { introduced in "0.7.0" ("unicode\'" ((EscapeSequence | (! "\n\r\'\\")) *) "\'") } ;

diff --git a/crates/solidity/outputs/cargo/crate/src/generated/language.rs b/crates/solidity/outputs/cargo/crate/src/generated/language.rs
diff --git a/crates/solidity/outputs/cargo/tests/src/scanner/mod.rs b/crates/solidity/outputs/cargo/tests/src/scanner/mod.rs
@@ -18,15 +18,15 @@ fn test_next_token() {
         ("1", DecimalLiteral),
         ("\n", EndOfLine),
         ("unicode'abc'", UnicodeStringLiteral),
-        ("unicode'abc'ZZ", Identifier), // TODO: This needs to be further checked against solc
         ("hex'abcd'", HexStringLiteral),
-        ("hex'abcd'ZZz", HexKeyword), // TODO: This needs to be further checked against solc
+        ("'abc'ZZ", AsciiStringLiteral), // with an identifier afterwards
+        ("unicode'abc'ZZ", UnicodeStringLiteral), // with an identifier afterwards
+        ("hex'abcd'ZZz", HexStringLiteral), // with an identifier afterwards
         ("// single line\n", SingleLineComment),
         ("/* multi-line\n   comment */ blah", MultilineComment),
         ("/* multi-line comment **/ blah", MultilineComment),
         ("0ZZ", SKIPPED),
         ("0xabZZ", SKIPPED),
-        ("'abc'ZZ", SKIPPED),
     ] {
         assert_eq!(language.scan(LexicalContext::Default, s), Some(*k));
     }

diff --git a/crates/solidity/outputs/npm/crate/src/generated/language.rs b/crates/solidity/outputs/npm/crate/src/generated/language.rs