add support for i and iq{} tokens

adamdruppe · Nov 14, 2023 · 52c960e · 52c960e
1 parent c6c4ec7
commit 52c960e
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 26 deletions.
diff --git a/compiler/src/dmd/lexer.d b/compiler/src/dmd/lexer.d
@@ -511,8 +511,20 @@ class Lexer
                     goto case_ident;
                 if (p[1] == '"')
                 {
-                    p++;
-                    interpolatedStringConstant(t);
+                    p++; // skip the i
+                    escapeStringConstant(t, true);
+                    return;
+                }
+                else if (p[1] == '`')
+                {
+                    p++; // skip the i
+                    wysiwygStringConstant(t, true);
+                    return;
+                }
+                else if (p[1] == 'q' && p[2] == '{')
+                {
+                    p += 2; // skip the i and q
+                    tokenStringConstant(t, true);
                     return;
                 }
                 else
@@ -1440,9 +1452,18 @@ class Lexer
     Params:
         result = pointer to the token that accepts the result
     */
-    private void wysiwygStringConstant(Token* result)
+    private void wysiwygStringConstant(Token* result, bool supportInterpolation = false)
     {
-        result.value = TOK.string_;
+        if (supportInterpolation)
+        {
+            result.value = TOK.interpolated;
+            result.interpolatedSet = null;
+        }
+        else
+        {
+            result.value = TOK.string_;
+        }
+
         Loc start = loc();
         auto terminator = p[0];
         p++;
@@ -1462,6 +1483,14 @@ class Lexer
                 c = '\n'; // treat EndOfLine as \n character
                 endOfLine();
                 break;
+            case '$':
+                if (!supportInterpolation)
+                    goto default;
+
+                if (!handleInterpolatedSegment(result, start))
+                    goto default;
+
+                continue;
             case 0:
             case 0x1A:
                 error("unterminated string constant starting at %s", start.toChars());
@@ -1472,7 +1501,11 @@ class Lexer
             default:
                 if (c == terminator)
                 {
-                    result.setString(stringbuffer);
+                    if (supportInterpolation)
+                        result.appendInterpolatedPart(stringbuffer);
+                    else
+                        result.setString(stringbuffer);
+
                     stringPostfix(result);
                     return;
                 }
@@ -1747,13 +1780,21 @@ class Lexer
     Params:
         result = pointer to the token that accepts the result
     */
-    private void tokenStringConstant(Token* result)
+    private void tokenStringConstant(Token* result, bool supportInterpolation = false)
     {
-        result.value = TOK.string_;
+        if (supportInterpolation)
+        {
+            result.value = TOK.interpolated;
+            result.interpolatedSet = null;
+        }
+        else
+        {
+            result.value = TOK.string_;
+        }
 
         uint nest = 1;
         const start = loc();
-        const pstart = ++p;
+        auto pstart = ++p;
         inTokenStringConstant++;
         scope(exit) inTokenStringConstant--;
         while (1)
@@ -1768,10 +1809,28 @@ class Lexer
             case TOK.rightCurly:
                 if (--nest == 0)
                 {
-                    result.setString(pstart, p - 1 - pstart);
+                    if (supportInterpolation)
+                        result.appendInterpolatedPart(pstart, p - 1 - pstart);
+                    else
+                        result.setString(pstart, p - 1 - pstart);
+
                     stringPostfix(result);
                     return;
                 }
+                continue;
+            case TOK.dollar:
+                if (!supportInterpolation)
+                    goto default;
+
+                stringbuffer.setsize(0);
+                stringbuffer.write(pstart, p - 1 - pstart);
+                if (!handleInterpolatedSegment(result, start))
+                    goto default;
+
+                stringbuffer.setsize(0);
+
+                pstart = p;
+
                 continue;
             case TOK.endOfFile:
                 error("unterminated token string constant starting at %s", start.toChars());
@@ -1783,11 +1842,6 @@ class Lexer
         }
     }
 
-    private void interpolatedStringConstant(Token* result)
-    {
-        escapeStringConstant(result, true);
-    }
-
     // returns true if it got special treatment as an interpolated segment
     // otherwise returns false, indicating to treat it as just part of a normal string
     private bool handleInterpolatedSegment(Token* token, Loc start)

diff --git a/compiler/test/runnable/interpolatedexpressionsequence.d b/compiler/test/runnable/interpolatedexpressionsequence.d
@@ -2,6 +2,22 @@ import core.interpolation;
 
 alias AliasSeq(T...) = T;
 
+string simpleToString(T...)(T thing) {
+    string s;
+    foreach(item; thing)
+        // all the items provided by core.interpolation have
+        // toString to return an appropriate value
+        //
+        // then this particular example only has embedded strings
+        // and chars, to we can append them directly
+        static if(__traits(hasMember, item, "toString"))
+            s ~= item.toString();
+        else
+            s ~= item;
+
+    return s;
+}
+
 void main() {
 	int a = 1;
 	string b = "one";
@@ -23,16 +39,11 @@ void main() {
 	// you can embed any D expressions inside the parenthesis, and the
 	// token is not ended until you get the *outer* ) and ".
 	auto thing = i"$b $("$" ~ ')' ~ `"`)";
-	string s;
-	foreach(item; thing)
-		// all the items provided by core.interpolation have
-		// toString to return an appropriate value
-		//
-		// then this particular example only has embedded strings
-		// and chars, to we can append them directly
-		static if(__traits(hasMember, item, "toString"))
-			s ~= item.toString();
-		else
-			s ~= item;
-	assert(s == "one $)\"");
+	assert(simpleToString(thing) == "one $)\"");
+
+        // i`` and iq{} should also work
+        assert(simpleToString(i` $b is $(b)!`) == " one is one!");
+        assert(simpleToString(iq{ $b is $(b)!}) == " one is one!");
+        assert(simpleToString(i`\$('$')`) == "\\$"); // no \ escape there
+        assert(simpleToString(iq{{$('$')}}) == "{$}"); // {} needs to work
 }