From ba72e6123d3b396e27c7079aaaa35f29160f44d0 Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:05:36 -0600 Subject: [PATCH 1/7] Support `@(XYZ)` raw strings --- .../Expression/String/raw2.dm | 0 .../DMPreprocessor/DMPreprocessorLexer.cs | 70 ++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) rename Content.Tests/DMProject/{Broken Tests => Tests}/Expression/String/raw2.dm (100%) diff --git a/Content.Tests/DMProject/Broken Tests/Expression/String/raw2.dm b/Content.Tests/DMProject/Tests/Expression/String/raw2.dm similarity index 100% rename from Content.Tests/DMProject/Broken Tests/Expression/String/raw2.dm rename to Content.Tests/DMProject/Tests/Expression/String/raw2.dm diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index 83a3a3a403..154b9cb3b1 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -301,12 +301,31 @@ public Token NextToken(bool ignoreWhitespace = false) { } case '@': { //Raw string char delimiter = Advance(); + var startLoc = CurrentLocation(); + + // @(XYZ) where XYZ is the delimiter + string complexDelimiter = string.Empty; + if (delimiter == '(') { + Advance(); + while (GetCurrent() != ')') { + if (AtEndOfSource()) { + _compiler.Emit(WarningCode.BadExpression, startLoc, + "Unterminated string delimiter"); + break; + } + + complexDelimiter += GetCurrent(); + Advance(); + } + } TokenTextBuilder.Clear(); TokenTextBuilder.Append('@'); TokenTextBuilder.Append(delimiter); + bool isComplex = complexDelimiter != string.Empty; bool isLong = false; + c = Advance(); if (delimiter == '{') { TokenTextBuilder.Append(c); @@ -314,7 +333,33 @@ public Token NextToken(bool ignoreWhitespace = false) { if (c == '"') isLong = true; } - if (isLong) { + if (isComplex) { + TokenTextBuilder.Append(complexDelimiter); + TokenTextBuilder.Append(')'); + + // Ignore a newline immediately after @(complexDelimiter) + if (c == '\r') c = Advance(); + if (c == '\n') c = Advance(); + + var delimIdx = 0; + do { + TokenTextBuilder.Append(c); + if (GetCurrent() == complexDelimiter[delimIdx]) delimIdx++; + if (delimIdx + 1 == complexDelimiter.Length) { + TokenTextBuilder.Remove(TokenTextBuilder.Length - delimIdx, complexDelimiter.Length - 1); + break; + } + + c = Advance(); + } while (!AtEndOfSource()); + + if (AtEndOfSource()) { + _compiler.Emit(WarningCode.BadExpression, startLoc, + "Unterminated string delimiter"); + } + + TokenTextBuilder.Append(complexDelimiter); + } else if (isLong) { bool nextCharCanTerm = false; Advance(); @@ -335,6 +380,11 @@ public Token NextToken(bool ignoreWhitespace = false) { if (c == '"') nextCharCanTerm = true; } while (!AtEndOfSource()); + + if (AtEndOfSource()) { + _compiler.Emit(WarningCode.BadExpression, startLoc, + "Unterminated string delimiter"); + } } else { while (c != delimiter && !AtLineEnd() && !AtEndOfSource()) { TokenTextBuilder.Append(c); @@ -342,14 +392,24 @@ public Token NextToken(bool ignoreWhitespace = false) { } } - TokenTextBuilder.Append(c); + if (isComplex) Advance(); + else TokenTextBuilder.Append(c); + if (!HandleLineEnd()) Advance(); string text = TokenTextBuilder.ToString(); string value; - if (isLong) { + if (isComplex) { + // Complex strings need to strip @(complexDelimiter) and a potential final newline. Newline after @(complexDelimiter) is already handled + var trimEnd = complexDelimiter.Length; + if (TokenTextBuilder[^(complexDelimiter.Length + 1)] == '\n') trimEnd += 1; + if (TokenTextBuilder[^(complexDelimiter.Length + 2)] == '\r') trimEnd += 1; + var trimStart = 3 + complexDelimiter.Length; + value = TokenTextBuilder.ToString(trimStart, TokenTextBuilder.Length - (trimStart + trimEnd)); + } + else if (isLong) { // Long strings ignore a newline immediately after the @{" and before the "} if (TokenTextBuilder[3] == '\n') TokenTextBuilder.Remove(3, 1); @@ -639,6 +699,10 @@ private char GetCurrent() { return _current; } + private Location CurrentLocation() { + return new Location(File, _previousLine, _previousColumn, _isDMStandard); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private char Advance() { int value = _source.Read(); From 4f5027bfc2c665cb17314808d10faae85d2ffe38 Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:16:05 -0600 Subject: [PATCH 2/7] more robust checking --- DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index 154b9cb3b1..bde1ab358b 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -344,8 +344,11 @@ public Token NextToken(bool ignoreWhitespace = false) { var delimIdx = 0; do { TokenTextBuilder.Append(c); + if (GetCurrent() == complexDelimiter[delimIdx]) delimIdx++; - if (delimIdx + 1 == complexDelimiter.Length) { + else delimIdx = 0; + + if (delimIdx + 1 == complexDelimiter.Length && c == complexDelimiter[^1]) { // latter check ensures a 1-char delimiter actually matches TokenTextBuilder.Remove(TokenTextBuilder.Length - delimIdx, complexDelimiter.Length - 1); break; } From 61defaaf144be78b4dc939b054a15a8066baabbd Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:21:06 -0600 Subject: [PATCH 3/7] cope with \r in long strings --- DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index bde1ab358b..861626ecee 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -414,8 +414,12 @@ public Token NextToken(bool ignoreWhitespace = false) { } else if (isLong) { // Long strings ignore a newline immediately after the @{" and before the "} + if (TokenTextBuilder[3] == '\r') + TokenTextBuilder.Remove(3, 1); if (TokenTextBuilder[3] == '\n') TokenTextBuilder.Remove(3, 1); + if (TokenTextBuilder[^3] == '\r') + TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); if (TokenTextBuilder[^3] == '\n') TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); From 16c8fbf01281089be4cbaf676b843f3b4ebe4a14 Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:27:01 -0600 Subject: [PATCH 4/7] I don't think this advance() was needed? --- DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index 861626ecee..6fbf24e4b4 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -395,8 +395,7 @@ public Token NextToken(bool ignoreWhitespace = false) { } } - if (isComplex) Advance(); - else TokenTextBuilder.Append(c); + TokenTextBuilder.Append(c); if (!HandleLineEnd()) Advance(); From 7b358e78e666fdf3feb7d21b25999aca9bf3bd8e Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:44:40 -0600 Subject: [PATCH 5/7] zoo wee mama --- DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index 6fbf24e4b4..da99701220 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -417,10 +417,10 @@ public Token NextToken(bool ignoreWhitespace = false) { TokenTextBuilder.Remove(3, 1); if (TokenTextBuilder[3] == '\n') TokenTextBuilder.Remove(3, 1); - if (TokenTextBuilder[^3] == '\r') - TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); if (TokenTextBuilder[^3] == '\n') TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); + if (TokenTextBuilder[^3] == '\r') + TokenTextBuilder.Remove(TokenTextBuilder.Length - 3, 1); value = TokenTextBuilder.ToString(3, TokenTextBuilder.Length - 5); } else { From d01386129d24795190437b311f5b231510932626 Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:50:37 -0600 Subject: [PATCH 6/7] add another comment --- DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index da99701220..b18f0518b3 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -408,7 +408,7 @@ public Token NextToken(bool ignoreWhitespace = false) { var trimEnd = complexDelimiter.Length; if (TokenTextBuilder[^(complexDelimiter.Length + 1)] == '\n') trimEnd += 1; if (TokenTextBuilder[^(complexDelimiter.Length + 2)] == '\r') trimEnd += 1; - var trimStart = 3 + complexDelimiter.Length; + var trimStart = 3 + complexDelimiter.Length; // 3 is from these chars: @() value = TokenTextBuilder.ToString(trimStart, TokenTextBuilder.Length - (trimStart + trimEnd)); } else if (isLong) { From 77256230340f1282db722355cd4c34775b442e07 Mon Sep 17 00:00:00 2001 From: ike709 Date: Tue, 26 Nov 2024 20:57:27 -0600 Subject: [PATCH 7/7] yep that advance() was needed --- DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index da99701220..031b16a0e4 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -395,7 +395,8 @@ public Token NextToken(bool ignoreWhitespace = false) { } } - TokenTextBuilder.Append(c); + if (isComplex) Advance(); + else TokenTextBuilder.Append(c); if (!HandleLineEnd()) Advance();