From 897946e34032a65b52580eb6ef4b6c469eeaa009 Mon Sep 17 00:00:00 2001 From: Saphire Lattice Date: Mon, 18 Nov 2024 01:25:44 +0700 Subject: [PATCH] Fix locations reporting (#2093) --- .../Compiler/DM/AST/DMAST.Expression.cs | 7 ++++ DMCompiler/Compiler/DM/AST/DMAST.cs | 14 +++++++ DMCompiler/Compiler/DM/DMLexer.cs | 27 +++++++------ DMCompiler/Compiler/DM/DMParser.cs | 6 +-- .../Compiler/DMPreprocessor/DMPreprocessor.cs | 4 +- .../DMPreprocessor/DMPreprocessorLexer.cs | 16 +++++--- DMCompiler/Compiler/Lexer.cs | 39 +++++++++++++++++-- 7 files changed, 85 insertions(+), 28 deletions(-) diff --git a/DMCompiler/Compiler/DM/AST/DMAST.Expression.cs b/DMCompiler/Compiler/DM/AST/DMAST.Expression.cs index a879b380fb..f880bfe587 100644 --- a/DMCompiler/Compiler/DM/AST/DMAST.Expression.cs +++ b/DMCompiler/Compiler/DM/AST/DMAST.Expression.cs @@ -15,6 +15,13 @@ public virtual IEnumerable Leaves() { public virtual DMASTExpression GetUnwrapped() { return this; } + + public override string ToStringNoLocation() { + var leaves = Leaves().ToList(); + if (leaves.Count == 0) + return $"{GetType().Name}"; + return $"{GetType().Name}({string.Join(", ", Leaves().Select(l => l.ToString(Location)))})"; + } } /// diff --git a/DMCompiler/Compiler/DM/AST/DMAST.cs b/DMCompiler/Compiler/DM/AST/DMAST.cs index 4092e2f2a8..993e2b28fa 100644 --- a/DMCompiler/Compiler/DM/AST/DMAST.cs +++ b/DMCompiler/Compiler/DM/AST/DMAST.cs @@ -4,6 +4,20 @@ namespace DMCompiler.Compiler.DM.AST; public abstract class DMASTNode(Location location) { public readonly Location Location = location; + + public override string ToString() { + return $"{ToString(null)}"; + } + + public string ToString(Location? loc) { + if (loc is not null && Location.SourceFile == loc.Value.SourceFile && Location.Line == loc.Value.Line) + return ToStringNoLocation(); + return $"{ToStringNoLocation()} [{Location}]"; + } + + public virtual string ToStringNoLocation() { + return GetType().Name; + } } public sealed class DMASTFile(Location location, DMASTBlockInner blockInner) : DMASTNode(location) { diff --git a/DMCompiler/Compiler/DM/DMLexer.cs b/DMCompiler/Compiler/DM/DMLexer.cs index df06797f36..8e4e710034 100644 --- a/DMCompiler/Compiler/DM/DMLexer.cs +++ b/DMCompiler/Compiler/DM/DMLexer.cs @@ -114,6 +114,7 @@ protected override Token ParseNextToken() { token = preprocToken; } } else { + var firstTokenLocation = CurrentLocation; switch (preprocToken.Type) { case TokenType.DM_Preproc_Whitespace: Advance(); token = CreateToken(TokenType.DM_Whitespace, preprocToken.Text); break; case TokenType.DM_Preproc_Punctuator_LeftParenthesis: BracketNesting++; Advance(); token = CreateToken(TokenType.DM_LeftParenthesis, preprocToken.Text); break; @@ -125,19 +126,19 @@ protected override Token ParseNextToken() { case TokenType.DM_Preproc_Punctuator_Question: switch (Advance().Type) { case TokenType.DM_Preproc_Punctuator_Period: - token = CreateToken(TokenType.DM_QuestionPeriod, "?."); Advance(); + token = CreateToken(TokenType.DM_QuestionPeriod, "?.", firstTokenLocation); break; case TokenType.DM_Preproc_Punctuator_Colon: - token = CreateToken(TokenType.DM_QuestionColon, "?:"); Advance(); + token = CreateToken(TokenType.DM_QuestionColon, "?:", firstTokenLocation); break; case TokenType.DM_Preproc_Punctuator_LeftBracket: - token = CreateToken(TokenType.DM_QuestionLeftBracket, "?["); - BracketNesting++; Advance(); + token = CreateToken(TokenType.DM_QuestionLeftBracket, "?[", firstTokenLocation); + BracketNesting++; break; default: @@ -149,11 +150,10 @@ protected override Token ParseNextToken() { switch (Advance().Type) { case TokenType.DM_Preproc_Punctuator_Period: if (Advance().Type == TokenType.DM_Preproc_Punctuator_Period) { - token = CreateToken(TokenType.DM_IndeterminateArgs, "..."); - Advance(); + token = CreateToken(TokenType.DM_IndeterminateArgs, "...", firstTokenLocation); } else { - token = CreateToken(TokenType.DM_SuperProc, ".."); + token = CreateToken(TokenType.DM_SuperProc, "..", firstTokenLocation); } break; @@ -231,6 +231,7 @@ protected override Token ParseNextToken() { break; } case TokenType.DM_Preproc_ConstantString: { + Advance(); string tokenText = preprocToken.Text; switch (preprocToken.Text[0]) { case '"': @@ -239,21 +240,19 @@ protected override Token ParseNextToken() { case '@': token = CreateToken(TokenType.DM_RawString, tokenText, preprocToken.Value); break; default: token = CreateToken(TokenType.Error, tokenText, "Invalid string"); break; } - - Advance(); break; } case TokenType.DM_Preproc_StringBegin: - token = CreateToken(TokenType.DM_StringBegin, preprocToken.Text, preprocToken.Value); Advance(); + token = CreateToken(TokenType.DM_StringBegin, preprocToken.Text, preprocToken.Value); break; case TokenType.DM_Preproc_StringMiddle: - token = CreateToken(TokenType.DM_StringMiddle, preprocToken.Text, preprocToken.Value); Advance(); + token = CreateToken(TokenType.DM_StringMiddle, preprocToken.Text, preprocToken.Value); break; case TokenType.DM_Preproc_StringEnd: - token = CreateToken(TokenType.DM_StringEnd, preprocToken.Text, preprocToken.Value); Advance(); + token = CreateToken(TokenType.DM_StringEnd, preprocToken.Text, preprocToken.Value); break; case TokenType.DM_Preproc_Identifier: { TokenTextBuilder.Clear(); @@ -267,7 +266,7 @@ protected override Token ParseNextToken() { var identifierText = TokenTextBuilder.ToString(); var tokenType = Keywords.GetValueOrDefault(identifierText, TokenType.DM_Identifier); - token = CreateToken(tokenType, identifierText); + token = CreateToken(tokenType, identifierText, firstTokenLocation); break; } case TokenType.DM_Preproc_Number: { @@ -290,7 +289,7 @@ protected override Token ParseNextToken() { break; } - case TokenType.EndOfFile: token = preprocToken; Advance(); break; + case TokenType.EndOfFile: Advance(); token = preprocToken; break; default: token = CreateToken(TokenType.Error, preprocToken.Text, "Invalid token"); break; } } diff --git a/DMCompiler/Compiler/DM/DMParser.cs b/DMCompiler/Compiler/DM/DMParser.cs index c0ecd037e1..00c566fd16 100644 --- a/DMCompiler/Compiler/DM/DMParser.cs +++ b/DMCompiler/Compiler/DM/DMParser.cs @@ -337,7 +337,7 @@ public DMASTFile File() { } //Empty object definition - Compiler.VerbosePrint($"Parsed object {CurrentPath}"); + Compiler.VerbosePrint($"Parsed object {CurrentPath} - empty"); return new DMASTObjectDefinition(loc, CurrentPath, null); } @@ -1747,10 +1747,10 @@ private void ExpressionTo(out DMASTExpression endRange, out DMASTExpression? ste } private DMASTExpression? ExpressionIn() { + var loc = Current().Location; // Don't check this inside, as Check() will advance and point at next token instead DMASTExpression? value = ExpressionAssign(); while (value != null && Check(TokenType.DM_In)) { - var loc = Current().Location; Whitespace(); DMASTExpression? list = ExpressionAssign(); @@ -2151,7 +2151,7 @@ private void ExpressionTo(out DMASTExpression endRange, out DMASTExpression? ste if (inner is null) { inner = new DMASTVoid(loc); } else { - inner = new DMASTExpressionWrapped(inner.Location, inner); + inner = new DMASTExpressionWrapped(loc, inner); } return inner; diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs index b30049cadb..de4ad91b44 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs @@ -638,7 +638,7 @@ private void HandlePragmaDirective() { Token warningTypeToken = GetNextToken(true); if (warningTypeToken.Type != TokenType.DM_Preproc_Identifier) { - compiler.Emit(WarningCode.BadDirective, warningNameToken.Location, $"Warnings can only be set to disabled, notice, warning, or error"); + compiler.Emit(WarningCode.BadDirective, warningTypeToken.Location, "Warnings can only be set to disabled, notice, warning, or error"); return; } switch(warningTypeToken.Text.ToLower()) { @@ -660,7 +660,7 @@ private void HandlePragmaDirective() { compiler.SetPragma(warningCode, ErrorLevel.Error); break; default: - compiler.Emit(WarningCode.BadDirective, warningNameToken.Location, $"Warnings can only be set to disabled, notice, warning, or error"); + compiler.Emit(WarningCode.BadDirective, warningTypeToken.Location, "Warnings can only be set to disabled, notice, warning, or error"); return; } } diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs index ae2f5a9e1f..83a3a3a403 100644 --- a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs +++ b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs @@ -20,6 +20,7 @@ internal sealed class DMPreprocessorLexer { private readonly bool _isDMStandard; private char _current; private int _currentLine = 1, _currentColumn; + private int _previousLine = 1, _previousColumn; private readonly Queue _pendingTokenQueue = new(); // TODO: Possible to remove this? public DMPreprocessorLexer(DMCompiler compiler, string includeDirectory, string file, string source) { @@ -61,6 +62,9 @@ public Token NextToken(bool ignoreWhitespace = false) { char c = GetCurrent(); + _previousLine = _currentLine; + _previousColumn = _currentColumn; + switch (c) { case '\0': return CreateToken(TokenType.EndOfFile, c); @@ -367,7 +371,7 @@ public Token NextToken(bool ignoreWhitespace = false) { LexString(true) : CreateToken(TokenType.DM_Preproc_Punctuator, c); case '#': { - bool isConcat = (Advance() == '#'); + bool isConcat = Advance() == '#'; if (isConcat) Advance(); // Whitespace after '#' is ignored @@ -444,7 +448,7 @@ public Token NextToken(bool ignoreWhitespace = false) { } Advance(); - return CreateToken(TokenType.Error, string.Empty, $"Unknown character: {c.ToString()}"); + return CreateToken(TokenType.Error, string.Empty, $"Unknown character: {c}"); } } } @@ -619,7 +623,7 @@ private bool HandleLineEnd() { goto case '\n'; case '\n': _currentLine++; - _currentColumn = 1; + _currentColumn = 0; // Because Advance will bump this to 1 and any position reads will happen next NextToken() call if (c == '\n') // This line could have ended with only \r Advance(); @@ -641,7 +645,7 @@ private char Advance() { if (value == -1) { _current = '\0'; - } else { + } else { _currentColumn++; _current = (char)value; } @@ -656,11 +660,11 @@ private bool AtEndOfSource() { [MethodImpl(MethodImplOptions.AggressiveInlining)] private Token CreateToken(TokenType type, string text, object? value = null) { - return new Token(type, text, new Location(File, _currentLine, _currentColumn, _isDMStandard), value); + return new Token(type, text, new Location(File, _previousLine, _previousColumn, _isDMStandard), value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private Token CreateToken(TokenType type, char text, object? value = null) { - return new Token(type, text.ToString(), new Location(File, _currentLine, _currentColumn, _isDMStandard), value); + return new Token(type, text.ToString(), new Location(File, _previousLine, _previousColumn, _isDMStandard), value); } } diff --git a/DMCompiler/Compiler/Lexer.cs b/DMCompiler/Compiler/Lexer.cs index 4e336b140e..27b3560120 100644 --- a/DMCompiler/Compiler/Lexer.cs +++ b/DMCompiler/Compiler/Lexer.cs @@ -3,17 +3,32 @@ namespace DMCompiler.Compiler; internal class Lexer { + /// + /// Location of token that'll be output by . If you skip through more + /// public Location CurrentLocation { get; protected set; } + + /// + /// Location of a previous token. + /// + public Location PreviousLocation { get; private set; } + public IEnumerable Source { get; } public bool AtEndOfSource { get; private set; } - protected Queue _pendingTokenQueue = new(); private readonly IEnumerator _sourceEnumerator; private TSourceType _current; + /// + /// Given a stream of some type, allows to advance through it and create tokens + /// + /// Used to build the initial Location, access through + /// Source of input + /// Thrown if is null protected Lexer(string sourceName, IEnumerable source) { CurrentLocation = new Location(sourceName, 1, 0); + PreviousLocation = CurrentLocation; Source = source; if (source == null) throw new FileNotFoundException("Source file could not be read: " + sourceName); @@ -24,7 +39,7 @@ public Token GetNextToken() { if (_pendingTokenQueue.Count > 0) return _pendingTokenQueue.Dequeue(); - Token nextToken = ParseNextToken(); + var nextToken = ParseNextToken(); while (nextToken.Type == TokenType.Skip) nextToken = ParseNextToken(); if (_pendingTokenQueue.Count > 0) { @@ -39,10 +54,24 @@ protected virtual Token ParseNextToken() { return CreateToken(TokenType.Unknown, GetCurrent()?.ToString() ?? string.Empty); } + protected Token CreateToken(TokenType type, string text, Location location, object? value = null) { + var token = new Token(type, text, location, value); + return token; + } + + /// + /// Creates a new located at + /// + /// + /// If you have used more than once, the will be incorrect, + /// and you'll need to use + /// with a previously recorded + /// protected Token CreateToken(TokenType type, string text, object? value = null) { - return new Token(type, text, CurrentLocation, value); + return CreateToken(type, text, PreviousLocation, value); } + /// protected Token CreateToken(TokenType type, char text, object? value = null) { return CreateToken(type, char.ToString(text), value); } @@ -51,7 +80,10 @@ protected virtual TSourceType GetCurrent() { return _current; } + /// Call before CreateToken to make sure the location is correct protected virtual TSourceType Advance() { + PreviousLocation = CurrentLocation; + if (_sourceEnumerator.MoveNext()) { _current = _sourceEnumerator.Current; } else { @@ -63,6 +95,7 @@ protected virtual TSourceType Advance() { } internal class TokenLexer : Lexer { + /// protected TokenLexer(string sourceName, IEnumerable source) : base(sourceName, source) { Advance(); }