Fix corruption of sliding text window when trying to peek backwards. (#78774)

Co-authored-by: Fred Silberberg <[email protected]>

Author: CyrusNajmabadi

src/Compilers/CSharp/Portable/Parser/Lexer.cs

@@ -475,12 +475,7 @@ private void ScanSyntaxToken(ref TokenInfo info)
                             // dots followed by an integer (which will be treated as a range expression).
                             //
                             // Move back one space to see what's before this dot and adjust accordingly.
-
-                            this.TextWindow.Reset(atDotPosition - 1);
-                            var priorCharacterIsDot = this.TextWindow.PeekChar() is '.';
-                            this.TextWindow.Reset(atDotPosition);
-
-                            if (priorCharacterIsDot)
+                            if (this.TextWindow.PreviousChar() is '.')
                             {
                                 // We have two dots in a row.  Treat the second dot as a dot, not the start of a number literal.
                                 TextWindow.AdvanceChar();

src/Compilers/CSharp/Portable/Parser/SlidingTextWindow.cs

@@ -3,12 +3,8 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-using System.Collections.Concurrent;
 using System.Diagnostics;
 using System.Text;
-using Microsoft.CodeAnalysis.Collections;
-using Microsoft.CodeAnalysis.CSharp.Symbols;
-using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Microsoft.CodeAnalysis.PooledObjects;
 using Microsoft.CodeAnalysis.Text;
 using Roslyn.Utilities;
@@ -374,6 +370,25 @@ public char PeekChar(int delta)
             return ch;
         }
 
+        public char PreviousChar()
+        {
+            Debug.Assert(this.Position > 0);
+            if (_offset > 0)
+            {
+                // The allowed region of the window that can be read is from 0 to _characterWindowCount (which _offset
+                // is in between).  So as long as _offset is greater than 0, we can read the previous character directly
+                // from the current chunk of characters in the window.
+                return this.CharacterWindow[_offset - 1];
+            }
+
+            // The prior character isn't in the window (trying to read the current character caused us to
+            // read in the next chunk of text into the window, throwing out the preceding characters).
+            // Just go back to the source text to find this character.  While more expensive, this should
+            // be rare given that most of the time we won't be calling this right after loading a new text
+            // chunk.
+            return this.Text[this.Position - 1];
+        }
+
         /// <summary>
         /// If the next characters in the window match the given string,
         /// then advance past those characters.  Otherwise, do nothing.
@@ -480,5 +495,16 @@ internal static char GetCharsFromUtf32(uint codepoint, out char lowSurrogate)
                 return (char)((codepoint - 0x00010000) / 0x0400 + 0xD800);
             }
         }
+
+        internal TestAccessor GetTestAccessor()
+            => new TestAccessor(this);
+
+        internal readonly struct TestAccessor(SlidingTextWindow window)
+        {
+            private readonly SlidingTextWindow _window = window;
+
+            internal void SetDefaultCharacterWindow()
+                => _window._characterWindow = new char[DefaultWindowLength];
+        }
     }
 }

src/Compilers/CSharp/Test/Syntax/LexicalAndXml/LexicalTests.cs

@@ -8,6 +8,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Threading;
 using Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax;
 using Microsoft.CodeAnalysis.CSharp.Test.Utilities;
 using Microsoft.CodeAnalysis.Text;
@@ -4572,5 +4573,90 @@ disabled text 2
             Assert.True(trivia.ContainsDiagnostics);
             Assert.Equal((int)ErrorCode.ERR_Merge_conflict_marker_encountered, trivia.Errors().Single().Code);
         }
+
+        [Fact, WorkItem("https://github.com/dotnet/roslyn/issues/78593")]
+        public void TestDotPrefixedNumberStartingAtStartOfSlidingTextWindow()
+        {
+            // This test depends on the line endings for the file being \r\n to ensure the right contents lines up at
+            // the right locations.
+            //
+            // It specifically validates what happens when we see `.0` at the start of the
+            // sliding text window, where the lexer tries to peek back one char to see if this
+            // is actually `..0` (a range expr) or `.0` (a floating point number).
+            var code = Resources.DotPrefixedNumberStartingAtStartOfSlidingTextWindow;
+            if (!code.Contains("\r\n"))
+                code = code.Replace("\n", "\r\n");
+
+            var sourceText = SourceText.From(code);
+
+            {
+                // Run a full parse, and validate the tree returned).
+
+                using var lexer = new Lexer(sourceText, CSharpParseOptions.Default);
+
+                // Ensure we have a normal window size, not some larger array that another test created and cached in
+                // the window pool
+                lexer.TextWindow.GetTestAccessor().SetDefaultCharacterWindow();
+
+                using var parser = new LanguageParser(lexer, oldTree: null, changes: null);
+
+                Microsoft.CodeAnalysis.SyntaxTreeExtensions.VerifySource(
+                    sourceText, parser.ParseCompilationUnit().CreateRed());
+            }
+
+            {
+                // Now, replicate the same conditions that hte parser runs through by driving the a new lexer here
+                // directly.  That ensures that we are actually validating exactly the conditions that led to the bug
+                // (a dot token starting a number, right at the start of the character window).
+                var lexer = new Lexer(sourceText, CSharpParseOptions.Default);
+
+                // Ensure we have a normal window size, not some larger array that another test created and cached in
+                // the window pool
+                lexer.TextWindow.GetTestAccessor().SetDefaultCharacterWindow();
+
+                var mode = LexerMode.Syntax;
+                for (var i = 0; i < 1326; i++)
+                    lexer.Lex(ref mode);
+
+                // Lexer will read from index 0 in the arrray.
+                Assert.Equal(0, lexer.TextWindow.Offset);
+
+                // We have 205 real chars in the window
+                Assert.Equal(205, lexer.TextWindow.CharacterWindowCount);
+
+                // The lexer is at position 10199 in the file.
+                Assert.Equal(10199, lexer.TextWindow.Position);
+
+                /// The 205 characters represent the final part of the doc
+                Assert.Equal(lexer.TextWindow.Text.Length, lexer.TextWindow.Position + lexer.TextWindow.CharacterWindowCount);
+
+                // We're at the start of a token.
+                Assert.Equal(lexer.TextWindow.LexemeStartPosition, lexer.TextWindow.Position);
+
+                // Ensure that the lexer's window is starting with the next FP number (".03") right at
+                // the start of the window.
+                Assert.True(lexer.TextWindow.CharacterWindow is ['.', '0', '3', ',', ..], $"Start of window was '{new string(lexer.TextWindow.CharacterWindow, 0, 4)}'");
+
+                var token = lexer.Lex(ref mode);
+                Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind);
+                Assert.Equal(3, token.FullWidth);
+                Assert.Equal(".03", token.ToString());
+
+                // But we moved 3 characters forward.
+                Assert.Equal(3, lexer.TextWindow.Offset);
+
+                // We still have 205 real chars in the window
+                Assert.Equal(205, lexer.TextWindow.CharacterWindowCount);
+
+                // The lexer position has moved 3 characters forward as well.
+                Assert.Equal(10202, lexer.TextWindow.Position);
+
+                // We're at the start of a token.
+                Assert.Equal(lexer.TextWindow.LexemeStartPosition, lexer.TextWindow.Position);
+
+                // Character window didn't changee.
+                Assert.True(lexer.TextWindow.CharacterWindow is ['.', '0', '3', ',', ..], $"Start of window was '{new string(lexer.TextWindow.CharacterWindow, 0, 4)}'");
+            }
+        }
     }
 }