fix: handle line removal diff

.vscode/launch.json

@@ -95,6 +95,22 @@
         // Avoid timing out when stopping on breakpoints during debugging in VSCode
         "MOCHA_TIMEOUT": "0"
       }
+    },
+    {
+      "name": "[Core] Jest Test Debugger, Current Open File",
+      "type": "node",
+      "request": "launch",
+      "runtimeArgs": [
+        "--inspect-brk",
+        "${workspaceRoot}/core/node_modules/jest/bin/jest.js",
+        "--runInBand",
+        "--config",
+        "${workspaceRoot}/core/jest.config.js",
+        "${relativeFile}"
+      ],
+      "cwd": "${workspaceRoot}/core",
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
     }
   ]
 }

core/diff/streamDiff.ts

@@ -13,46 +13,69 @@ export async function* streamDiff(
   oldLines: string[],
   newLines: LineStream,
 ): AsyncGenerator<DiffLine> {
-  const mutatedOldLines = [...oldLines]; // be careful
-
   // If one indentation mistake is made, others are likely. So we are more permissive about matching
   let seenIndentationMistake = false;
 
   let newLineResult = await newLines.next();
+
   while (oldLines.length > 0 && !newLineResult.done) {
-    const [matchIndex, isPerfectMatch, newLine] = matchLine(
+    const { matchIndex, isPerfectMatch, newLine } = matchLine(
       newLineResult.value,
       oldLines,
       seenIndentationMistake,
     );
+
     if (!seenIndentationMistake && newLineResult.value !== newLine) {
       seenIndentationMistake = true;
     }
 
-    if (matchIndex < 0) {
-      // Insert new line
-      yield { type: "new", line: newLine };
+    let type: DiffLine["type"];
+
+    let isLineRemoval = false;
+    const isNewLine = matchIndex === -1;
+
+    if (isNewLine) {
+      type = "new";
     } else {
       // Insert all deleted lines before match
       for (let i = 0; i < matchIndex; i++) {
         yield { type: "old", line: oldLines.shift()! };
       }
 
-      if (isPerfectMatch) {
-        // Same
-        yield { type: "same", line: oldLines.shift()! };
-      } else {
-        // Delete old line and insert the new
-        yield { type: "old", line: oldLines.shift()! };
-        yield { type: "new", line: newLine };
-      }
+      type = isPerfectMatch ? "same" : "old";
     }
 
-    newLineResult = await newLines.next();
+    switch (type) {
+      case "new":
+        yield { type, line: newLine };
+        break;
+
+      case "same":
+        yield { type, line: oldLines.shift()! };
+        break;
+
+      case "old":
+        yield { type, line: oldLines.shift()! };
+
+        if (oldLines[0] !== newLine) {
+          yield { type: "new", line: newLine };
+        } else {
+          isLineRemoval = true;
+        }
+
+        break;
+
+      default:
+        console.error(`Error streaming diff, unrecognized diff type: ${type}`);
+    }
+
+    if (!isLineRemoval) {
+      newLineResult = await newLines.next();
+    }
   }
 
   // Once at the edge, only one choice
-  if (newLineResult.done === true && oldLines.length > 0) {
+  if (newLineResult.done && oldLines.length > 0) {
     for (const oldLine of oldLines) {
       yield { type: "old", line: oldLine };
     }

core/diff/util.ts

@@ -4,6 +4,16 @@ import { stripImages } from "../llm/countTokens.js";
 
 export type LineStream = AsyncGenerator<string>;
 
+export type MatchLineResult = {
+  /**
+   * -1 if it's a new line, otherwise the index of the first match
+   * in the old lines.
+   */
+  matchIndex: number;
+  isPerfectMatch: boolean;
+  newLine: string;
+};
+
 function linesMatchPerfectly(lineA: string, lineB: string): boolean {
   return lineA === lineB && lineA !== "";
 }
@@ -18,6 +28,7 @@ function linesMatch(lineA: string, lineB: string, linesBetween = 0): boolean {
   }
 
   const d = distance(lineA, lineB);
+
   return (
     // Should be more unlikely for lines to fuzzy match if they are further away
     (d / Math.max(lineA.length, lineB.length) < 0.5 - linesBetween * 0.05 ||
@@ -34,35 +45,44 @@ export function matchLine(
   newLine: string,
   oldLines: string[],
   permissiveAboutIndentation = false,
-): [number, boolean, string] {
+): MatchLineResult {
   // Only match empty lines if it's the next one:
   if (newLine.trim() === "" && oldLines[0]?.trim() === "") {
-    return [0, true, newLine.trim()];
+    return {
+      matchIndex: 0,
+      isPerfectMatch: true,
+      newLine: newLine.trim(),
+    };
   }
 
   const isEndBracket = END_BRACKETS.includes(newLine.trim());
+
   for (let i = 0; i < oldLines.length; i++) {
     // Don't match end bracket lines if too far away
     if (i > 4 && isEndBracket) {
-      return [-1, false, newLine];
+      return { matchIndex: -1, isPerfectMatch: false, newLine };
     }
 
     if (linesMatchPerfectly(newLine, oldLines[i])) {
-      return [i, true, newLine];
+      return { matchIndex: i, isPerfectMatch: true, newLine };
     }
     if (linesMatch(newLine, oldLines[i], i)) {
       // This is a way to fix indentation, but only for sufficiently long lines to avoid matching whitespace or short lines
       if (
         newLine.trimStart() === oldLines[i].trimStart() &&
         (permissiveAboutIndentation || newLine.trim().length > 8)
       ) {
-        return [i, true, oldLines[i]];
+        return {
+          matchIndex: i,
+          isPerfectMatch: true,
+          newLine: oldLines[i],
+        };
       }
-      return [i, false, newLine];
+      return { matchIndex: i, isPerfectMatch: false, newLine };
     }
   }
 
-  return [-1, false, newLine];
+  return { matchIndex: -1, isPerfectMatch: false, newLine };
 }
 
 /**