Merge pull request continuedev#4587 from continuedev/dallin/cache-walkdir

Walkdir caching first pass

Author: sestinj

core/config/loadLocalAssistants.ts

@@ -31,11 +31,14 @@ export async function getAssistantFilesFromDir(
       return [];
     }
 
-    const overrideIgnore = ignore()
+    const overrideDefaultIgnores = ignore()
       .add(DEFAULT_IGNORE_FILETYPES.filter((t) => t !== "config.yaml"))
       .add(DEFAULT_IGNORE_DIRS);
 
-    const uris = await walkDir(dir, ide, { overrideIgnore });
+    const uris = await walkDir(dir, ide, {
+      overrideDefaultIgnores,
+      source: "get assistant files",
+    });
     const assistantFilePaths = uris.filter(
       (p) => p.endsWith(".yaml") || p.endsWith(".yml"),
     );

core/context/providers/FileContextProvider.ts

@@ -8,9 +8,9 @@ import {
 } from "../../";
 import { walkDirs } from "../../indexing/walkDir";
 import {
-  getUriPathBasename,
   getShortestUniqueRelativeUriPaths,
   getUriDescription,
+  getUriPathBasename,
 } from "../../util/uri";
 
 const MAX_SUBMENU_ITEMS = 10_000;
@@ -53,7 +53,13 @@ class FileContextProvider extends BaseContextProvider {
     args: LoadSubmenuItemsArgs,
   ): Promise<ContextSubmenuItem[]> {
     const workspaceDirs = await args.ide.getWorkspaceDirs();
-    const results = await walkDirs(args.ide, undefined, workspaceDirs);
+    const results = await walkDirs(
+      args.ide,
+      {
+        source: "load submenu items - file",
+      },
+      workspaceDirs,
+    );
     const files = results.flat().slice(-MAX_SUBMENU_ITEMS);
     const withUniquePaths = getShortestUniqueRelativeUriPaths(
       files,

core/context/providers/FileTreeContextProvider.ts

@@ -50,7 +50,9 @@ class FileTreeContextProvider extends BaseContextProvider {
         directories: [],
       };
 
-      const uris = await walkDir(workspaceDir, extras.ide);
+      const uris = await walkDir(workspaceDir, extras.ide, {
+        source: "get context items - file tree",
+      });
       const relativePaths = uris.map(
         (uri) => findUriInDirs(uri, [workspaceDir]).relativePathOrBasename,
       );

core/context/providers/FolderContextProvider.ts

@@ -36,6 +36,7 @@ class FolderContextProvider extends BaseContextProvider {
       args.ide,
       {
         onlyDirs: true,
+        source: "load submenu items - folder",
       },
       workspaceDirs,
     );

core/context/providers/RepoMapContextProvider.ts

@@ -55,6 +55,7 @@ class RepoMapContextProvider extends BaseContextProvider {
       args.ide,
       {
         onlyDirs: true,
+        source: "load submenu items - repo map",
       },
       workspaceDirs,
     );

core/core.ts

@@ -46,6 +46,7 @@ import {
 
 import { isLocalAssistantFile } from "./config/loadLocalAssistants";
 import { shouldIgnore } from "./indexing/shouldIgnore";
+import { walkDirCache } from "./indexing/walkDir";
 import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
 import type { IMessenger, Message } from "./protocol/messenger";
 
@@ -834,6 +835,7 @@ export class Core {
       if (!config || config.disableIndexing) {
         return; // TODO silent in case of commands?
       }
+      walkDirCache.invalidate();
       if (data?.shouldClearIndexes) {
         const codebaseIndexer = await this.codebaseIndexerPromise;
         await codebaseIndexer.clearIndexes();
@@ -861,9 +863,8 @@ export class Core {
     // TODO - remove remaining logic for these from IDEs where possible
     on("files/changed", async ({ data }) => {
       if (data?.uris?.length) {
+        walkDirCache.invalidate(); // safe approach for now - TODO - only invalidate on relevant changes
         for (const uri of data.uris) {
-          // Listen for file changes in the workspace
-          // URI TODO is this equality statement valid?
           const currentProfileUri =
             this.configHandler.currentProfile?.profileDescription.uri ?? "";
 
@@ -934,6 +935,7 @@ export class Core {
 
     on("files/created", async ({ data }) => {
       if (data?.uris?.length) {
+        walkDirCache.invalidate();
         void refreshIfNotIgnored(data.uris);
 
         // If it's a local assistant being created, we want to reload all assistants so it shows up in the list
@@ -947,6 +949,7 @@ export class Core {
 
     on("files/deleted", async ({ data }) => {
       if (data?.uris?.length) {
+        walkDirCache.invalidate();
         void refreshIfNotIgnored(data.uris);
       }
     });

core/indexing/CodebaseIndexer.test.ts

@@ -14,12 +14,12 @@ import {
 } from "../test/testDir.js";
 import { getIndexSqlitePath } from "../util/paths.js";
 
+import { localPathToUri } from "../util/pathToUri.js";
 import { CodebaseIndexer, PauseToken } from "./CodebaseIndexer.js";
 import { getComputeDeleteAddRemove } from "./refreshIndex.js";
 import { TestCodebaseIndex } from "./TestCodebaseIndex.js";
 import { CodebaseIndex } from "./types.js";
-import { walkDir } from "./walkDir.js";
-import { localPathToUri } from "../util/pathToUri.js";
+import { walkDir, walkDirCache } from "./walkDir.js";
 
 jest.useFakeTimers();
 
@@ -87,6 +87,10 @@ describe("CodebaseIndexer", () => {
     tearDownTestDir();
   });
 
+  afterEach(() => {
+    walkDirCache.invalidate();
+  });
+
   async function refreshIndex() {
     const abortController = new AbortController();
     const abortSignal = abortController.signal;

core/indexing/CodebaseIndexer.ts

@@ -281,7 +281,9 @@ export class CodebaseIndexer {
         status: "indexing",
       };
       const directoryFiles = [];
-      for await (const p of walkDirAsync(directory, this.ide)) {
+      for await (const p of walkDirAsync(directory, this.ide, {
+        source: "codebase indexing: refresh dirs",
+      })) {
         directoryFiles.push(p);
         if (abortSignal.aborted) {
           yield {

core/indexing/walkDir.test.ts

@@ -1,7 +1,12 @@
 // Generated by continue
 import fs from "fs";
 import path from "path";
-import { walkDir, walkDirAsync, walkDirs } from "../indexing/walkDir";
+import {
+  walkDir,
+  walkDirAsync,
+  walkDirCache,
+  walkDirs,
+} from "../indexing/walkDir";
 import { testIde } from "../test/fixtures";
 import {
   addToTestDir,
@@ -17,6 +22,7 @@ describe("walkDir functions", () => {
 
   afterEach(async () => {
     tearDownTestDir();
+    walkDirCache.invalidate();
   });
 
   // describe.only("test speed", () => {