DocManager 懒加载，初始化加速

Author: SOVLOOKUP

packages/app/src/docbase/index.ts

@@ -39,7 +39,8 @@ const getDocBaseConfigFromEnv = (): DocBaseOptions => {
       host: MEILI_URL,
       apiKey: MEILI_MASTER_KEY
     },
-    initPaths: [INIT_PATH]
+    initPaths: [INIT_PATH],
+    initscan: true
   }
 }
 

packages/core/src/DocBase.ts

@@ -57,7 +57,7 @@ export interface DocBaseOptions {
   initPlugins?: PluginWithParams<any>[];
   /**
    * 是否在初始化时扫描初始化知识库目录
-   * @default true
+   * @default false
    */
   initscan?: boolean;
   /**
@@ -217,14 +217,13 @@ export class DocBase {
         },
       },
     ],
-    initscan = true,
+    initscan = false,
     fileOpThrottleMs,
   }: DocBaseOptions) => {
     console.info("Starting DocBase...");
     this.fileOpThrottleMs = fileOpThrottleMs;
-    // 加载所有插件
-    // 并行加载所有插件以提高效率
     console.info("Loading all plugins...");
+    // 加载所有插件
     await Promise.all(
       initPlugins.map((initPlugin) => this.loadPlugin(initPlugin))
     );
@@ -248,7 +247,6 @@ export class DocBase {
       docLoader: (path) => this.#hyperDocLoader(path),
       docSplitter: (text) => this.#docSplitter.func(text),
     });
-    await this.#docManager.init();
 
     // 初始化监视器扫描器
     console.info("Initializing watcher and scanner...");

packages/core/src/DocManager.ts

@@ -60,6 +60,7 @@ export const chunkDiff = (
  * 基于 MeiliSearch 实现的文档管理器
  */
 export class DocManager {
+  #alreadyInit = false
   /** MeiliSearch 客户端实例 */
   #client: MeiliSearch;
 
@@ -211,49 +212,72 @@ export class DocManager {
   };
 
   /**
-   * 初始化文档管理器
+   * 操作前确保初始化完成
    */
-  init = async () => {
-    console.info("Initializing DocManager...");
-    await this.#ensureContainsFilterFeatureOn();
-
-    this.#docIndex = await this.#getIndexOrCreate(`${this.#indexPrefix}docs`);
-
-    // 获取已有的可筛选属性
-    const docIndexFilterableAttributes =
-      await this.#docIndex.getFilterableAttributes();
-
-    // 需要设置的可筛选的属性
-    const docIndexFilterableAttributesNeedCreate = difference(
-      ["path", "chunkHashs"],
-      docIndexFilterableAttributes
-    );
-
-    // 设置可筛选属性
-    if (docIndexFilterableAttributesNeedCreate.length > 0) {
-      console.info(
-        `Creating filterable attributes for doc index: ${docIndexFilterableAttributesNeedCreate}`
-      );
-      await this.#docIndex.updateSettings({
-        filterableAttributes: [
-          ...docIndexFilterableAttributes,
-          ...docIndexFilterableAttributesNeedCreate,
-        ],
-      });
+  #ensureInit = async () => {
+    if (!this.#alreadyInit) {
+      await this.#init()
+      this.#alreadyInit = true
     }
+  }
 
-    this.#docChunkIndex = await this.#getIndexOrCreate(
-      `${this.#indexPrefix}chunks`
-    );
+  /**
+   * 初始化文档管理器
+   */
+  #init = async () => {
+    await Promise.all([
+      // 确保 ContainsFilter 功能开启
+      (async () => {
+        console.info("Initializing DocManager...");
+        await this.#ensureContainsFilterFeatureOn();
+      })(),
+      // 确保 docIndex 存在
+      (async () => {
+        this.#docIndex = await this.#getIndexOrCreate(`${this.#indexPrefix}docs`);
+
+        // 获取已有的可筛选属性
+        const docIndexFilterableAttributes = await this.#docIndex.getFilterableAttributes();
+
+        // 需要设置的可筛选的属性
+        const docIndexFilterableAttributesNeedCreate = difference(
+          ["path", "chunkHashs"],
+          docIndexFilterableAttributes
+        );
+
+        // 设置可筛选属性
+        if (docIndexFilterableAttributesNeedCreate.length > 0) {
+          console.info(
+            `Creating filterable attributes for doc index: ${docIndexFilterableAttributesNeedCreate}`
+          );
+          const task = await this.#docIndex.updateSettings({
+            filterableAttributes: [
+              ...docIndexFilterableAttributes,
+              ...docIndexFilterableAttributesNeedCreate,
+            ],
+          });
+          await this.#docIndex.waitForTask(task.taskUid);
+        }
+      })(),
+      // 确保 docChunkIndex 存在
+      (async () => {
+        this.#docChunkIndex = await this.#getIndexOrCreate(
+          `${this.#indexPrefix}chunks`
+        );
+      })()
+    ]);
 
     console.info("DocManager initialized successfully");
   };
 
   /** 获取已有 embedders */
-  getEmbedders = async () => await this.#docChunkIndex.getEmbedders()
+  getEmbedders = async () => {
+    await this.#ensureInit()
+    await this.#docChunkIndex.getEmbedders()
+  }
 
   /** 重置已有 embedders */
   resetEmbedders = async (wait = false) => {
+    await this.#ensureInit()
     const task = await this.#docChunkIndex.resetEmbedders()
     if (wait) {
       await this.#docChunkIndex.waitForTask(task.taskUid)
@@ -262,6 +286,7 @@ export class DocManager {
 
   /** 增删改 embedders */
   updateEmbedders = async (embedders: Embedders, wait = false) => {
+    await this.#ensureInit()
     const task = await this.#docChunkIndex.updateEmbedders(embedders)
     if (wait) {
       await this.#docChunkIndex.waitForTask(task.taskUid)
@@ -403,6 +428,7 @@ export class DocManager {
   };
 
   upsertDoc = async (path: string) => {
+    await this.#ensureInit()
     // 查询该路径有无文档
     const [{ mtimeMs }, remoteDocByPath] = await Promise.all([
       stat(path),
@@ -509,6 +535,7 @@ export class DocManager {
    * @param path - 文档路径
    */
   deleteDocByPath = async (path: string) => {
+    await this.#ensureInit()
     console.info(`Deleting document at path: ${path}`);
     const doc = await this.#getDocByPathIfExist(path);
 
@@ -524,6 +551,7 @@ export class DocManager {
    * @param path - 目录路径
    */
   deleteDocByPathPrefix = async (path: string) => {
+    await this.#ensureInit()
     console.info(`Deleting all documents under path prefix: ${path}`);
     const getDocs = async () =>
       await this.#docIndex.getDocuments({
@@ -552,6 +580,7 @@ export class DocManager {
    * @param hash - 文档 hash
    */
   deleteDocByHash = async (hash: string) => {
+    await this.#ensureInit()
     console.info(`Deleting document with hash: ${hash}`);
     const doc = await this.#getDocIfExist(hash);
 
@@ -569,6 +598,7 @@ export class DocManager {
    * @returns 返回搜索结果
    */
   search = async (query: string, opts?: SearchParams) => {
+    await this.#ensureInit()
     console.debug(`Searching for query: ${query}`);
     const result = await this.#docChunkIndex.search(query, opts);
     const hits = result.hits;