多知识库结构初步

Author: SOVLOOKUP

packages/core/src/DocBase.ts

@@ -256,20 +256,7 @@ export class DocBase {
 
     // 初始化监视器扫描器
     console.info("Initializing watcher and scanner...");
-    const { watcher, scanner } = FSLayer({
-      filter: (path: string) => {
-        const ext = getExtFromPath(path);
-        return this.#docExtToLoaderName.has(ext);
-      },
-      upsert: (path: string) => {
-        this.#watcherTaskCache.set(path, "upsert");
-        this.#doWatcherTask();
-      },
-      remove: (path: string) => {
-        this.#watcherTaskCache.set(path, "remove");
-        this.#doWatcherTask();
-      },
-    });
+    const { watcher, scanner } = FSLayer();
     this.#docWatcher = watcher;
     this.#docScanner = scanner;
     console.info("Watcher and scanner initialized successfully.");
@@ -278,43 +265,63 @@ export class DocBase {
     console.info("Initializing DocManager...");
     const docLoader = (input: DocLoaderInput) => this.#hyperDocLoader(input)
     const docSplitter = (content: AsyncIterable<Content>) => this.#docSplitter.func(content)
+    const filter = (path: string) => {
+      const ext = getExtFromPath(path);
+      return this.#docExtToLoaderName.has(ext);
+    }
     const base = await db.base.get()
 
-    base.map(async ({ path, id }) => {
-      const docm = new DocManager({
-        indexPrefix: id,
-        meiliSearch: this.#meiliSearch,
-        docLoader,
-        docSplitter,
-      });
-      await docm.init()
-      // 扫描目录
-      await this.#scan([path]);
-      // 监视目录
-      this.#docWatcher.watch(path)
-      return docm
-    })
-    console.info("DocManager initialized successfully.");
-
-    // 扫描加载默认目录下文档
-    // console.info("Scanning initial directories...");
-    // console.info("Initial directories scanned successfully.");
-
-    // 开启监视，同步变动文档
-    // console.info("Starting to watch directories...");
-    // initPaths.map((initPath) => this.#docWatcher.watch(initPath));
-    // console.info("Directories are being watched.");
+    const result = await Promise.allSettled(
+      base.map(async ({ path, id }) => {
+        console.info(`Init base ${id}...`);
+        const docm = new DocManager({
+          indexPrefix: id,
+          meiliSearch: this.#meiliSearch,
+          docLoader,
+          docSplitter,
+        });
+        await docm.init()
+        this.#docManagers.set(id, docm)
+        console.info(`Base ${id} init successfully.`);
+
+        // 扫描目录
+        console.info(`Scanning base ${id}...`);
+        await this.#scan(id, [path]);
+        console.info(`Base ${id} scanned successfully.`);
+
+        // 监视目录
+        this.#docWatcher.watch(path, {
+          filter,
+          upsert: (path: string) => {
+            this.#watcherTaskCache.set(path, { docManagerId: id, type: "upsert" });
+            this.#doWatcherTask();
+          },
+          remove: (path: string) => {
+            this.#watcherTaskCache.set(path, { docManagerId: id, type: "remove" });
+            this.#doWatcherTask();
+          },
+        })
+        console.info(`Base ${id} directories are being watched.`);
+      })
+    )
+
+    // TODO 打印表格
+    console.info("DocManager initialized successfully.", result);
+
     console.info("DocBase started successfully.");
   };
 
+  // TODO 新安装插件立即扫描所有目录
   /**
    * 立即扫描所有目录
    */
-  scanAllNow = async (id: string) => {
-    console.info("Starting to scan all directories immediately...");
-    await this.#scan(this.dirs);
-    console.info("All directories scanned immediately.");
-  };
+  // scanAllNow = async (id: string) => {
+  //   console.info("Starting to scan all directories immediately...");
+  //   await this.#scan(id, this.dirs);
+  //   console.info("All directories scanned immediately.");
+  // };
+
+  // TODO 增加、删除、查询 知识库
 
   /**
    * 卸载文档加载器插件
@@ -463,7 +470,8 @@ export class DocBase {
     knowledgeId: string;
   }) => {
     console.info(`Searching for documents with query: ${params.q}`);
-    const results = await this.#docManager.search(query, opt);
+    const docManager = this.#docManagers.get(params.knowledgeId)
+    const results = await docManager.search(params);
     console.info(`Search completed. Found ${results.length} documents.`);
     return results;
   };

packages/core/src/DocManager.ts

@@ -550,9 +550,9 @@ export class DocManager {
    * @param hybrid - 启用向量搜索
    * @returns 返回搜索结果
    */
-  search = async (query: string, opts?: SearchParams) => {
-    console.debug(`Searching for query: ${query}`);
-    const result = await this.#docChunkIndex.search(query, opts);
+  search = async (opts: SearchParams) => {
+    console.debug(`Searching for query: ${opts.q}`);
+    const result = await this.#docChunkIndex.search(opts.q, opts);
     const hits = result.hits;
 
     // 查询后校验结果中引用到的本地文档是否存在，不存在则删除知识库内文档

packages/core/src/FSlayer.ts

@@ -15,22 +15,48 @@ export type Scanner = (params: {
   load: (paths: string[]) => Promise<void>;
 }) => Promise<void>;
 
-// 监视器
-export type Watcher = Pick<
-  DirectoryWatcher,
-  "getWatchedPaths" | "unwatch" | "watch"
->;
-
-export const FSLayer = ({
-  filter,
-  upsert,
-  remove,
-}: {
+interface WatchAction {
   filter: (path: string) => boolean;
   upsert: (path: string) => void;
   remove: (path: string) => void;
-}) => {
+}
+
+// 监视器
+export type Watcher = {
+  getWatchedPaths: () => string[];
+  unwatch: (path: string) => boolean;
+  watch: (path: string, actions: WatchAction) => void;
+}
+
+/**
+ * 文件系统监视勾子
+ * @param event 
+ * @param actions 
+ */
+const eventHook = (event: string, actions: WatchAction) => {
+  const { filter, upsert, remove } = actions
+  try {
+    const e = JSON.parse(event);
+    const type = e.event.type;
+    const path = e.event.paths[0];
+
+    // 过滤需要的路径
+    if (filter(path)) {
+      console.info(`[${type}] ${path}`);
+      if (type === "create" || type === "modify") {
+        upsert(slash(path));
+      } else if (type === "remove") {
+        remove(slash(path));
+      }
+    }
+  } catch (parseError) {
+    console.error(`Error parsing event data: ${parseError}`);
+  }
+}
+
+export const FSLayer = () => {
   const fd = new fdir().withBasePath();
+  const watchers = new Map<string, Watcher>();
 
   // 扫描器
   const scanner: Scanner = async ({ dirs, exts, load }) => {
@@ -48,26 +74,18 @@ export const FSLayer = ({
     }
   };
 
-  // 监视器
-  const watcher: DirectoryWatcher = DirectoryWatcher.new((_err, event) => {
-    try {
-      const e = JSON.parse(event);
-      const type = e.event.type;
-      const path = e.event.paths[0];
-
-      // 过滤需要的路径
-      if (filter(path)) {
-        console.info(`[${type}] ${path}`);
-        if (type === "create" || type === "modify") {
-          upsert(slash(path));
-        } else if (type === "remove") {
-          remove(slash(path));
-        }
-      }
-    } catch (parseError) {
-      console.error(`Error parsing event data: ${parseError}`);
+  const watcher = {
+    getWatchedPaths: () => watchers.keys().toArray(),
+    unwatch: (path: string) => {
+      watchers.get(path)?.unwatch(path)
+      return watchers.delete(path)
+    },
+    watch: (path: string, actions: WatchAction) => {
+      const watcher = DirectoryWatcher.new((_err, event) => eventHook(event, actions));
+      watcher.watch(path);
+      watchers.set(path, watcher)
     }
-  });
+  }
 
   return { watcher, scanner };
 };