Merge pull request #610 from hwchase17/nc/web-loader-async-caller

Add timeout and retry to web loaders

Author: nfcampos

langchain/src/document_loaders/cheerio_web_base.ts

@@ -2,24 +2,49 @@ import type { CheerioAPI, load as LoadT } from "cheerio";
 import { Document } from "../document.js";
 import { BaseDocumentLoader } from "./base.js";
 import type { DocumentLoader } from "./base.js";
+import { AsyncCaller, AsyncCallerParams } from "../util/async_caller.js";
+
+export interface WebBaseLoaderParams extends AsyncCallerParams {
+  /**
+   * The timeout in milliseconds for the fetch request. Defaults to 10s.
+   */
+  timeout?: number;
+}
 
 export class CheerioWebBaseLoader
   extends BaseDocumentLoader
   implements DocumentLoader
 {
-  constructor(public webPath: string) {
+  timeout: number;
+
+  caller: AsyncCaller;
+
+  constructor(public webPath: string, fields?: WebBaseLoaderParams) {
     super();
+    const { timeout, ...rest } = fields ?? {};
+    this.timeout = timeout ?? 10000;
+    this.caller = new AsyncCaller(rest);
   }
 
-  static async _scrape(url: string): Promise<CheerioAPI> {
+  static async _scrape(
+    url: string,
+    caller: AsyncCaller,
+    timeout: number | undefined
+  ): Promise<CheerioAPI> {
     const { load } = await CheerioWebBaseLoader.imports();
-    const response = await fetch(url);
+    const response = await caller.call(fetch, url, {
+      signal: timeout ? AbortSignal.timeout(timeout) : undefined,
+    });
     const html = await response.text();
     return load(html);
   }
 
   async scrape(): Promise<CheerioAPI> {
-    return CheerioWebBaseLoader._scrape(this.webPath);
+    return CheerioWebBaseLoader._scrape(
+      this.webPath,
+      this.caller,
+      this.timeout
+    );
   }
 
   async load(): Promise<Document[]> {

langchain/src/document_loaders/gitbook.ts

@@ -54,7 +54,7 @@ export class GitbookLoader extends CheerioWebBaseLoader {
     for (const path of relative_paths) {
       const url = this.webPath + path;
       console.log(`Fetching text from ${url}`);
-      const html = await GitbookLoader._scrape(url);
+      const html = await GitbookLoader._scrape(url, this.caller, this.timeout);
       documents.push(...this.loadPath(html, url));
     }
     console.log(`Fetched ${documents.length} documents.`);