increase timeout, better log

Author: sestinj

core/indexing/docs/crawlers/DefaultCrawler.ts

@@ -26,7 +26,7 @@ export class DefaultCrawler {
     });
     if (!resp.ok) {
       const text = await resp.text();
-      throw new Error(`Failed to crawl site: ${text}`);
+      throw new Error(`Failed to crawl site (${resp.status}): ${text}`);
     }
     const json = (await resp.json()) as PageData[];
     return json;

core/indexing/docs/crawlers/DocsCrawler.test.ts

@@ -152,11 +152,15 @@ describe("DocsCrawler", () => {
     );
 
     commonDocsSites.forEach((url) => {
-      test(`Default crawler common site: ${url}`, async () => {
-        const { pages, crawler } = await runCrawl(url);
-        expect(pages.length).toBeGreaterThanOrEqual(1);
-        expect(crawler).toEqual("default");
-      });
+      test(
+        `Default crawler common site: ${url}`,
+        async () => {
+          const { pages, crawler } = await runCrawl(url);
+          expect(pages.length).toBeGreaterThanOrEqual(1);
+          expect(crawler).toEqual("default");
+        },
+        TIMEOUT_MS,
+      );
     });
   });