feat: use s3 for global docs cache on all docs

Author: Patrick-Erichsen

CONTRIBUTING.md

@@ -26,7 +26,6 @@
     - [Writing Context Providers](#writing-context-providers)
     - [Adding an LLM Provider](#adding-an-llm-provider)
     - [Adding Models](#adding-models)
-    - [Adding Pre-indexed Documentation](#adding-pre-indexed-documentation)
   - [📐 Continue Architecture](#-continue-architecture)
     - [Continue VS Code Extension](#continue-vs-code-extension)
     - [Continue JetBrains Extension](#continue-jetbrains-extension)
@@ -212,10 +211,6 @@ While any model that works with a supported provider can be used with Continue,
 - LLM Providers: Since many providers use their own custom strings to identify models, you'll have to add the translation from Continue's model name (the one you added to `index.d.ts`) and the model string for each of these providers: [Ollama](./core/llm/llms/Ollama.ts), [Together](./core/llm/llms/Together.ts), and [Replicate](./core/llm/llms/Replicate.ts). You can find their full model lists here: [Ollama](https://ollama.ai/library), [Together](https://docs.together.ai/docs/inference-models), [Replicate](https://replicate.com/collections/streaming-language-models).
 - [Prompt Templates](./core/llm/index.ts) - In this file you'll find the `autodetectTemplateType` function. Make sure that for the model name you just added, this function returns the correct template type. This is assuming that the chat template for that model is already built in Continue. If not, you will have to add the template type and corresponding edit and chat templates.
 
-### Adding Pre-indexed Documentation
-
-Continue's @docs context provider lets you easily reference entire documentation sites and then uses embeddings to add the most relevant pages to context. To make the experience as smooth as possible, we pre-index many of the most popular documentation sites. If you'd like to add new documentation to this list, just add an object to the list in [preIndexedDocs.ts](./core/indexing/docs/preIndexedDocs.ts). `startUrl` is where the crawler will start and `rootUrl` will filter out any pages not on that site and under the path of `rootUrl`.
-
 ## 📐 Continue Architecture
 
 Continue consists of 2 parts that are split so that it can be extended to work in other IDEs as easily as possible:

core/context/providers/DocsContextProvider.ts

@@ -9,7 +9,6 @@ import {
   LoadSubmenuItemsArgs,
 } from "../..";
 import DocsService from "../../indexing/docs/DocsService";
-import preIndexedDocs from "../../indexing/docs/preIndexedDocs";
 
 import { INSTRUCTIONS_BASE_ITEM } from "./utils";
 
@@ -64,23 +63,12 @@ class DocsContextProvider extends BaseContextProvider {
     return chunksCopy;
   }
 
-  private _sortByPreIndexedDocs(
+  private _sortAlphabetically(
     submenuItems: ContextSubmenuItem[],
   ): ContextSubmenuItem[] {
-    // Sort submenuItems such that the objects with titles which don't occur in configs occur first, and alphabetized
+    // Sort submenu items alphabetically by title
     return submenuItems.sort((a, b) => {
-      const aTitleInConfigs = a.metadata?.preIndexed ?? false;
-      const bTitleInConfigs = b.metadata?.preIndexed ?? false;
-
-      // Primary criterion: Items not in configs come first
-      if (!aTitleInConfigs && bTitleInConfigs) {
-        return -1;
-      } else if (aTitleInConfigs && !bTitleInConfigs) {
-        return 1;
-      } else {
-        // Secondary criterion: Alphabetical order when both items are in the same category
-        return a.title.toString().localeCompare(b.title.toString());
-      }
+      return a.title.toString().localeCompare(b.title.toString());
     });
   }
 
@@ -165,46 +153,22 @@ class DocsContextProvider extends BaseContextProvider {
     }
     await docsService.isInitialized;
 
-    // Create map of docs url -> submenu item
-    const submenuItemsMap = new Map<string, ContextSubmenuItem>();
+    // Create an array to hold submenu items
+    const submenuItems: ContextSubmenuItem[] = [];
 
-    // Add custom docs from config
+    // Get all indexed docs from the database
     const docs = (await docsService.listMetadata()) ?? [];
     for (const { startUrl, title, favicon } of docs) {
-      submenuItemsMap.set(startUrl, {
+      submenuItems.push({
         title,
         id: startUrl,
         description: new URL(startUrl).hostname,
         icon: favicon,
       });
     }
 
-    // Add pre-indexed docs if supported
-    const canUsePreindexedDocs = await docsService.canUsePreindexedDocs();
-    if (canUsePreindexedDocs) {
-      for (const { startUrl, title } of Object.values(preIndexedDocs)) {
-        // Skip if overridden in config
-        if (docs.find((d) => d.startUrl === startUrl)) {
-          continue;
-        }
-        submenuItemsMap.set(startUrl, {
-          title,
-          id: startUrl,
-          description: new URL(startUrl).hostname,
-          metadata: {
-            preIndexed: true,
-          },
-        });
-      }
-    }
-
-    // Create array and sort if pre-indexed is supported
-    const submenuItems = Array.from(submenuItemsMap.values());
-    if (canUsePreindexedDocs) {
-      return this._sortByPreIndexedDocs(submenuItems);
-    }
-
-    return submenuItems;
+    // Sort alphabetically
+    return this._sortAlphabetically(submenuItems);
   }
 }
 

core/index.d.ts

@@ -164,7 +164,11 @@ export interface ILLM extends LLMOptions {
 }
 
 export interface ModelInstaller {
-  installModel(modelName: string, signal: AbortSignal, progressReporter?: (task: string, increment: number, total: number) => void): Promise<any>;
+  installModel(
+    modelName: string,
+    signal: AbortSignal,
+    progressReporter?: (task: string, increment: number, total: number) => void,
+  ): Promise<any>;
 }
 
 export type ContextProviderType = "normal" | "query" | "submenu";
@@ -240,7 +244,6 @@ export interface DocsIndexingDetails {
   config: SiteIndexingConfig;
   indexingStatus: IndexingStatus | undefined;
   chunks: Chunk[];
-  isPreIndexedDoc: boolean;
 }
 
 export interface IContextProvider {