continuedev
diff --git a/‎core/config/types.ts
+1 b/‎core/config/types.ts
+1
diff --git a/‎core/context/rerankers/index.ts
+2 b/‎core/context/rerankers/index.ts
+2
diff --git a/‎core/context/rerankers/tei.ts
+49 b/‎core/context/rerankers/tei.ts
+49
diff --git a/‎core/diff/streamDiff.ts
+14-12 b/‎core/diff/streamDiff.ts
+14-12
diff --git a/‎core/diff/util.ts
+2 b/‎core/diff/util.ts
+2
diff --git a/‎core/index.d.ts
+5-2 b/‎core/index.d.ts
+5-2
diff --git a/‎core/indexing/docs/DocsService.ts
+13-5 b/‎core/indexing/docs/DocsService.ts
+13-5
diff --git a/‎core/indexing/ignore.ts
+1 b/‎core/indexing/ignore.ts
+1
diff --git a/‎core/indexing/walkDir.ts
+3-6 b/‎core/indexing/walkDir.ts
+3-6
diff --git a/‎core/llm/autodetect.ts
+1 b/‎core/llm/autodetect.ts
+1
diff --git a/‎core/llm/constants.ts
+1 b/‎core/llm/constants.ts
+1
diff --git a/‎core/llm/llms/Azure.ts
+8 b/‎core/llm/llms/Azure.ts
+8
diff --git a/‎core/llm/llms/OpenAI.ts
+1 b/‎core/llm/llms/OpenAI.ts
+1
diff --git a/‎core/llm/llms/index.ts
+2 b/‎core/llm/llms/index.ts
+2
diff --git a/‎core/llm/llms/stubs/ContinueProxy.ts
+1 b/‎core/llm/llms/stubs/ContinueProxy.ts
+1
diff --git a/‎core/package-lock.json
+10 b/‎core/package-lock.json
+10
diff --git a/‎core/package.json
+1 b/‎core/package.json
+1
diff --git a/‎core/protocol/ide.ts
+1 b/‎core/protocol/ide.ts
+1
@@ -523,6 +523,7 @@ declare global {
     | "gpt-4-32k"
     | "gpt-4-turbo"
     | "gpt-4o"
+    | "gpt-4o-mini"
     | "gpt-4-turbo-preview"
     | "gpt-4-vision-preview"
     // Mistral
 
@@ -3,10 +3,12 @@ import { CohereReranker } from "./cohere.js";
 import { FreeTrialReranker } from "./freeTrial.js";
 import { LLMReranker } from "./llm.js";
 import { VoyageReranker } from "./voyage.js";
+import {HuggingFaceTEIReranker} from "./tei.js"
 
 export const AllRerankers: { [key in RerankerName]: any } = {
   cohere: CohereReranker,
   llm: LLMReranker,
   voyage: VoyageReranker,
   "free-trial": FreeTrialReranker,
+  "huggingface-tei": HuggingFaceTEIReranker
 };
@@ -0,0 +1,49 @@
+import fetch from "node-fetch";
+import { Chunk, Reranker } from "../../index.js";
+
+export class HuggingFaceTEIReranker implements Reranker {
+  name = "huggingface-tei";
+
+  static defaultOptions = {
+    apiBase: "http://localhost:8080",
+    truncate: true,
+    truncation_direction: "Right"
+  };
+
+  constructor(
+    private readonly params: {
+      apiBase?: string;
+      truncate?: boolean;
+      truncation_direction?: string;
+    },
+  ) {}
+
+  async rerank(query: string, chunks: Chunk[]): Promise<number[]> {
+    let apiBase = this.params.apiBase ?? HuggingFaceTEIReranker.defaultOptions.apiBase;
+    if (!apiBase.endsWith("/")) {
+      apiBase += "/";
+    }
+
+    const resp = await fetch(new URL("rerank", apiBase), {
+      method: "POST",
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        query: query,
+        return_text: false,
+        raw_scores: false,
+        texts: chunks.map((chunk) => chunk.content),
+        truncation_direction: this.params.truncation_direction ?? HuggingFaceTEIReranker.defaultOptions.truncation_direction,
+        truncate: this.params.truncate ?? HuggingFaceTEIReranker.defaultOptions.truncate
+      }),
+    });
+
+    if (!resp.ok) {
+      throw new Error(await resp.text());
+    }
+
+    const data = (await resp.json()) as any;
+    // Resort into original order and extract scores
+    const results = data.sort((a: any, b: any) => a.index - b.index);
+    return results.map((result: any) => result.score);
+  }
+}
@@ -1,35 +1,37 @@
-import { DiffLine } from "../index.js";
+import { DiffLine, DiffLineType } from "../index.js";
 import { LineStream, matchLine } from "./util.js";
 
 /**
  * https://blog.jcoglan.com/2017/02/12/the-myers-diff-algorithm-part-1/
  * Invariants:
  * - new + same = newLines.length
- * - old + same = oldLines.length
+ * - old + same = oldLinesCopy.length
  * ^ (above two guarantee that all lines get represented)
  * - Lines are always output in order, at least among old and new separately
  */
 export async function* streamDiff(
   oldLines: string[],
   newLines: LineStream,
 ): AsyncGenerator<DiffLine> {
+  const oldLinesCopy = [...oldLines];
+
   // If one indentation mistake is made, others are likely. So we are more permissive about matching
   let seenIndentationMistake = false;
 
   let newLineResult = await newLines.next();
 
-  while (oldLines.length > 0 && !newLineResult.done) {
+  while (oldLinesCopy.length > 0 && !newLineResult.done) {
     const { matchIndex, isPerfectMatch, newLine } = matchLine(
       newLineResult.value,
-      oldLines,
+      oldLinesCopy,
       seenIndentationMistake,
     );
 
     if (!seenIndentationMistake && newLineResult.value !== newLine) {
       seenIndentationMistake = true;
     }
 
-    let type: DiffLine["type"];
+    let type: DiffLineType;
 
     let isLineRemoval = false;
     const isNewLine = matchIndex === -1;
@@ -39,7 +41,7 @@ export async function* streamDiff(
     } else {
       // Insert all deleted lines before match
       for (let i = 0; i < matchIndex; i++) {
-        yield { type: "old", line: oldLines.shift()! };
+        yield { type: "old", line: oldLinesCopy.shift()! };
       }
 
       type = isPerfectMatch ? "same" : "old";
@@ -51,13 +53,13 @@ export async function* streamDiff(
         break;
 
       case "same":
-        yield { type, line: oldLines.shift()! };
+        yield { type, line: oldLinesCopy.shift()! };
         break;
 
       case "old":
-        yield { type, line: oldLines.shift()! };
+        yield { type, line: oldLinesCopy.shift()! };
 
-        if (oldLines[0] !== newLine) {
+        if (oldLinesCopy[0] !== newLine) {
           yield { type: "new", line: newLine };
         } else {
           isLineRemoval = true;
@@ -75,13 +77,13 @@ export async function* streamDiff(
   }
 
   // Once at the edge, only one choice
-  if (newLineResult.done && oldLines.length > 0) {
-    for (const oldLine of oldLines) {
+  if (newLineResult.done && oldLinesCopy.length > 0) {
+    for (const oldLine of oldLinesCopy) {
       yield { type: "old", line: oldLine };
     }
   }
 
-  if (!newLineResult.done && oldLines.length === 0) {
+  if (!newLineResult.done && oldLinesCopy.length === 0) {
     yield { type: "new", line: newLineResult.value };
     for await (const newLine of newLines) {
       yield { type: "new", line: newLine };
 
@@ -38,6 +38,8 @@ function linesMatch(lineA: string, lineB: string, linesBetween = 0): boolean {
 }
 
 /**
+ * Used to find a match for a new line in an array of old lines.
+ *
  * Return the index of the first match and whether it is a perfect match
  * Also return a version of the line with correct indentation if needs fixing
  */
 
@@ -374,8 +374,10 @@ export type CustomLLM = RequireAtLeastOne<
 
 // IDE
 
+export type DiffLineType = "new" | "old" | "same";
+
 export interface DiffLine {
-  type: "new" | "old" | "same";
+  type: DiffLineType;
   line: string;
 }
 
@@ -596,6 +598,7 @@ export type ModelName =
   | "gpt-3.5-turbo-0613"
   | "gpt-4-32k"
   | "gpt-4o"
+  | "gpt-4o-mini"
   | "gpt-4-turbo"
   | "gpt-4-turbo-preview"
   | "gpt-4-vision-preview"
@@ -756,7 +759,7 @@ export interface EmbeddingsProvider {
   embed(chunks: string[]): Promise<number[][]>;
 }
 
-export type RerankerName = "cohere" | "voyage" | "llm" | "free-trial";
+export type RerankerName = "cohere" | "voyage" | "llm" | "free-trial" | "huggingface-tei";
 
 export interface RerankerDescription {
   name: RerankerName;
 
@@ -214,7 +214,7 @@ export class DocsService {
       return;
     }
 
-    if (!reIndex && await this.has(startUrl.toString())) {
+    if (!reIndex && (await this.has(startUrl.toString()))) {
       yield {
         progress: 1,
         desc: "Already indexed",
@@ -275,11 +275,13 @@ export class DocsService {
 
       try {
         const subpathEmbeddings = await embeddingsProvider.embed(
-          chunkArticle(article, embeddingsProvider.maxChunkSize).map((chunk) => {
-            chunks.push(chunk);
+          chunkArticle(article, embeddingsProvider.maxChunkSize).map(
+            (chunk) => {
+              chunks.push(chunk);
 
-            return chunk.content;
-          }),
+              return chunk.content;
+            },
+          ),
         );
 
         embeddings.push(...subpathEmbeddings);
@@ -296,6 +298,12 @@ export class DocsService {
       status: "indexing",
     };
 
+    // Clear old index if re-indexing.
+    if (reIndex) {
+      console.log("Deleting old embeddings");
+      await this.delete(startUrl.toString());
+    }
+
     await this.add(siteIndexingConfig.title, startUrl, chunks, embeddings);
     this.docsIndexingQueue.delete(startUrl.toString());
 
 
@@ -64,6 +64,7 @@ export const DEFAULT_IGNORE_FILETYPES = [
   ".gitkeep",
   ".continueignore",
   "config.json",
+  ".csv",
   // "*.prompt", // can be incredibly confusing for the LLM to have another set of instructions injected into the prompt
 ];
 export const defaultIgnoreFile = ignore().add(DEFAULT_IGNORE_FILETYPES);
 
@@ -1,8 +1,8 @@
 import { EventEmitter } from "events";
 import { Minimatch } from "minimatch";
 import path from "node:path";
-import { FileType, IDE } from "..";
-import { DEFAULT_IGNORE_DIRS, DEFAULT_IGNORE_FILETYPES } from "./ignore";
+import { FileType, IDE } from "../index.js";
+import { DEFAULT_IGNORE_DIRS, DEFAULT_IGNORE_FILETYPES } from "./ignore.js";
 
 export interface WalkerOptions {
   isSymbolicLink?: boolean;
@@ -34,10 +34,7 @@ class Walker extends EventEmitter {
   sawError: boolean;
   exact: boolean | undefined;
   onlyDirs: boolean | undefined;
-  constructor(
-    opts: WalkerOptions = {},
-    protected readonly ide: IDE,
-  ) {
+  constructor(opts: WalkerOptions = {}, protected readonly ide: IDE) {
     super(opts as any);
     this.isSymbolicLink = opts.isSymbolicLink || false;
     this.path = opts.path || process.cwd();
 
@@ -61,6 +61,7 @@ const MODEL_SUPPORTS_IMAGES: string[] = [
   "llava",
   "gpt-4-turbo",
   "gpt-4o",
+  "gpt-4o-mini",
   "gpt-4-vision",
   "claude-3",
   "gemini-ultra",
 
@@ -20,6 +20,7 @@ const CONTEXT_LENGTH_FOR_MODEL: { [name: string]: number } = {
   "gpt-4-32k": GPT_4_CTX_LEN,
   "gpt-4-turbo-preview": GPT_4_CTX_LEN,
   "gpt-4o": GPT_4_CTX_LEN,
+  "gpt-4o-mini": GPT_4_CTX_LEN,
   "gpt-4-vision": GPT_4_CTX_LEN,
   "gpt-4-0125-preview": GPT_4_CTX_LEN,
   "gpt-4-1106-preview": GPT_4_CTX_LEN,
 
@@ -0,0 +1,8 @@
+import { ModelProvider } from "../../index.js";
+import OpenAI from "./OpenAI.js";
+
+class Azure extends OpenAI {
+  static providerName: ModelProvider = "azure";
+}
+
+export default Azure;
@@ -36,6 +36,7 @@ const CHAT_ONLY_MODELS = [
   "gpt-4-vision",
   "gpt-4-0125-preview",
   "gpt-4-1106-preview",
+  "gpt-4o-mini",
 ];
 
 class OpenAI extends BaseLLM {
 
@@ -27,6 +27,7 @@ import LlamaCpp from "./LlamaCpp.js";
 import Llamafile from "./Llamafile.js";
 import Mistral from "./Mistral.js";
 import Msty from "./Msty.js";
+import Azure from "./Azure.js";
 import Ollama from "./Ollama.js";
 import OpenAI from "./OpenAI.js";
 import Replicate from "./Replicate.js";
@@ -124,6 +125,7 @@ const LLMs = [
   Cloudflare,
   Deepseek,
   Msty,
+  Azure,
 ];
 
 export async function llmFromDescription(
 
@@ -18,6 +18,7 @@ class ContinueProxy extends OpenAI {
   static defaultOptions: Partial<LLMOptions> = {
     apiBase:
       "https://control-plane-api-service-i3dqylpbqa-uc.a.run.app/model-proxy/v1",
+    useLegacyCompletionsEndpoint: false,
   };
 
   supportsFim(): boolean {
 
@@ -24,6 +24,7 @@
     "eslint": "^8",
     "eslint-plugin-import": "^2.29.1",
     "jest": "^29.7.0",
+    "myers-diff": "^2.1.0",
     "onnxruntime-common": "1.14.0",
     "onnxruntime-web": "1.14.0",
     "ts-jest": "^29.1.1"
 
@@ -80,6 +80,7 @@ export type ToIdeFromWebviewOrCoreProtocol = {
     { silent: boolean },
     ControlPlaneSessionInfo | undefined,
   ];
+  logoutOfControlPlane: [undefined, void];
   pathSep: [undefined, string];
 };
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,8 @@ function linesMatch(lineA: string, lineB: string, linesBetween = 0): boolean {`
`38`	`38`	`}`
`39`	`39`
`40`	`40`	`/**`
	`41`	`+ * Used to find a match for a new line in an array of old lines.`
	`42`	`+ *`
`41`	`43`	`* Return the index of the first match and whether it is a perfect match`
`42`	`44`	`* Also return a version of the line with correct indentation if needs fixing`
`43`	`45`	`*/`