feat: perform FTS on file paths

.github/workflows/jetbrains-build.yaml

@@ -152,11 +152,11 @@ jobs:
         id: artifact
         shell: bash
         run: |
-          cd ${{ github.workspace }}/extensions/intellij/build/distributions
+          cd ../../extensions/intellij/build/distributions
           FILENAME=`ls *.zip`
           unzip "$FILENAME" -d content
 
-          echo "filename=${FILENAME:0:-4}" >> $GITHUB_OUTPUT
+          echo "filename=${FILENAME%.????}" >> $GITHUB_OUTPUT
 
       # Store already-built plugin as an artifact for downloading
       - name: Upload artifact

core/autocomplete/completionProvider.ts

@@ -28,6 +28,7 @@ import AutocompleteLruCache from "./cache.js";
 import {
   noFirstCharNewline,
   onlyWhitespaceAfterEndOfLine,
+  stopAtStopTokens,
 } from "./charStream.js";
 import {
   constructAutocompletePrompt,
@@ -582,6 +583,7 @@ export class CompletionProvider {
         suffix,
         filename,
         reponame,
+        language: lang.name,
       });
     } else {
       // Let the template function format snippets
@@ -662,6 +664,7 @@ export class CompletionProvider {
         lang.endOfLine,
         fullStop,
       );
+      charGenerator = stopAtStopTokens(charGenerator, stop);
       charGenerator = this.bracketMatchingService.stopOnUnmatchedClosingBracket(
         charGenerator,
         prefix,

core/autocomplete/languages.ts

@@ -1,6 +1,7 @@
 import type { LineFilter } from "./lineStream.js";
 
 export interface AutocompleteLanguageInfo {
+  name: string;
   topLevelKeywords: string[];
   singleLineComment: string;
   endOfLine: string[];
@@ -14,13 +15,15 @@ export interface AutocompleteLanguageInfo {
 
 // TypeScript
 export const Typescript = {
+  name: "TypeScript",
   topLevelKeywords: ["function", "class", "module", "export", "import"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Python
 export const Python = {
+  name: "Python",
   // """"#" is for .ipynb files, where we add '"""' surrounding markdown blocks.
   // This stops the model from trying to complete the start of a new markdown block
   topLevelKeywords: ["def", "class", '"""#'],
@@ -30,55 +33,63 @@ export const Python = {
 
 // Java
 export const Java = {
+  name: "Java",
   topLevelKeywords: ["class", "function"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // C++
 export const Cpp = {
+  name: "C++",
   topLevelKeywords: ["class", "namespace", "template"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // C#
 export const CSharp = {
+  name: "C#",
   topLevelKeywords: ["class", "namespace", "void"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // C
 export const C = {
+  name: "C",
   topLevelKeywords: ["if", "else", "while", "for", "switch", "case"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Scala
 export const Scala = {
+  name: "Scala",
   topLevelKeywords: ["def", "val", "var", "class", "object", "trait"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Go
 export const Go = {
+  name: "Go",
   topLevelKeywords: ["func", "package", "import", "type"],
   singleLineComment: "//",
   endOfLine: [],
 };
 
 // Rust
 export const Rust = {
+  name: "Rust",
   topLevelKeywords: ["fn", "mod", "pub", "struct", "enum", "trait"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Haskell
 export const Haskell = {
+  name: "Haskell",
   topLevelKeywords: [
     "data",
     "type",
@@ -95,48 +106,55 @@ export const Haskell = {
 
 // PHP
 export const PHP = {
+  name: "PHP",
   topLevelKeywords: ["function", "class", "namespace", "use"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Ruby on Rails
 export const RubyOnRails = {
+  name: "Ruby on Rails",
   topLevelKeywords: ["def", "class", "module"],
   singleLineComment: "#",
   endOfLine: [],
 };
 
 // Swift
 export const Swift = {
+  name: "Swift",
   topLevelKeywords: ["func", "class", "struct", "import"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Kotlin
 export const Kotlin = {
+  name: "Kotlin",
   topLevelKeywords: ["fun", "class", "package", "import"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Ruby
 export const Ruby = {
+  name: "Ruby",
   topLevelKeywords: ["class", "module", "def"],
   singleLineComment: "#",
   endOfLine: [],
 };
 
 // Clojure
 export const Clojure = {
+  name: "Clojure",
   topLevelKeywords: ["def", "fn", "let", "do", "if", "defn", "ns", "defmacro"],
   singleLineComment: ";",
   endOfLine: [],
 };
 
 // Julia
 export const Julia = {
+  name: "Julia",
   topLevelKeywords: [
     "function",
     "macro",
@@ -155,6 +173,7 @@ export const Julia = {
 
 // F#
 export const FSharp = {
+  name: "F#",
   topLevelKeywords: [
     "let",
     "type",
@@ -173,6 +192,7 @@ export const FSharp = {
 
 // R
 export const R = {
+  name: "R",
   topLevelKeywords: [
     "function",
     "if",
@@ -189,13 +209,15 @@ export const R = {
 
 // Dart
 export const Dart = {
+  name: "Dart",
   topLevelKeywords: ["class", "import", "void", "enum"],
   singleLineComment: "//",
   endOfLine: [";"],
 };
 
 // Solidity
 export const Solidity = {
+  name: "Solidity",
   topLevelKeywords: [
     "contract",
     "event",
@@ -218,6 +240,7 @@ export const Solidity = {
 
 // YAML
 export const YAML: AutocompleteLanguageInfo = {
+  name: "YAML",
   topLevelKeywords: [],
   singleLineComment: "#",
   endOfLine: [],
@@ -258,6 +281,7 @@ export const YAML: AutocompleteLanguageInfo = {
 };
 
 export const Markdown: AutocompleteLanguageInfo = {
+  name: "Markdown",
   topLevelKeywords: [],
   singleLineComment: "",
   endOfLine: [],

core/autocomplete/retrieval.ts

@@ -10,13 +10,11 @@ export async function fullTextRetrieve(
   const searchStrings = prefix.split("\n").slice(-3);
   const results: Chunk[] = [];
   searchStrings.forEach(async (searchString) => {
-    const chunks = await index.retrieve(
-      [indexTag],
-      searchString,
-      3,
-      undefined,
-      undefined,
-    );
+    const chunks = await index.retrieve({
+      tags: [indexTag],
+      text: searchString,
+      n: 3,
+    });
     results.push(...chunks);
   });
   return results;

core/autocomplete/templates.ts

@@ -155,9 +155,34 @@ const deepseekFimTemplate: AutocompleteTemplate = {
   },
 };
 
-const deepseekFimTemplateWrongPipeChar: AutocompleteTemplate = {
-  template: "<|fim▁begin|>{{{prefix}}}<|fim▁hole|>{{{suffix}}}<|fim▁end|>",
-  completionOptions: { stop: ["<|fim▁begin|>", "<|fim▁hole|>", "<|fim▁end|>"] },
+// https://github.com/THUDM/CodeGeeX4/blob/main/guides/Infilling_guideline.md
+const codegeexFimTemplate: AutocompleteTemplate = {
+  template: (
+    prefix: string,
+    suffix: string,
+    filepath: string,
+    reponame: string,
+    snippets: AutocompleteSnippet[],
+  ): string => {
+    const relativePaths = shortestRelativePaths([
+      ...snippets.map((snippet) => snippet.filepath),
+      filepath,
+    ]);
+    const baseTemplate = `###PATH:${relativePaths[relativePaths.length - 1]}\n###LANGUAGE:\n###MODE:\n<｜code_suffix｜>${suffix}<｜code_prefix｜>${prefix}<｜code_middle｜>`;
+    if (snippets.length == 0)
+    {
+      return baseTemplate;
+    }
+    const references =
+      `###REFERENCE:\n${snippets
+      .map((snippet, i) => `###PATH:${relativePaths[i]}\n${snippet.contents}\n`)
+      .join("###REFERENCE:\n")}`;
+    const prompt = `${references}\n${baseTemplate}`;
+    return prompt;
+  },
+  completionOptions: {
+    stop: ["<｜code_suffix｜>", "<｜code_prefix｜>", "<｜code_middle｜>"],
+  },
 };
 
 const gptAutocompleteTemplate: AutocompleteTemplate = {
@@ -307,6 +332,10 @@ export function getTemplateForModel(model: string): AutocompleteTemplate {
     return deepseekFimTemplate;
   }
 
+  if (lowerCaseModel.includes("codegeex")) {
+    return codegeexFimTemplate;
+  }
+
   if (
     lowerCaseModel.includes("gpt") ||
     lowerCaseModel.includes("davinci-002") ||