From 656fc6ec97c48e718eb4131280899fde28293bbf Mon Sep 17 00:00:00 2001 From: ngxson Date: Thu, 27 Jun 2024 23:28:06 +0200 Subject: [PATCH 1/5] better llama.cpp snippet --- packages/tasks/src/local-apps.ts | 63 +++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 24aa971671..1a69155866 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -1,6 +1,12 @@ import type { ModelData } from "./model-data"; import type { PipelineType } from "./pipelines"; +type Snippet = { + title: string; + setup: string; + command: string; +}; + /** * Elements configurable by a local app. */ @@ -39,7 +45,7 @@ export type LocalApp = { * And if not (mostly llama.cpp), snippet to copy/paste in your terminal * Support the placeholder {{GGUF_FILE}} that will be replaced by the gguf file path or the list of available files. */ - snippet: (model: ModelData) => string | string[]; + snippet: (model: ModelData) => Snippet | Snippet[]; } ); @@ -47,28 +53,41 @@ function isGgufModel(model: ModelData) { return model.tags.includes("gguf"); } -const snippetLlamacpp = (model: ModelData): string[] => { +const snippetLlamacpp = (model: ModelData): Snippet[] => { + const command = (binary: string) => + [ + "# Load and run the model:", + `${binary} \\`, + ' --hf-repo "${model.id}" \\', + " --hf-file {{GGUF_FILE}} \\", + ' -p "You are a helpful assistant" \\', + " --conversation", + ].join("\n"); return [ - `# Option 1: use llama.cpp with brew -brew install llama.cpp - -# Load and run the model -llama \\ - --hf-repo "${model.id}" \\ - --hf-file {{GGUF_FILE}} \\ - -p "I believe the meaning of life is" \\ - -n 128`, - `# Option 2: build llama.cpp from source with curl support -git clone https://github.com/ggerganov/llama.cpp.git -cd llama.cpp -LLAMA_CURL=1 make - -# Load and run the model -./main \\ - --hf-repo "${model.id}" \\ - -m {{GGUF_FILE}} \\ - -p "I believe the meaning of life is" \\ - -n 128`, + { + title: "Install from brew", + setup: "brew install llama.cpp", + command: command("llama-cli"), + }, + { + title: "Use pre-built binary", + setup: [ + // prettier-ignore + "# Download pre-built binary from:", + "# https://github.com/ggerganov/llama.cpp/releases", + ].join("\n"), + command: command("./llama-cli"), + }, + { + title: "Build from source code", + setup: [ + // prettier-ignore + "git clone https://github.com/ggerganov/llama.cpp.git", + "cd llama.cpp", + "LLAMA_CURL=1 make", + ].join("\n"), + command: command("./llama-cli"), + }, ]; }; From d8dafa28eb372eee514ec56f9fb069aed187342d Mon Sep 17 00:00:00 2001 From: ngxson Date: Fri, 28 Jun 2024 00:20:12 +0200 Subject: [PATCH 2/5] make single target --- packages/tasks/src/local-apps.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 1a69155866..d815b3add5 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -81,10 +81,13 @@ const snippetLlamacpp = (model: ModelData): Snippet[] => { { title: "Build from source code", setup: [ - // prettier-ignore + "# Install required packages", + "sudo apt install build-essential libcurl4-openssl-dev", + "", + "# Clone and build", "git clone https://github.com/ggerganov/llama.cpp.git", "cd llama.cpp", - "LLAMA_CURL=1 make", + "LLAMA_CURL=1 make -j llama-cli", ].join("\n"), command: command("./llama-cli"), }, From 98a263767c4fd37b000ba94bc92dfa5ea82c6399 Mon Sep 17 00:00:00 2001 From: ngxson Date: Fri, 28 Jun 2024 15:16:24 +0200 Subject: [PATCH 3/5] fix lint --- packages/tasks/src/local-apps.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 0639de66d3..26c6f9f6a9 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -1,11 +1,11 @@ import type { ModelData } from "./model-data"; import type { PipelineType } from "./pipelines"; -type Snippet = { +interface Snippet { title: string; setup: string; command: string; -}; +} /** * Elements configurable by a local app. @@ -53,13 +53,13 @@ function isGgufModel(model: ModelData) { return model.tags.includes("gguf"); } -const snippetLlamacpp = (model: ModelData): Snippet[] => { +const snippetLlamacpp = (model: ModelData, filepath?: string): Snippet[] => { const command = (binary: string) => [ "# Load and run the model:", `${binary} \\`, - ' --hf-repo "${model.id}" \\', - " --hf-file {{GGUF_FILE}} \\", + ` --hf-repo "${model.id}" \\`, + ` --hf-file ${filepath ?? "{{GGUF_FILE}}"} \\`, ' -p "You are a helpful assistant" \\', " --conversation", ].join("\n"); From 3d5db7d20e46b0d8e7569fbfb8023914dbb7773b Mon Sep 17 00:00:00 2001 From: ngxson Date: Fri, 28 Jun 2024 15:40:20 +0200 Subject: [PATCH 4/5] remove guide to install deps --- packages/tasks/src/local-apps.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 56b08fd38b..a20320168d 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -81,13 +81,9 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): Snippet[] => { { title: "Build from source code", setup: [ - "# Install required packages", - "sudo apt install build-essential libcurl4-openssl-dev", - "", - "# Clone and build", "git clone https://github.com/ggerganov/llama.cpp.git", "cd llama.cpp", - "LLAMA_CURL=1 make -j llama-cli", + "LLAMA_CURL=1 make llama-cli", ].join("\n"), command: command("./llama-cli"), }, From 7983478d5f136b2e101752b229fa906f06c719f1 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 6 Aug 2024 14:04:02 +0200 Subject: [PATCH 5/5] resolve review comments --- packages/tasks/src/index.ts | 2 +- packages/tasks/src/local-apps.ts | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/tasks/src/index.ts b/packages/tasks/src/index.ts index 213b324753..cefebfaeaa 100644 --- a/packages/tasks/src/index.ts +++ b/packages/tasks/src/index.ts @@ -47,7 +47,7 @@ export { snippets }; export { SKUS, DEFAULT_MEMORY_OPTIONS } from "./hardware"; export type { HardwareSpec, SkuType } from "./hardware"; export { LOCAL_APPS } from "./local-apps"; -export type { LocalApp, LocalAppKey } from "./local-apps"; +export type { LocalApp, LocalAppKey, LocalAppSnippet } from "./local-apps"; export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries"; export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries"; diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index a20320168d..759d32ef71 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -1,10 +1,19 @@ import type { ModelData } from "./model-data"; import type { PipelineType } from "./pipelines"; -interface Snippet { +export interface LocalAppSnippet { + /** + * Title of the snippet + */ title: string; - setup: string; - command: string; + /** + * Optional setup guide + */ + setup?: string; + /** + * Content (or command) to be run + */ + content: string; } /** @@ -45,7 +54,7 @@ export type LocalApp = { * And if not (mostly llama.cpp), snippet to copy/paste in your terminal * Support the placeholder {{GGUF_FILE}} that will be replaced by the gguf file path or the list of available files. */ - snippet: (model: ModelData, filepath?: string) => string | string[] | Snippet | Snippet[]; + snippet: (model: ModelData, filepath?: string) => string | string[] | LocalAppSnippet | LocalAppSnippet[]; } ); @@ -53,7 +62,7 @@ function isGgufModel(model: ModelData) { return model.tags.includes("gguf"); } -const snippetLlamacpp = (model: ModelData, filepath?: string): Snippet[] => { +const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] => { const command = (binary: string) => [ "# Load and run the model:", @@ -67,7 +76,7 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): Snippet[] => { { title: "Install from brew", setup: "brew install llama.cpp", - command: command("llama-cli"), + content: command("llama-cli"), }, { title: "Use pre-built binary", @@ -76,7 +85,7 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): Snippet[] => { "# Download pre-built binary from:", "# https://github.com/ggerganov/llama.cpp/releases", ].join("\n"), - command: command("./llama-cli"), + content: command("./llama-cli"), }, { title: "Build from source code", @@ -85,7 +94,7 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): Snippet[] => { "cd llama.cpp", "LLAMA_CURL=1 make llama-cli", ].join("\n"), - command: command("./llama-cli"), + content: command("./llama-cli"), }, ]; };