Skip to content

Commit 1d36b36

Browse files
slarenngxson
slaren
and
ngxson
authored
llama : move end-user examples to tools directory (ggml-org#13249)
* llama : move end-user examples to tools directory --------- Co-authored-by: Xuan Son Nguyen <[email protected]>
1 parent b344439 commit 1d36b36

File tree

213 files changed

+226
-190
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

213 files changed

+226
-190
lines changed

‎.editorconfig

+4-4
Original file line numberDiff line numberDiff line change
@@ -21,23 +21,23 @@ indent_style = tab
2121
[prompts/*.txt]
2222
insert_final_newline = unset
2323

24-
[examples/server/public/*]
24+
[tools/server/public/*]
2525
indent_size = 2
2626

27-
[examples/server/public/deps_*]
27+
[tools/server/public/deps_*]
2828
trim_trailing_whitespace = unset
2929
indent_style = unset
3030
indent_size = unset
3131

32-
[examples/server/deps_*]
32+
[tools/server/deps_*]
3333
trim_trailing_whitespace = unset
3434
indent_style = unset
3535
indent_size = unset
3636

3737
[examples/llama.swiftui/llama.swiftui.xcodeproj/*]
3838
indent_style = tab
3939

40-
[examples/cvector-generator/*.txt]
40+
[tools/cvector-generator/*.txt]
4141
trim_trailing_whitespace = unset
4242
insert_final_newline = unset
4343

‎.flake8

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
max-line-length = 125
33
ignore = E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503
44
exclude =
5-
# Do not traverse examples
5+
# Do not traverse examples and tools
66
examples,
7+
tools,
78
# Do not include package initializers
89
__init__.py,
910
# No need to traverse our git directory

‎.github/labeler.yml

+4-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@ build:
4545
- CMakePresets.json
4646
examples:
4747
- changed-files:
48-
- any-glob-to-any-file: examples/**
48+
- any-glob-to-any-file:
49+
- examples/**
50+
- tools/**
4951
devops:
5052
- changed-files:
5153
- any-glob-to-any-file:
@@ -70,7 +72,7 @@ android:
7072
server:
7173
- changed-files:
7274
- any-glob-to-any-file:
73-
- examples/server/**
75+
- tools/server/**
7476
ggml:
7577
- changed-files:
7678
- any-glob-to-any-file:

‎.github/workflows/bench.yml.disabled

+15-15
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ on:
2727
push:
2828
branches:
2929
- master
30-
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'examples/server/*.h*', 'examples/server/*.cpp']
30+
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'tools/server/*.h*', 'tools/server/*.cpp']
3131
pull_request_target:
3232
types: [opened, synchronize, reopened]
33-
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'examples/server/*.h*', 'examples/server/*.cpp']
33+
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'tools/server/*.h*', 'tools/server/*.cpp']
3434
schedule:
3535
- cron: '04 2 * * *'
3636

@@ -69,7 +69,7 @@ jobs:
6969
- name: Install python env
7070
id: pipenv
7171
run: |
72-
cd examples/server/bench
72+
cd tools/server/bench
7373
python3 -m venv venv
7474
source venv/bin/activate
7575
pip install -r requirements.txt
@@ -79,7 +79,7 @@ jobs:
7979
run: |
8080
wget --quiet https://github.com/prometheus/prometheus/releases/download/v2.51.0/prometheus-2.51.0.linux-amd64.tar.gz
8181
tar xzf prometheus*.tar.gz --strip-components=1
82-
./prometheus --config.file=examples/server/bench/prometheus.yml &
82+
./prometheus --config.file=tools/server/bench/prometheus.yml &
8383
while ! nc -z localhost 9090; do
8484
sleep 0.1
8585
done
@@ -92,7 +92,7 @@ jobs:
9292
- name: Install k6 and xk6-sse
9393
id: k6_installation
9494
run: |
95-
cd examples/server/bench
95+
cd tools/server/bench
9696
go install go.k6.io/xk6/cmd/xk6@latest
9797
xk6 build master \
9898
--with github.com/phymbert/xk6-sse
@@ -116,7 +116,7 @@ jobs:
116116
- name: Download the dataset
117117
id: download_dataset
118118
run: |
119-
cd examples/server/bench
119+
cd tools/server/bench
120120
wget --quiet https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
121121

122122
- name: Server bench
@@ -126,7 +126,7 @@ jobs:
126126
run: |
127127
set -eux
128128

129-
cd examples/server/bench
129+
cd tools/server/bench
130130
source venv/bin/activate
131131
python bench.py \
132132
--runner-label ${{ env.RUNNER_LABEL }} \
@@ -157,9 +157,9 @@ jobs:
157157
name: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }}
158158
compression-level: 9
159159
path: |
160-
examples/server/bench/*.jpg
161-
examples/server/bench/*.json
162-
examples/server/bench/*.log
160+
tools/server/bench/*.jpg
161+
tools/server/bench/*.json
162+
tools/server/bench/*.log
163163

164164
- name: Commit status
165165
uses: Sibz/github-status-action@v1
@@ -178,17 +178,17 @@ jobs:
178178
with:
179179
client_id: ${{secrets.IMGUR_CLIENT_ID}}
180180
path: |
181-
examples/server/bench/prompt_tokens_seconds.jpg
182-
examples/server/bench/predicted_tokens_seconds.jpg
183-
examples/server/bench/kv_cache_usage_ratio.jpg
184-
examples/server/bench/requests_processing.jpg
181+
tools/server/bench/prompt_tokens_seconds.jpg
182+
tools/server/bench/predicted_tokens_seconds.jpg
183+
tools/server/bench/kv_cache_usage_ratio.jpg
184+
tools/server/bench/requests_processing.jpg
185185

186186
- name: Extract mermaid
187187
id: set_mermaid
188188
run: |
189189
set -eux
190190

191-
cd examples/server/bench
191+
cd tools/server/bench
192192
PROMPT_TOKENS_SECONDS=$(cat prompt_tokens_seconds.mermaid)
193193
echo "PROMPT_TOKENS_SECONDS<<EOF" >> $GITHUB_ENV
194194
echo "$PROMPT_TOKENS_SECONDS" >> $GITHUB_ENV

‎.github/workflows/build-linux-cross.yml

+3
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ jobs:
3434
cmake -B build -DCMAKE_BUILD_TYPE=Release \
3535
-DGGML_OPENMP=OFF \
3636
-DLLAMA_BUILD_EXAMPLES=ON \
37+
-DLLAMA_BUILD_TOOLS=ON \
3738
-DLLAMA_BUILD_TESTS=OFF \
3839
-DCMAKE_SYSTEM_NAME=Linux \
3940
-DCMAKE_SYSTEM_PROCESSOR=riscv64 \
@@ -80,6 +81,7 @@ jobs:
8081
-DGGML_VULKAN=ON \
8182
-DGGML_OPENMP=OFF \
8283
-DLLAMA_BUILD_EXAMPLES=ON \
84+
-DLLAMA_BUILD_TOOLS=ON \
8385
-DLLAMA_BUILD_TESTS=OFF \
8486
-DCMAKE_SYSTEM_NAME=Linux \
8587
-DCMAKE_SYSTEM_PROCESSOR=riscv64 \
@@ -125,6 +127,7 @@ jobs:
125127
-DGGML_VULKAN=ON \
126128
-DGGML_OPENMP=OFF \
127129
-DLLAMA_BUILD_EXAMPLES=ON \
130+
-DLLAMA_BUILD_TOOLS=ON \
128131
-DLLAMA_BUILD_TESTS=OFF \
129132
-DCMAKE_SYSTEM_NAME=Linux \
130133
-DCMAKE_SYSTEM_PROCESSOR=aarch64 \

‎.github/workflows/build.yml

+5
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,7 @@ jobs:
633633
-DGGML_METAL_EMBED_LIBRARY=ON \
634634
-DLLAMA_BUILD_COMMON=OFF \
635635
-DLLAMA_BUILD_EXAMPLES=OFF \
636+
-DLLAMA_BUILD_TOOLS=OFF \
636637
-DLLAMA_BUILD_TESTS=OFF \
637638
-DLLAMA_BUILD_SERVER=OFF \
638639
-DCMAKE_SYSTEM_NAME=iOS \
@@ -669,6 +670,7 @@ jobs:
669670
-DGGML_METAL_EMBED_LIBRARY=ON \
670671
-DLLAMA_BUILD_COMMON=OFF \
671672
-DLLAMA_BUILD_EXAMPLES=OFF \
673+
-DLLAMA_BUILD_TOOLS=OFF \
672674
-DLLAMA_BUILD_TESTS=OFF \
673675
-DLLAMA_BUILD_SERVER=OFF \
674676
-DCMAKE_SYSTEM_NAME=tvOS \
@@ -699,6 +701,7 @@ jobs:
699701
-DGGML_METAL_EMBED_LIBRARY=ON \
700702
-DLLAMA_BUILD_COMMON=OFF \
701703
-DLLAMA_BUILD_EXAMPLES=OFF \
704+
-DLLAMA_BUILD_TOOLS=OFF \
702705
-DLLAMA_BUILD_TESTS=OFF \
703706
-DLLAMA_BUILD_SERVER=OFF \
704707
-DCMAKE_SYSTEM_NAME=visionOS \
@@ -739,6 +742,7 @@ jobs:
739742
-DGGML_METAL_EMBED_LIBRARY=ON \
740743
-DLLAMA_CURL=OFF \
741744
-DLLAMA_BUILD_EXAMPLES=OFF \
745+
-DLLAMA_BUILD_TOOLS=OFF \
742746
-DLLAMA_BUILD_TESTS=OFF \
743747
-DLLAMA_BUILD_SERVER=OFF \
744748
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
@@ -1417,6 +1421,7 @@ jobs:
14171421
-DGGML_METAL_EMBED_LIBRARY=ON \
14181422
-DLLAMA_CURL=OFF \
14191423
-DLLAMA_BUILD_EXAMPLES=OFF \
1424+
-DLLAMA_BUILD_TOOLS=OFF \
14201425
-DLLAMA_BUILD_TESTS=OFF \
14211426
-DLLAMA_BUILD_SERVER=OFF \
14221427
-DCMAKE_SYSTEM_NAME=iOS \

‎.github/workflows/server.yml

+12-12
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ on:
1515
push:
1616
branches:
1717
- master
18-
paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*']
18+
paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'tools/server/**.*']
1919
pull_request:
2020
types: [opened, synchronize, reopened]
21-
paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*']
21+
paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'tools/server/**.*']
2222

2323
env:
2424
LLAMA_LOG_COLORS: 1
@@ -74,7 +74,7 @@ jobs:
7474
- name: Tests dependencies
7575
id: test_dependencies
7676
run: |
77-
pip install -r examples/server/tests/requirements.txt
77+
pip install -r tools/server/tests/requirements.txt
7878
7979
# Setup nodejs (to be used for verifying bundled index.html)
8080
- uses: actions/setup-node@v4
@@ -84,14 +84,14 @@ jobs:
8484
- name: WebUI - Install dependencies
8585
id: webui_lint
8686
run: |
87-
cd examples/server/webui
87+
cd tools/server/webui
8888
npm ci
8989
9090
- name: WebUI - Check code format
9191
id: webui_format
9292
run: |
9393
git config --global --add safe.directory $(realpath .)
94-
cd examples/server/webui
94+
cd tools/server/webui
9595
git status
9696
9797
npm run format
@@ -108,7 +108,7 @@ jobs:
108108
id: verify_server_index_html
109109
run: |
110110
git config --global --add safe.directory $(realpath .)
111-
cd examples/server/webui
111+
cd tools/server/webui
112112
git status
113113
114114
npm run build
@@ -161,21 +161,21 @@ jobs:
161161
env:
162162
GITHUB_ACTIONS: "true"
163163
run: |
164-
cd examples/server/tests
164+
cd tools/server/tests
165165
./tests.sh
166166
167167
- name: Tests (sanitizers)
168168
id: server_integration_tests_sanitizers
169169
if: ${{ matrix.sanitizer != '' }}
170170
run: |
171-
cd examples/server/tests
171+
cd tools/server/tests
172172
LLAMA_SANITIZE=1 ./tests.sh
173173
174174
- name: Slow tests
175175
id: server_integration_tests_slow
176176
if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
177177
run: |
178-
cd examples/server/tests
178+
cd tools/server/tests
179179
SLOW_TESTS=1 ./tests.sh
180180
181181
@@ -211,7 +211,7 @@ jobs:
211211
- name: Tests dependencies
212212
id: test_dependencies
213213
run: |
214-
pip install -r examples/server/tests/requirements.txt
214+
pip install -r tools/server/tests/requirements.txt
215215
216216
- name: Copy Libcurl
217217
id: prepare_libcurl
@@ -224,14 +224,14 @@ jobs:
224224
id: server_integration_tests
225225
if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }}
226226
run: |
227-
cd examples/server/tests
227+
cd tools/server/tests
228228
$env:PYTHONIOENCODING = ":replace"
229229
pytest -v -x -m "not slow"
230230
231231
- name: Slow tests
232232
id: server_integration_tests_slow
233233
if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
234234
run: |
235-
cd examples/server/tests
235+
cd tools/server/tests
236236
$env:SLOW_TESTS = "1"
237237
pytest -v -x

‎.gitignore

+6-6
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,11 @@ perf-*.txt
9696
# Examples
9797

9898
examples/jeopardy/results.txt
99-
examples/server/*.css.hpp
100-
examples/server/*.html.hpp
101-
examples/server/*.js.hpp
102-
examples/server/*.mjs.hpp
103-
examples/server/*.gz.hpp
99+
tools/server/*.css.hpp
100+
tools/server/*.html.hpp
101+
tools/server/*.js.hpp
102+
tools/server/*.mjs.hpp
103+
tools/server/*.gz.hpp
104104
!build_64.sh
105105
!examples/*.bat
106106
!examples/*/*.kts
@@ -110,7 +110,7 @@ examples/server/*.gz.hpp
110110

111111
# Server Web UI temporary files
112112
node_modules
113-
examples/server/webui/dist
113+
tools/server/webui/dist
114114

115115
# Python
116116

‎CMakeLists.txt

+5
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ option(LLAMA_BUILD_COMMON "llama: build common utils library" ${LLAMA_STANDALONE
7777

7878
# extra artifacts
7979
option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE})
80+
option(LLAMA_BUILD_TOOLS "llama: build tools" ${LLAMA_STANDALONE})
8081
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
8182
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
8283

@@ -187,6 +188,10 @@ if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_EXAMPLES)
187188
add_subdirectory(pocs)
188189
endif()
189190

191+
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)
192+
add_subdirectory(tools)
193+
endif()
194+
190195
#
191196
# install
192197
#

‎CODEOWNERS

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
/ci/ @ggerganov
44
/.devops/*.Dockerfile @ngxson
5-
/examples/server/ @ngxson
5+
/tools/server/ @ngxson
66
/ggml/src/ggml-cuda/fattn* @JohannesGaessler
77
/ggml/src/ggml-cuda/mmq.* @JohannesGaessler
88
/ggml/src/ggml-cuda/mmv.* @JohannesGaessler

0 commit comments

Comments
 (0)