Skip to content

Commit c3597d4

Browse files
foldlfoldl
committed
update bindings; clean some warnings.
1 parent 3d9b004 commit c3597d4

18 files changed

+396
-52
lines changed

‎bindings/chatllm.py

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from enum import IntEnum
33
import os, sys, signal, queue
44
import threading
5-
import json
5+
import json, base64
66
from typing import Any, Iterable, List, Union
77

88
try:
@@ -37,6 +37,10 @@ class PrintType(IntEnum):
3737
PRINT_EVT_ASYNC_COMPLETED = 100, # last async operation completed (utf8_str is null)
3838
PRINT_EVT_THOUGHT_COMPLETED = 101, # thought completed
3939

40+
class EmbeddingPurpose(IntEnum):
41+
Document = 0, # for document
42+
Query = 1, # for query
43+
4044
class LibChatLLM:
4145

4246
_obj2id = {}
@@ -100,11 +104,15 @@ def __init__(self, lib: str = '', model_storage: str = '', init_params: list[str
100104
self._chatllm_show_statistics = self._lib.chatllm_show_statistics
101105
self._chatllm_save_session = self._lib.chatllm_save_session
102106
self._chatllm_load_session = self._lib.chatllm_load_session
107+
self._chatllm_multimedia_msg_prepare = self._lib.chatllm_multimedia_msg_prepare
108+
self._chatllm_multimedia_msg_append = self._lib.chatllm_multimedia_msg_append
109+
self._chatllm_user_input_multimedia_msg = self._lib.chatllm_user_input_multimedia_msg
103110

104111
self._chatllm_async_user_input = self._lib.chatllm_async_user_input
105112
self._chatllm_async_ai_continue = self._lib.chatllm_async_ai_continue
106113
self._chatllm_async_tool_input = self._lib.chatllm_async_tool_input
107114
self._chatllm_async_tool_completion = self._lib.chatllm_async_tool_completion
115+
self._chatllm_async_user_input_multimedia_msg = self._lib.chatllm_async_user_input_multimedia_msg
108116

109117
self._chatllm_create.restype = c_void_p
110118
self._chatllm_create.argtypes = []
@@ -123,11 +131,20 @@ def __init__(self, lib: str = '', model_storage: str = '', init_params: list[str
123131
self._chatllm_async_ai_continue.restype = c_int
124132
self._chatllm_async_ai_continue.argtypes = [c_void_p, c_char_p]
125133

134+
self._chatllm_multimedia_msg_prepare.argtypes = [c_void_p]
135+
self._chatllm_multimedia_msg_append.restype = c_int
136+
self._chatllm_multimedia_msg_append.argtypes = [c_void_p, c_char_p, c_char_p]
137+
126138
self._chatllm_user_input.restype = c_int
127139
self._chatllm_user_input.argtypes = [c_void_p, c_char_p]
128140
self._chatllm_async_user_input.restype = c_int
129141
self._chatllm_async_user_input.argtypes = [c_void_p, c_char_p]
130142

143+
self._chatllm_user_input_multimedia_msg.restype = c_int
144+
self._chatllm_user_input_multimedia_msg.argtypes = [c_void_p]
145+
self._chatllm_async_user_input_multimedia_msg.restype = c_int
146+
self._chatllm_async_user_input_multimedia_msg.argtypes = [c_void_p]
147+
131148
self._chatllm_tool_input.restype = c_int
132149
self._chatllm_tool_input.argtypes = [c_void_p, c_char_p]
133150
self._chatllm_async_tool_input.restype = c_int
@@ -139,7 +156,7 @@ def __init__(self, lib: str = '', model_storage: str = '', init_params: list[str
139156
self._chatllm_async_tool_completion.argtypes = [c_void_p, c_char_p]
140157

141158
self._chatllm_text_embedding.restype = c_int
142-
self._chatllm_text_embedding.argtypes = [c_void_p, c_char_p]
159+
self._chatllm_text_embedding.argtypes = [c_void_p, c_char_p, c_int]
143160

144161
self._chatllm_text_tokenize.restype = c_int
145162
self._chatllm_text_tokenize.argtypes = [c_void_p, c_char_p]
@@ -241,11 +258,44 @@ def start(self, obj: c_void_p, callback_obj: Any) -> int:
241258
def set_ai_prefix(self, obj: c_void_p, prefix: str) -> int:
242259
return self._chatllm_set_ai_prefix(obj, c_char_p(prefix.encode()))
243260

244-
def chat(self, obj: c_void_p, user_input: str) -> int:
245-
return self._chatllm_user_input(obj, c_char_p(user_input.encode()))
246-
247-
def async_chat(self, obj: c_void_p, user_input: str) -> int:
248-
return self._chatllm_async_user_input(obj, c_char_p(user_input.encode()))
261+
def _input_multimedia_msg(self, obj: c_void_p, user_input: List[dict | str]) -> int:
262+
self._chatllm_multimedia_msg_prepare(obj)
263+
for x in user_input:
264+
if isinstance(x, str):
265+
self._chatllm_multimedia_msg_append(obj, c_char_p('text'), c_char_p(x))
266+
elif isinstance(x, dict):
267+
t = x['type']
268+
if t == 'text':
269+
data = x['text'].encode()
270+
else:
271+
if 'file' in x:
272+
with open(x['file'], 'rb') as f:
273+
data = f.read()
274+
elif 'url' in x:
275+
url: str = x['url']
276+
if url.startswith('data:'):
277+
i = url.find('base64,')
278+
data = base64.decodebytes(url[i + 7 :].encode())
279+
else:
280+
data = model_downloader.download_file_to_bytes(url)
281+
else:
282+
raise Exception(f'unknown message piece: {x}')
283+
data = base64.b64encode(data)
284+
self._chatllm_multimedia_msg_append(obj, c_char_p(t.encode()), c_char_p(data))
285+
286+
def chat(self, obj: c_void_p, user_input: str | List[dict | str]) -> int:
287+
if isinstance(user_input, str):
288+
return self._chatllm_user_input(obj, c_char_p(user_input.encode()))
289+
elif isinstance(user_input, list):
290+
self._input_multimedia_msg(obj, user_input)
291+
return self._chatllm_user_input_multimedia_msg(obj)
292+
293+
def async_chat(self, obj: c_void_p, user_input: str | List[dict | str]) -> int:
294+
if isinstance(user_input, str):
295+
return self._chatllm_async_user_input(obj, c_char_p(user_input.encode()))
296+
else:
297+
self._input_multimedia_msg(obj, user_input)
298+
self._chatllm_async_user_input_multimedia_msg(obj)
249299

250300
def ai_continue(self, obj: c_void_p, suffix: str) -> int:
251301
return self._chatllm_ai_continue(obj, c_char_p(suffix.encode()))
@@ -268,8 +318,8 @@ def tool_completion(self, obj: c_void_p, user_input: str) -> int:
268318
def text_tokenize(self, obj: c_void_p, text: str) -> str:
269319
return self._chatllm_text_tokenize(obj, c_char_p(text.encode()))
270320

271-
def text_embedding(self, obj: c_void_p, text: str) -> str:
272-
return self._chatllm_text_embedding(obj, c_char_p(text.encode()))
321+
def text_embedding(self, obj: c_void_p, text: str, purpose: EmbeddingPurpose = EmbeddingPurpose.Document) -> str:
322+
return self._chatllm_text_embedding(obj, c_char_p(text.encode()), c_int(purpose.value))
273323

274324
def qa_rank(self, obj: c_void_p, q: str, a: str) -> float:
275325
return self._chatllm_qa_rank(obj, c_char_p(q.encode()), c_char_p(a.encode()))

‎bindings/libchatllm.h

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,29 @@ DLL_DECL void API_CALL chatllm_set_gen_max_tokens(struct chatllm_obj *obj, int g
154154
*/
155155
DLL_DECL void API_CALL chatllm_restart(struct chatllm_obj *obj, const char *utf8_sys_prompt);
156156

157+
/**
158+
* @brief prepare to generate a multimedia input, i.e. clear previously added pieces.
159+
*
160+
* Each `chatllm_obj` has a global multimedia message object, which can be used as user input,
161+
* or chat history, etc.
162+
*
163+
* @param[in] obj model object
164+
* @return 0 if succeeded
165+
*/
166+
DLL_DECL void API_CALL chatllm_multimedia_msg_prepare(struct chatllm_obj *obj);
167+
168+
/**
169+
* @brief add a piece to a multimedia message
170+
*
171+
* Remember to clear the message by `chatllm_multimedia_msg_prepare` when starting a new message.
172+
*
173+
* @param[in] obj model object
174+
* @param[in] type type ::= "text" | "image" | "video" | "audio" | ...
175+
* @param[in] utf8_str content, i.e. utf8 text content, or base64 encoded data of multimedia data.
176+
* @return 0 if succeeded
177+
*/
178+
DLL_DECL int API_CALL chatllm_multimedia_msg_append(struct chatllm_obj *obj, const char *type, const char *utf8_str);
179+
157180
enum RoleType
158181
{
159182
ROLE_USER = 2,
@@ -165,7 +188,7 @@ enum RoleType
165188
* @brief push back a message to the end of chat history.
166189
*
167190
* This can be used to restore session after `chatllm_restart`.
168-
* This would not trigger generation. Use `chatllm_user_input`, etc to start generation.
191+
* This would not trigger generation. Use `chatllm_user_input`, etc to start generation.
169192
*
170193
* @param[in] obj model object
171194
* @param[in] role_type message type (see `RoleType`)
@@ -184,6 +207,16 @@ DLL_DECL void API_CALL chatllm_history_append(struct chatllm_obj *obj, int role_
184207
*/
185208
DLL_DECL int API_CALL chatllm_user_input(struct chatllm_obj *obj, const char *utf8_str);
186209

210+
/**
211+
* @brief take current multimedia message as user input and run
212+
*
213+
* This function is synchronized, i.e. it returns after model generation ends and `f_end` is called.
214+
*
215+
* @param[in] obj model object
216+
* @return 0 if succeeded
217+
*/
218+
DLL_DECL int API_CALL chatllm_user_input_multimedia_msg(struct chatllm_obj *obj);
219+
187220
/**
188221
* @brief set prefix for AI generation
189222
*
@@ -248,16 +281,25 @@ DLL_DECL int chatllm_tool_completion(struct chatllm_obj *obj, const char *utf8_s
248281
*/
249282
DLL_DECL int chatllm_text_tokenize(struct chatllm_obj *obj, const char *utf8_str);
250283

284+
enum EmbeddingPurpose
285+
{
286+
EMBEDDING_FOR_DOC = 0, // for document
287+
EMBEDDING_FOR_QUERY = 1, // for query
288+
};
289+
251290
/**
252291
* @brief text embedding
253292
*
254293
* embedding is emitted through `PRINTLN_EMBEDDING`.
255294
*
295+
* Note: Not all models support specifying purpose.(see _Qwen3-Embedding_).
296+
*
256297
* @param[in] obj model object
257298
* @param[in] utf8_str text
299+
* @param[in] purpose purpose, see `EmbeddingPurpose`
258300
* @return 0 if succeeded
259301
*/
260-
DLL_DECL int chatllm_text_embedding(struct chatllm_obj *obj, const char *utf8_str);
302+
DLL_DECL int chatllm_text_embedding(struct chatllm_obj *obj, const char *utf8_str, int purpose);
261303

262304
/**
263305
* @brief question & answer ranking
@@ -346,6 +388,14 @@ DLL_DECL int API_CALL chatllm_async_start(struct chatllm_obj *obj, f_chatllm_pri
346388
*/
347389
DLL_DECL int API_CALL chatllm_async_user_input(struct chatllm_obj *obj, const char *utf8_str);
348390

391+
/**
392+
* @brief async version of `chatllm_user_input_multimedia_msg`
393+
*
394+
* @param ...
395+
* @return 0 if started else -1
396+
*/
397+
DLL_DECL int API_CALL chatllm_async_user_input_multimedia_msg(struct chatllm_obj *obj);
398+
349399
/**
350400
* @brief async version of `chatllm_ai_continue`
351401
@@ -376,7 +426,7 @@ DLL_DECL int chatllm_async_tool_completion(struct chatllm_obj *obj, const char *
376426
* @param ...
377427
* @return 0 if started else -1
378428
*/
379-
DLL_DECL int chatllm_async_text_embedding(struct chatllm_obj *obj, const char *utf8_str);
429+
DLL_DECL int chatllm_async_text_embedding(struct chatllm_obj *obj, const char *utf8_str, int purpose);
380430

381431
/**
382432
* @brief async version of `chatllm_qa_rank`

‎bindings/libchatllm.nim

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,29 @@ proc chatllm_set_gen_max_tokens*(obj: ptr chatllm_obj; gen_max_tokens: cint) {.s
8787
##
8888
proc chatllm_restart*(obj: ptr chatllm_obj; utf8_sys_prompt: cstring) {.stdcall, dynlib: libName, importc.}
8989

90+
##
91+
## @brief prepare to generate a multimedia input, i.e. clear previously added pieces.
92+
##
93+
## Each `chatllm_obj` has a global multimedia message object, which can be used as user input,
94+
## or chat history, etc.
95+
##
96+
## @param[in] obj model object
97+
## @return 0 if succeeded
98+
##
99+
proc chatllm_multimedia_msg_prepare(obj: ptr chatllm_obj) {.stdcall, dynlib: libName, importc.}
100+
101+
##
102+
## @brief add a piece to a multimedia message
103+
##
104+
## Remember to clear the message by `chatllm_multimedia_msg_prepare` when starting a new message.
105+
##
106+
## @param[in] obj model object
107+
## @param[in] type type ::= "text" | "image" | "video" | "audio" | ...
108+
## @param[in] utf8_str content, i.e. utf8 text content, or base64 encoded data of multimedia data.
109+
## @return 0 if succeeded
110+
##
111+
proc chatllm_multimedia_msg_append(obj: ptr chatllm_obj; content_type: cstring; utf8_str: cstring): cint {.stdcall, dynlib: libName, importc.}
112+
90113
type
91114
RoleType* = enum
92115
ROLE_USER = 2,
@@ -116,6 +139,16 @@ proc chatllm_history_append*(obj: ptr chatllm_obj; role_type: int; utf8_str: cst
116139
##
117140
proc chatllm_user_input*(obj: ptr chatllm_obj; utf8_str: cstring): cint {.stdcall, dynlib: libName, importc.}
118141

142+
##
143+
## @brief take current multimedia message as user input and run
144+
##
145+
## This function is synchronized, i.e. it returns after model generation ends and `f_end` is called.
146+
##
147+
## @param[in] obj model object
148+
## @return 0 if succeeded
149+
##
150+
proc chatllm_user_input_multimedia_msg(obj: ptr chatllm_obj): cint {.stdcall, dynlib: libName, importc.}
151+
119152
##
120153
## @brief set prefix for AI generation
121154
##
@@ -171,16 +204,24 @@ proc chatllm_tool_completion*(obj: ptr chatllm_obj; utf8_str: cstring): cint {.s
171204
##
172205
proc chatllm_text_tokenize*(obj: ptr chatllm_obj; utf8_str: cstring): cint {.stdcall, dynlib: libName, importc.}
173206

207+
type
208+
EmbeddingPurpose* = enum
209+
EMBEDDING_FOR_DOC = 0, # for document
210+
EMBEDDING_FOR_QUERY = 1, # for query
211+
174212
##
175-
## @brief text embedding
213+
## @brief text embedding
176214
##
177-
## embedding is emitted through `PRINTLN_EMBEDDING`.
215+
## embedding is emitted through `PRINTLN_EMBEDDING`.
178216
##
179-
## @param[in] obj model object
180-
## @param[in] utf8_str text
181-
## @return 0 if succeeded
217+
## Note: Not all models support specifying purpose.(see _Qwen3-Embedding_).
218+
##
219+
## @param[in] obj model object
220+
## @param[in] utf8_str text
221+
## @param[in] purpose purpose, see `EmbeddingPurpose`
222+
## @return 0 if succeeded
182223
##
183-
proc chatllm_text_embedding*(obj: ptr chatllm_obj; utf8_str: cstring): cint {.stdcall, dynlib: libName, importc.}
224+
proc chatllm_text_embedding*(obj: ptr chatllm_obj; utf8_str: cstring; purpose: cint): cint {.stdcall, dynlib: libName, importc.}
184225

185226
##
186227
## @brief question & answer ranking
@@ -271,6 +312,14 @@ proc chatllm_async_start*(obj: ptr chatllm_obj; f_print: f_chatllm_print;
271312
##
272313
proc chatllm_async_user_input*(obj: ptr chatllm_obj; utf8_str: cstring): cint {.stdcall, dynlib: libName, importc.}
273314

315+
##
316+
## @brief async version of `chatllm_user_input_multimedia_msg`
317+
##
318+
## @param ...
319+
## @return 0 if started else -1
320+
##
321+
proc chatllm_async_user_input_multimedia_msg(obj: ptr chatllm_obj): cint {.stdcall, dynlib: libName, importc.}
322+
274323
##
275324
## @brief async version of `chatllm_tool_input`
276325
##
@@ -293,7 +342,7 @@ proc chatllm_async_tool_completion*(obj: ptr chatllm_obj; utf8_str: cstring): ci
293342
## @param ...
294343
## @return 0 if started else -1
295344
##
296-
proc chatllm_async_text_embedding*(obj: ptr chatllm_obj; utf8_str: cstring): cint {.stdcall, dynlib: libName, importc.}
345+
proc chatllm_async_text_embedding*(obj: ptr chatllm_obj; utf8_str: cstring; purpose: cint): cint {.stdcall, dynlib: libName, importc.}
297346

298347
##
299348
## @brief async version of `chatllm_qa_rank`

0 commit comments

Comments
 (0)