File tree Expand file tree Collapse file tree 1 file changed +4
-4
lines changed
api/core/model_runtime/model_providers/__base/tokenizers Expand file tree Collapse file tree 1 file changed +4
-4
lines changed Original file line number Diff line number Diff line change 1
- from concurrent .futures import ProcessPoolExecutor
2
1
from os .path import abspath , dirname , join
3
2
from threading import Lock
4
3
from typing import Any , cast
5
4
5
+ import gevent .threadpool # type: ignore
6
6
from transformers import GPT2Tokenizer as TransformerGPT2Tokenizer # type: ignore
7
7
8
8
_tokenizer : Any = None
9
9
_lock = Lock ()
10
- _executor = ProcessPoolExecutor ( max_workers = 1 )
10
+ _pool = gevent . threadpool . ThreadPool ( 1 )
11
11
12
12
13
13
class GPT2Tokenizer :
@@ -22,8 +22,8 @@ def _get_num_tokens_by_gpt2(text: str) -> int:
22
22
23
23
@staticmethod
24
24
def get_num_tokens (text : str ) -> int :
25
- future = _executor . submit (GPT2Tokenizer ._get_num_tokens_by_gpt2 , text )
26
- result = future .result ( )
25
+ future = _pool . spawn (GPT2Tokenizer ._get_num_tokens_by_gpt2 , text )
26
+ result = future .get ( block = True )
27
27
return cast (int , result )
28
28
29
29
@staticmethod
You can’t perform that action at this time.
0 commit comments