Skip to content

Commit 39faec3

Browse files
cheesewaferYu Xiachenweize1998
authored
feat: support local llms (#68)
* support local LLMs --------- Co-authored-by: Yu Xia <[email protected]> Co-authored-by: Weize Chen <[email protected]> Co-authored-by: chenweize1998 <[email protected]>
1 parent 53f2ec0 commit 39faec3

File tree

5 files changed

+217
-1
lines changed

5 files changed

+217
-1
lines changed

‎agentverse/llms/openai.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,12 @@ class OpenAIChatArgs(BaseModelArgs):
9292
# total_tokens=response["usage"]["total_tokens"],
9393
# )
9494

95-
95+
# To support your own local LLMs, register it here and add it into LOCAL_LLMS.
96+
LOCAL_LLMS = ['llama-2-7b-chat-hf']
9697
@llm_registry.register("gpt-35-turbo")
9798
@llm_registry.register("gpt-3.5-turbo")
9899
@llm_registry.register("gpt-4")
100+
@llm_registry.register("llama-2-7b-chat-hf")
99101
class OpenAIChat(BaseChatModel):
100102
args: OpenAIChatArgs = Field(default_factory=OpenAIChatArgs)
101103

@@ -109,6 +111,8 @@ def __init__(self, max_retry: int = 3, **kwargs):
109111
args[k] = kwargs.pop(k, v)
110112
if len(kwargs) > 0:
111113
logging.warning(f"Unused arguments: {kwargs}")
114+
if args['model'] in LOCAL_LLMS:
115+
openai.api_base = "http://localhost:5000/v1"
112116
super().__init__(args=args, max_retry=max_retry)
113117

114118
# def _construct_messages(self, history: List[Message]):
@@ -301,6 +305,7 @@ def get_spend(self) -> int:
301305
"gpt-4": 0.03,
302306
"gpt-4-0613": 0.03,
303307
"gpt-4-32k": 0.06,
308+
"llama-2-7b-chat-hf": 0.0,
304309
}
305310

306311
output_cost_map = {
@@ -311,6 +316,7 @@ def get_spend(self) -> int:
311316
"gpt-4": 0.06,
312317
"gpt-4-0613": 0.06,
313318
"gpt-4-32k": 0.12,
319+
"llama-2-7b-chat-hf": 0.0,
314320
}
315321

316322
model = self.args.model

‎agentverse/tasks/tasksolving/commongen/llama-2-7b-chat-hf/config.yaml

+197
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
cnt_agents: &cnt_agents 2
2+
max_turn: &max_turn 3
3+
max_inner_turns: &max_inner_turns 3
4+
5+
prompts:
6+
role_assigner_prepend_prompt: &role_assigner_prepend_prompt |-
7+
8+
role_assigner_append_prompt: &role_assigner_append_prompt |-
9+
# Role Description
10+
You are the leader of a group of experts, now you need to recruit a small group of experts with diverse identity to generate coherent and grammatically correct sentences containing the following given words:
11+
${task_description}
12+
13+
You can recruit ${cnt_critic_agents} expert in different fields. What experts will you recruit?
14+
15+
# Response Format Guidance
16+
You should respond with a list of expert description. For example:
17+
1. an electrical engineer specified in the filed of xxx.
18+
2. an economist who is good at xxx.
19+
3. a lawyer with a good knowledge of xxx.
20+
...
21+
22+
Only respond with the description of each role. Do not include your reason.
23+
24+
solver_prepend_prompt: &solver_prepend_prompt |-
25+
You are ${role_description}. Generate a coherent and grammatically correct paragraph containing the following given words (or their variations):
26+
WORDS:
27+
${task_description}
28+
29+
solver_append_prompt: &solver_append_prompt |-
30+
31+
critic_prepend_prompt: &critic_prepend_prompt |-
32+
You are in a discussion group, aiming to generate coherent and grammatically correct sentences containing the following given words (or their variations):
33+
WORDS:
34+
${task_description}
35+
36+
Below is the chat history in your group.
37+
38+
critic_append_prompt: &critic_append_prompt |-
39+
You are ${role_description}. Based on your knowledge, can you check whether the latest provided paragraph contains all the given words or their variations? When responding, you should follow the following rules:
40+
1. If the above latest provided solution has covered all the given words or their variations, end your response with a special token "[Agree]".
41+
1. If not, double-check the above solutions, give your critics, and generate a better solution.
42+
43+
manager_prompt: &manager_prompt |-
44+
45+
executor_prepend_prompt: &executor_prepend_prompt |-
46+
47+
executor_append_prompt: &executor_append_prompt |-
48+
49+
evaluator_prepend_prompt: &evaluator_prepend_prompt |-
50+
51+
evaluator_append_prompt: &evaluator_append_prompt |-
52+
You are a reviewer who checks whether a paragraph contains all the given words (including their variations). When some words are missing, you should patiently point out, and output a score of 0. When the paragraph contains all the words, you should output a score of 1.
53+
54+
WORDS:
55+
${task_description}
56+
57+
SOLUTION:
58+
```
59+
${solution}
60+
```
61+
62+
TEST RESULT:
63+
${result}
64+
65+
RESPONSE FORMAT:
66+
You must respond in the following format:
67+
Score: (0 or 1. 0 if there are some missing words, 1 if there is no missing words)
68+
Advice: (point out all the missing words)
69+
70+
71+
name: pipeline
72+
73+
74+
environment:
75+
env_type: task-basic
76+
max_turn: *max_turn
77+
rule:
78+
role_assigner:
79+
type: role_description
80+
cnt_agents: *cnt_agents
81+
decision_maker:
82+
type: vertical-solver-first
83+
max_inner_turns: *max_inner_turns
84+
executor:
85+
type: coverage-test
86+
evaluator:
87+
type: basic
88+
89+
agents:
90+
- #role_assigner_agent:
91+
agent_type: role_assigner
92+
name: role assigner
93+
max_retry: 1000
94+
prepend_prompt_template: *role_assigner_prepend_prompt
95+
append_prompt_template: *role_assigner_append_prompt
96+
memory:
97+
memory_type: chat_history
98+
llm:
99+
llm_type: llama-2-7b-chat-hf
100+
model: "llama-2-7b-chat-hf"
101+
temperature: 0
102+
max_tokens: 512
103+
output_parser:
104+
type: role_assigner
105+
106+
- #solver_agent:
107+
agent_type: solver
108+
name: Planner
109+
max_retry: 1000
110+
max_history: 4
111+
prepend_prompt_template: *solver_prepend_prompt
112+
append_prompt_template: *solver_append_prompt
113+
memory:
114+
memory_type: chat_history
115+
llm:
116+
llm_type: llama-2-7b-chat-hf
117+
model: "llama-2-7b-chat-hf"
118+
temperature: 0
119+
max_tokens: 1024
120+
output_parser:
121+
type: commongen
122+
# max_tokens: 1024
123+
# stop:
124+
# - "\ndef "
125+
# - "\nclass "
126+
# - "\nif "
127+
# - "\n\n#"
128+
129+
- #critic_agents:
130+
agent_type: critic
131+
name: Critic 1
132+
max_retry: 1000
133+
max_history: 4
134+
role_description: |-
135+
Waiting to be assigned.
136+
prepend_prompt_template: *critic_prepend_prompt
137+
append_prompt_template: *critic_append_prompt
138+
memory:
139+
memory_type: chat_history
140+
llm:
141+
llm_type: llama-2-7b-chat-hf
142+
model: "llama-2-7b-chat-hf"
143+
temperature: 0
144+
max_tokens: 1024
145+
output_parser:
146+
type: mgsm-critic-agree
147+
148+
- #executor_agent:
149+
agent_type: executor
150+
name: Executor
151+
max_retry: 1000
152+
prepend_prompt_template: *executor_prepend_prompt
153+
append_prompt_template: *executor_append_prompt
154+
memory:
155+
memory_type: chat_history
156+
llm:
157+
llm_type: llama-2-7b-chat-hf
158+
model: llama-2-7b-chat-hf
159+
temperature: 0
160+
max_tokens: 1024
161+
output_parser:
162+
type: commongen
163+
164+
- #evaluator_agent:
165+
agent_type: evaluator
166+
name: Evaluator
167+
max_retry: 1000
168+
role_description: |-
169+
Evaluator
170+
prepend_prompt_template: *evaluator_prepend_prompt
171+
append_prompt_template: *evaluator_append_prompt
172+
memory:
173+
memory_type: chat_history
174+
llm:
175+
llm_type: llama-2-7b-chat-hf
176+
model: llama-2-7b-chat-hf
177+
temperature: 0.3
178+
max_tokens: 1024
179+
output_parser:
180+
type: humaneval-evaluator
181+
dimensions:
182+
- Score
183+
184+
- #manager_agent:
185+
agent_type: manager
186+
name: Manager
187+
max_retry: 1000
188+
prompt_template: *manager_prompt
189+
memory:
190+
memory_type: chat_history
191+
llm:
192+
llm_type: llama-2-7b-chat-hf
193+
model: "llama-2-7b-chat-hf"
194+
temperature: 0
195+
max_tokens: 1024
196+
output_parser:
197+
type: humaneval-manager

‎dataloader/commongen.py

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
@dataloader_registry.register("tasksolving/commongen/gpt-4")
77
@dataloader_registry.register("tasksolving/commongen/gpt-3.5")
8+
@dataloader_registry.register("tasksolving/commongen/llama-2-7b-chat-hf")
89
class CommongenLoader(DataLoader):
910
def __init__(self, path: str):
1011
super().__init__(path)

‎requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ colorlog
1717
rapidfuzz
1818
spacy
1919
colorama==0.4.6
20+
fschat[model_worker,webui]

‎scripts/run_local_model_server.sh

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
:<<COMMENT
2+
See https://github.com/lm-sys/FastChat/blob/main/docs/openai_api.md for more usages.
3+
COMMENT
4+
# export CUDA_VISIBLE_DEVICES=0
5+
MODEL_PATH="path_to_the_downloaded_model_dir"
6+
MODEL_NAME="name_of_the_model"
7+
python3 -m fastchat.serve.controller & \
8+
python3 -m fastchat.serve.multi_model_worker \
9+
--model-path ${MODEL_PATH} \
10+
--model-names ${MODEL_NAME} & \
11+
python3 -m fastchat.serve.openai_api_server --host localhost --port 5000

0 commit comments

Comments
 (0)