更新demo

1. 支持清除历史/重试
2. 更新了新的requirement.txt
3. 记得去huggingface下载最新的配置文件（几天前）

Author: zRzRzRzRzRzRzR

.gitignore

@@ -7,4 +7,5 @@ finetune_demo/formatted_data
 ToolAlpaca/
 AdvertiseGen/
 *.gz
-*.idea
+*.idea
+.DS_Store

composite_demo/client.py

@@ -79,6 +79,10 @@ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> to
 
     if history is None:
         history = []
+
+    print("\n== Input ==\n", query)
+    print("\n==History==\n", history)
+
     if logits_processor is None:
         logits_processor = LogitsProcessorList()
     logits_processor.append(InvalidScoreLogitsProcessor())
@@ -109,7 +113,6 @@ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> to
         attention_mask = torch.cat((attention_mask.new_ones(1, past_length), attention_mask), dim=1)
         inputs['attention_mask'] = attention_mask
     history.append({"role": role, "content": query})
-    # print("input_shape>", inputs['input_ids'].shape)
     input_sequence_length = inputs['input_ids'].shape[1]
     if input_sequence_length + max_new_tokens >= self.config.seq_length:
         yield "Current input sequence length {} plus max_new_tokens {} is too long. The maximum model sequence length is {}. You may adjust the generation parameter to enable longer chat history.".format(
@@ -181,9 +184,7 @@ def generate_stream(self,
 
         query = history[-1].content
         role = str(history[-1].role).removeprefix('<|').removesuffix('|>')
-
         text = ''
-
         for new_text, _ in stream_chat(self.model,
                                        self.tokenizer,
                                        query,

composite_demo/demo_chat.py

@@ -17,12 +17,15 @@ def append_conversation(
     conversation.show(placeholder)
 
 
-def main(top_p: float,
-         temperature: float,
-         system_prompt: str,
-         prompt_text: str,
-         repetition_penalty: float,
-         max_new_tokens: int):
+def main(
+        prompt_text: str,
+        system_prompt: str,
+        top_p: float = 0.8,
+        temperature: float = 0.95,
+        repetition_penalty: float = 1.0,
+        max_new_tokens: int = 1024,
+        retry: bool = False
+):
     placeholder = st.empty()
     with placeholder.container():
         if 'chat_history' not in st.session_state:
@@ -33,6 +36,16 @@ def main(top_p: float,
         for conversation in history:
             conversation.show()
 
+    if retry:
+        last_user_conversation_idx = None
+        for idx, conversation in enumerate(history):
+            if conversation.role == Role.USER:
+                last_user_conversation_idx = idx
+        if last_user_conversation_idx is not None:
+            prompt_text = history[last_user_conversation_idx].content
+            del history[last_user_conversation_idx:]
+
+
     if prompt_text:
         prompt_text = prompt_text.strip()
         append_conversation(Conversation(Role.USER, prompt_text), history)
@@ -42,11 +55,6 @@ def main(top_p: float,
             tools=None,
             history=history,
         )
-        print("=== Input:")
-        print(input_text)
-        print("=== History:")
-        print(history)
-
         placeholder = st.empty()
         message_placeholder = placeholder.chat_message(name="assistant", avatar="assistant")
         markdown_placeholder = message_placeholder.empty()
@@ -65,9 +73,7 @@ def main(top_p: float,
         ):
             token = response.token
             if response.token.special:
-                print("=== Output:")
-                print(output_text)
-
+                print("\n==Output:==\n", output_text)
                 match token.text.strip():
                     case '<|user|>':
                         break
@@ -81,3 +87,5 @@ def main(top_p: float,
             Role.ASSISTANT,
             postprocess_text(output_text),
         ), history, markdown_placeholder)
+    else:
+        st.session_state.chat_history = []

composite_demo/demo_ci.py

@@ -221,12 +221,15 @@ def append_conversation(
     conversation.show(placeholder)
 
 
-def main(top_p: float,
-         temperature: float,
-         prompt_text: str,
-         repetition_penalty: float,
-         max_new_tokens: int,
-         truncate_length: int = 1024):
+def main(
+        prompt_text: str,
+        top_p: float = 0.2,
+        temperature: float = 0.1,
+        repetition_penalty: float = 1.1,
+        max_new_tokens: int = 1024,
+        truncate_length: int = 1024,
+        retry: bool = False
+):
     if 'ci_history' not in st.session_state:
         st.session_state.ci_history = []
 
@@ -235,6 +238,15 @@ def main(top_p: float,
     for conversation in history:
         conversation.show()
 
+    if retry:
+        last_user_conversation_idx = None
+        for idx, conversation in enumerate(history):
+            if conversation.role == Role.USER:
+                last_user_conversation_idx = idx
+        if last_user_conversation_idx is not None:
+            prompt_text = history[last_user_conversation_idx].content
+            del history[last_user_conversation_idx:]
+
     if prompt_text:
         prompt_text = prompt_text.strip()
         role = Role.USER
@@ -245,10 +257,6 @@ def main(top_p: float,
             None,
             history,
         )
-        print("=== Input:")
-        print(input_text)
-        print("=== History:")
-        print(history)
 
         placeholder = st.container()
         message_placeholder = placeholder.chat_message(name="assistant", avatar="assistant")
@@ -269,9 +277,7 @@ def main(top_p: float,
             ):
                 token = response.token
                 if response.token.special:
-                    print("=== Output:")
-                    print(output_text)
-
+                    print("\n==Output:==\n", output_text)
                     match token.text.strip():
                         case '<|user|>':
                             append_conversation(Conversation(
@@ -335,3 +341,5 @@ def main(top_p: float,
                     postprocess_text(output_text),
                 ), history, markdown_placeholder)
                 return
+    else:
+        st.session_state.chat_history = []

composite_demo/demo_tool.py

@@ -29,7 +29,7 @@
 
 
 def tool_call(*args, **kwargs) -> dict:
-    print("=== Tool call:")
+    print("=== Tool call===")
     print(args)
     print(kwargs)
     st.session_state.calling_tool = True
@@ -60,13 +60,15 @@ def append_conversation(
     conversation.show(placeholder)
 
 
-def main(top_p: float,
-         temperature: float,
-         prompt_text: str,
-         repetition_penalty: float,
-         max_new_tokens: int,
-         truncate_length: int = 1024,
-         ):
+def main(
+        prompt_text: str,
+        top_p: float = 0.2,
+        temperature: float = 0.1,
+        repetition_penalty: float = 1.1,
+        max_new_tokens: int = 1024,
+        truncate_length: int = 1024,
+        retry: bool = False
+):
     manual_mode = st.toggle('Manual mode',
                             help='Define your tools in YAML format. You need to supply tool call results manually.'
                             )
@@ -95,22 +97,21 @@ def main(top_p: float,
     for conversation in history:
         conversation.show()
 
+    if retry:
+        last_user_conversation_idx = None
+        for idx, conversation in enumerate(history):
+            if conversation.role == Role.USER:
+                last_user_conversation_idx = idx
+        if last_user_conversation_idx is not None:
+            prompt_text = history[last_user_conversation_idx].content
+            del history[last_user_conversation_idx:]
+
     if prompt_text:
         prompt_text = prompt_text.strip()
         role = st.session_state.calling_tool and Role.OBSERVATION or Role.USER
         append_conversation(Conversation(role, prompt_text), history)
         st.session_state.calling_tool = False
 
-        input_text = preprocess_text(
-            None,
-            tools,
-            history,
-        )
-        print("=== Input:")
-        print(input_text)
-        print("=== History:")
-        print(history)
-
         placeholder = st.container()
         message_placeholder = placeholder.chat_message(name="assistant", avatar="assistant")
         markdown_placeholder = message_placeholder.empty()
@@ -130,9 +131,7 @@ def main(top_p: float,
             ):
                 token = response.token
                 if response.token.special:
-                    print("=== Output:")
-                    print(output_text)
-
+                    print("\n==Output:==\n", output_text)
                     match token.text.strip():
                         case '<|user|>':
                             append_conversation(Conversation(
@@ -199,3 +198,5 @@ def main(top_p: float,
                     postprocess_text(output_text),
                 ), history, markdown_placeholder)
                 return
+    else:
+        st.session_state.chat_history = []

composite_demo/main.py

@@ -40,6 +40,12 @@ class Mode(str, Enum):
     max_new_token = st.slider(
         'Output length', 5, 32000, 256, step=1
     )
+
+    cols = st.columns(2)
+    export_btn = cols[0]
+    clear_history = cols[1].button("Clear History", use_container_width=True)
+    retry = export_btn.button("Retry", use_container_width=True)
+
     system_prompt = st.text_area(
         label="System Prompt (Only for chat mode)",
         height=300,
@@ -58,27 +64,37 @@ class Mode(str, Enum):
     label_visibility='hidden',
 )
 
+if clear_history or retry:
+    prompt_text = ""
+
 match tab:
     case Mode.CHAT:
-        demo_chat.main(top_p=top_p,
-                       temperature=temperature,
-                       prompt_text=prompt_text,
-                       system_prompt=system_prompt,
-                       repetition_penalty=repetition_penalty,
-                       max_new_tokens=max_new_token)
+        demo_chat.main(
+            retry=retry,
+            top_p=top_p,
+            temperature=temperature,
+            prompt_text=prompt_text,
+            system_prompt=system_prompt,
+            repetition_penalty=repetition_penalty,
+            max_new_tokens=max_new_token
+        )
     case Mode.TOOL:
-        demo_tool.main(top_p=top_p,
-                       temperature=temperature,
-                       prompt_text=prompt_text,
-                       repetition_penalty=repetition_penalty,
-                       max_new_tokens=max_new_token,
-                       truncate_length=1024)
+        demo_tool.main(
+            retry=retry,
+            top_p=top_p,
+            temperature=temperature,
+            prompt_text=prompt_text,
+            repetition_penalty=repetition_penalty,
+            max_new_tokens=max_new_token,
+            truncate_length=1024)
     case Mode.CI:
-        demo_ci.main(top_p=top_p,
-                     temperature=temperature,
-                     prompt_text=prompt_text,
-                     repetition_penalty=repetition_penalty,
-                     max_new_tokens=max_new_token,
-                     truncate_length=1024)
+        demo_ci.main(
+            retry=retry,
+            top_p=top_p,
+            temperature=temperature,
+            prompt_text=prompt_text,
+            repetition_penalty=repetition_penalty,
+            max_new_tokens=max_new_token,
+            truncate_length=1024)
     case _:
         st.error(f'Unexpected tab: {tab}')

composite_demo/requirements.txt

@@ -1,12 +1,11 @@
-huggingface_hub
-ipykernel
-ipython
-jupyter_client
-pillow
-sentencepiece
-streamlit
-tokenizers
-torch
-transformers
-pyyaml
-requests
+huggingface_hub>=0.19.4
+pillow>=10.1.0
+streamlit>=1.29.0
+tokenizers>=0.15.0
+torch>=2.1.0
+transformers>=4.36.1
+pyyaml>=6.0.1
+requests>=2.31.0
+ipykernel>=6.26.0
+ipython>=8.18.1
+jupyter_client>=8.6.0

openai_api_demo/requirements.txt

@@ -1,14 +1,22 @@
-protobuf
-transformers>=4.30.2
-cpm_kernels
-torch>=2.0
+protobuf>=4.25.1
+transformers>=4.36.1
+cpm_kernels>=1.0.11
+torch>=2.1.0
 gradio~=3.39
-sentencepiece
-accelerate
-sse-starlette
+sentencepiece>=0.1.99
+accelerate>=0.25.0
 streamlit>=1.29.0
-fastapi>=0.104.1
+fastapi>=0.105.0
 uvicorn~=0.24.0
 loguru~=0.7.2
 mdtex2html>=1.2.0
-latex2mathml>=3.76.0
+latex2mathml>=3.77.0
+
+# for openai demo
+openai>=1.4.0
+pydantic>=2.5.2
+httpx>=0.25.2
+fastapi>=0.105.0
+sse-starlette>=1.8.2
+uvicorn~=0.24.0
+timm>=0.9.12