Chat Interface flagging and chatbot feedback (#10272)

* changes

* add changeset

* changes

* Update gradio/flagging.py

Co-authored-by: Abubakar Abid <[email protected]>

* Update gradio/chat_interface.py

Co-authored-by: Abubakar Abid <[email protected]>

* Update gradio/chat_interface.py

Co-authored-by: Abubakar Abid <[email protected]>

* Update gradio/chat_interface.py

Co-authored-by: Abubakar Abid <[email protected]>

* Update gradio/chat_interface.py

Co-authored-by: Abubakar Abid <[email protected]>

* Update gradio/components/chatbot.py

Co-authored-by: Abubakar Abid <[email protected]>

* changes

* changes

* changes

* Update gradio/components/chatbot.py

Co-authored-by: Abubakar Abid <[email protected]>

* changes

* changes

* doc changes

---------

Co-authored-by: Ali Abid <[email protected]>
Co-authored-by: gradio-pr-bot <[email protected]>
Co-authored-by: Abubakar Abid <[email protected]>

Author: 4 people

.changeset/dirty-sides-jam.md

@@ -0,0 +1,8 @@
+---
+"@gradio/atoms": minor
+"@gradio/chatbot": minor
+"@gradio/utils": minor
+"gradio": minor
+---
+
+feat:Chat Interface flagging and chatbot feedback

demo/chatinterface_streaming_echo/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatinterface_streaming_echo"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import time\n", "import gradio as gr\n", "\n", "def slow_echo(message, history):\n", "    for i in range(len(message)):\n", "        time.sleep(0.05)\n", "        yield \"You typed: \" + message[: i + 1]\n", "\n", "demo = gr.ChatInterface(slow_echo, type=\"messages\")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatinterface_streaming_echo"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import time\n", "import gradio as gr\n", "\n", "def slow_echo(message, history):\n", "    for i in range(len(message)):\n", "        time.sleep(0.05)\n", "        yield \"You typed: \" + message[: i + 1]\n", "\n", "demo = gr.ChatInterface(slow_echo, type=\"messages\", flagging_mode=\"manual\", flagging_options=[\"Like\", \"Spam\", \"Inappropriate\", \"Other\"])\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/chatinterface_streaming_echo/run.py

@@ -6,7 +6,7 @@ def slow_echo(message, history):
         time.sleep(0.05)
         yield "You typed: " + message[: i + 1]
 
-demo = gr.ChatInterface(slow_echo, type="messages")
+demo = gr.ChatInterface(slow_echo, type="messages", flagging_mode="manual", flagging_options=["Like", "Spam", "Inappropriate", "Other"])
 
 if __name__ == "__main__":
     demo.launch()

gradio/chat_interface.py

@@ -7,6 +7,7 @@
 import builtins
 import copy
 import inspect
+import os
 import warnings
 from collections.abc import AsyncGenerator, Callable, Generator, Sequence
 from pathlib import Path
@@ -37,6 +38,7 @@
 from gradio.components.multimodal_textbox import MultimodalPostprocess, MultimodalValue
 from gradio.context import get_blocks_context
 from gradio.events import Dependency, EditData, SelectData
+from gradio.flagging import ChatCSVLogger
 from gradio.helpers import create_examples as Examples  # noqa: N812
 from gradio.helpers import special_args, update
 from gradio.layouts import Accordion, Column, Group, Row
@@ -85,6 +87,9 @@ def __init__(
         title: str | None = None,
         description: str | None = None,
         theme: Theme | str | None = None,
+        flagging_mode: Literal["never", "manual"] | None = None,
+        flagging_options: list[str] | tuple[str, ...] | None = ("Like", "Dislike"),
+        flagging_dir: str = ".gradio/flagged",
         css: str | None = None,
         css_paths: str | Path | Sequence[str | Path] | None = None,
         js: str | None = None,
@@ -122,6 +127,9 @@ def __init__(
             title: a title for the interface; if provided, appears above chatbot in large font. Also used as the tab title when opened in a browser window.
             description: a description for the interface; if provided, appears above the chatbot and beneath the title in regular font. Accepts Markdown and HTML content.
             theme: a Theme object or a string representing a theme. If a string, will look for a built-in theme with that name (e.g. "soft" or "default"), or will attempt to load a theme from the Hugging Face Hub (e.g. "gradio/monochrome"). If None, will use the Default theme.
+            flagging_mode: one of "never", "manual". If "never", users will not see a button to flag an input and output. If "manual", users will see a button to flag.
+            flagging_options: a list of strings representing the options that users can choose from when flagging a message. Defaults to ["Like", "Dislike"]. These two case-sensitive strings will render as "thumbs up" and "thumbs down" icon respectively next to each bot message, but any other strings appear under a separate flag icon.
+            flagging_dir: path to the the directory where flagged data is stored. If the directory does not exist, it will be created.
             css: Custom css as a code string. This css will be included in the demo webpage.
             css_paths: Custom css as a pathlib.Path to a css file or a list of such paths. This css files will be read, concatenated, and included in the demo webpage. If the `css` parameter is also set, the css from `css` will be included first.
             js: Custom js as a code string. The custom js should be in the form of a single js function. This function will automatically be executed when the page loads. For more flexibility, use the head parameter to insert js inside <script> tags.
@@ -214,6 +222,18 @@ def __init__(
                 if self._additional_inputs_in_examples:
                     break
 
+        if flagging_mode is None:
+            flagging_mode = os.getenv("GRADIO_CHAT_FLAGGING_MODE", "never")  # type: ignore
+        if flagging_mode in ["manual", "never"]:
+            self.flagging_mode = flagging_mode
+        else:
+            raise ValueError(
+                "Invalid value for `flagging_mode` parameter."
+                "Must be: 'manual' or 'never'."
+            )
+        self.flagging_options = flagging_options
+        self.flagging_dir = flagging_dir
+
         with self:
             with Column():
                 if title:
@@ -501,6 +521,12 @@ def _setup_events(self) -> None:
                 show_api=False,
             ).success(**submit_fn_kwargs).success(**synchronize_chat_state_kwargs)
 
+        if self.flagging_mode != "never":
+            flagging_callback = ChatCSVLogger()
+            flagging_callback.setup(self.flagging_dir)
+            self.chatbot.feedback_options = self.flagging_options
+            self.chatbot.like(flagging_callback.flag, self.chatbot)
+
     def _setup_stop_events(
         self, event_triggers: list[Callable], events_to_cancel: list[Dependency]
     ) -> None:

gradio/components/chatbot.py

@@ -194,6 +194,7 @@ def __init__(
         avatar_images: tuple[str | Path | None, str | Path | None] | None = None,
         sanitize_html: bool = True,
         render_markdown: bool = True,
+        feedback_options: list[str] | tuple[str, ...] | None = ("Like", "Dislike"),
         bubble_full_width=None,
         line_breaks: bool = True,
         layout: Literal["panel", "bubble"] | None = None,
@@ -232,6 +233,7 @@ def __init__(
             avatar_images: Tuple of two avatar image paths or URLs for user and bot (in that order). Pass None for either the user or bot image to skip. Must be within the working directory of the Gradio app or an external URL.
             sanitize_html: If False, will disable HTML sanitization for chatbot messages. This is not recommended, as it can lead to security vulnerabilities.
             render_markdown: If False, will disable Markdown rendering for chatbot messages.
+            feedback_options: A list of strings representing the feedback options that will be displayed to the user. The exact case-sensitive strings "Like" and "Dislike" will render as thumb icons, but any other choices will appear under a separate flag icon.
             bubble_full_width: Deprecated.
             line_breaks: If True (default), will enable Github-flavored Markdown line breaks in chatbot messages. If False, single new lines will be ignored. Only applies if `render_markdown` is True.
             layout: If "panel", will display the chatbot in a llm style layout. If "bubble", will display the chatbot with message bubbles, with the user and bot messages on alterating sides. Will default to "bubble".
@@ -287,6 +289,7 @@ def __init__(
         self.layout = layout
         self.show_copy_all_button = show_copy_all_button
         self.allow_file_downloads = allow_file_downloads
+        self.feedback_options = feedback_options
         super().__init__(
             label=label,
             every=every,

gradio/events.py

@@ -300,9 +300,9 @@ def __init__(self, target: Block | None, data: Any):
         """
         The value of the liked/disliked item.
         """
-        self.liked: bool = data.get("liked", True)
+        self.liked: bool | str = data.get("liked", True)
         """
-        True if the item was liked, False if disliked.
+        True if the item was liked, False if disliked, or string value if any other feedback.
         """
 
 

gradio/flagging.py

@@ -3,6 +3,7 @@
 import contextlib
 import csv
 import datetime
+import json
 import os
 import re
 import time
@@ -17,6 +18,7 @@
 
 import gradio as gr
 from gradio import utils, wasm_utils
+from gradio.events import LikeData
 
 if TYPE_CHECKING:
     from gradio.components import Component
@@ -344,6 +346,50 @@ def flag(
         return line_count
 
 
+class ChatCSVLogger:
+    """
+    Flagging callback for chat conversations.
+    Flagged conversations and like/dislike reactions are logged to a CSV file on the machine running the gradio app.
+    """
+
+    def __init__(self):
+        pass
+
+    def setup(self, flagging_dir: str):
+        self.flagging_dir = flagging_dir
+        os.makedirs(flagging_dir, exist_ok=True)
+
+    def flag(
+        self,
+        like_data: LikeData,
+        messages: list,
+    ):
+        flagging_dir = self.flagging_dir
+        log_filepath = Path(flagging_dir) / "log.csv"
+        is_new = not Path(log_filepath).exists()
+
+        feedback = (
+            "Like"
+            if like_data.liked is True
+            else "Dislike"
+            if like_data.liked is False
+            else like_data.liked
+        )
+        csv_data = [
+            json.dumps(messages),
+            like_data.index,
+            feedback,
+            str(datetime.datetime.now()),
+        ]
+
+        with open(log_filepath, "a", encoding="utf-8", newline="") as csvfile:
+            if is_new:
+                writer = csv.writer(csvfile)
+                writer.writerow(["conversation", "index", "value", "flag", "timestamp"])
+            writer = csv.writer(csvfile)
+            writer.writerow(utils.sanitize_list_for_csv(csv_data))
+
+
 class FlagMethod:
     """
     Helper class that contains the flagging options and calls the flagging method. Also

guides/04_additional-features/09_environment-variables.md

@@ -185,6 +185,18 @@ Environment variables in Gradio provide a way to customize your applications and
   export GRADIO_RESET_EXAMPLES_CACHE="True"
   ```
 
+### 20. `GRADIO_CHAT_FLAGGING_MODE`
+
+- **Description**: Controls whether users can flag messages in `gr.ChatInterface` applications. Similar to `GRADIO_FLAGGING_MODE` but specifically for chat interfaces.
+- **Default**: `"never"`
+- **Options**: `"never"`, `"manual"`
+- **Example**:
+  ```sh
+  export GRADIO_CHAT_FLAGGING_MODE="manual"
+  ```
+
+
+
 ## How to Set Environment Variables
 
 To set environment variables in your terminal, use the `export` command followed by the variable name and its value. For example:

guides/05_chatbots/01_creating-a-chatbot-fast.md

@@ -341,6 +341,14 @@ To use the endpoint, you should use either the [Gradio Python Client](/guides/ge
 * Slack bot [[tutorial]](../guides/creating-a-slack-bot-from-a-gradio-app)
 * Website widget [[tutorial]](../guides/creating-a-website-widget-from-a-gradio-chatbot)
 
+## Collecting Feedback
+
+To gather feedback on your generations, set `gr.ChatInterface(flagging_mode="manual")` and users can thumbs-up and down assistant responses. Each flagged response, along with the entire chat history, will get saved in a CSV file in the app folder (or wherever `flagging_dir` specifies). 
+
+You can also specify more feedback options via `flagging_options`, which will appear under a dedicated flag button. Here's an example that shows several flagging options. Because the case-sensitive string "Like" is one of the flagging options, the user will see a "thumbs up" icon next to each assistant message. The three other flagging options will appear under a dedicated "flag" icon.
+
+$code_chatinterface_streaming_echo
+
 ## What's Next?
 
 Now that you've learned about the `gr.ChatInterface` class and how it can be used to create chatbot UIs quickly, we recommend reading one of the following:

js/atoms/src/IconButtonWrapper.svelte

@@ -58,8 +58,12 @@
 		margin-right: var(--spacing-xxs);
 	}
 
-	.icon-button-wrapper :global(a.download-link:not(:last-child)::after),
-	.icon-button-wrapper :global(button:not(:last-child)::after) {
+	.icon-button-wrapper
+		:global(
+			a.download-link:not(:last-child):not(.extra-feedback-option)::after
+		),
+	.icon-button-wrapper
+		:global(button:not(:last-child):not(.extra-feedback-option)::after) {
 		content: "";
 		position: absolute;
 		right: -4.5px;

js/chatbot/Index.svelte

@@ -32,6 +32,7 @@
 	export let root: string;
 	export let _selectable = false;
 	export let likeable = false;
+	export let feedback_options: string[] = ["Like", "Dislike"];
 	export let show_share_button = false;
 	export let rtl = false;
 	export let show_copy_button = true;
@@ -126,6 +127,7 @@
 			i18n={gradio.i18n}
 			selectable={_selectable}
 			{likeable}
+			{feedback_options}
 			{show_share_button}
 			{show_copy_all_button}
 			value={_value}

js/chatbot/shared/ButtonPanel.svelte

@@ -6,6 +6,7 @@
 	import { Retry, Undo, Edit, Check, Clear } from "@gradio/icons";
 	import { IconButtonWrapper, IconButton } from "@gradio/atoms";
 	export let likeable: boolean;
+	export let feedback_options: string[];
 	export let show_retry: boolean;
 	export let show_undo: boolean;
 	export let show_edit: boolean;
@@ -93,7 +94,7 @@
 					/>
 				{/if}
 				{#if likeable}
-					<LikeDislike {handle_action} />
+					<LikeDislike {handle_action} {feedback_options} />
 				{/if}
 			{/if}
 		</IconButtonWrapper>

js/chatbot/shared/ChatBot.svelte

@@ -67,6 +67,7 @@
 	export let generating = false;
 	export let selectable = false;
 	export let likeable = false;
+	export let feedback_options: string[];
 	export let editable: "user" | "all" | null = null;
 	export let show_share_button = false;
 	export let show_copy_all_button = false;
@@ -202,11 +203,17 @@
 				value: edit_message
 			});
 		} else {
+			let feedback =
+				selected === "like"
+					? true
+					: selected === "dislike"
+						? false
+						: selected?.substring(9); // remove "feedback:" prefix
 			if (msg_format === "tuples") {
 				dispatch("like", {
 					index: message.index,
 					value: message.content,
-					liked: selected === "like"
+					liked: feedback
 				});
 			} else {
 				if (!groupedMessages) return;
@@ -218,9 +225,9 @@
 				];
 
 				dispatch("like", {
-					index: [first.index, last.index] as [number, number],
+					index: first.index as number,
 					value: message_group.map((m) => m.content),
-					liked: selected === "like"
+					liked: feedback
 				});
 			}
 		}
@@ -301,6 +308,7 @@
 					{_components}
 					{generating}
 					{msg_format}
+					{feedback_options}
 					show_like={role === "user" ? likeable && like_user_message : likeable}
 					show_retry={_retryable && is_last_bot_message(messages, value)}
 					show_undo={_undoable && is_last_bot_message(messages, value)}

js/chatbot/shared/Flag.svelte

@@ -0,0 +1,10 @@
+<svg
+	id="icon"
+	xmlns="http://www.w3.org/2000/svg"
+	viewBox="0 0 32 32"
+	fill="none"
+	><path
+		fill="currentColor"
+		d="M6,30H4V2H28l-5.8,9L28,20H6ZM6,18H24.33L19.8,11l4.53-7H6Z"
+	/></svg
+>