feat: add sound and browser notifications for agent state changes (All-Hands-AI#6530)

Co-authored-by: openhands <[email protected]>
Co-authored-by: amanape <[email protected]>

Author: 3 people

frontend/__tests__/components/feedback-actions.test.tsx

@@ -1,6 +1,7 @@
-import { render, screen, within } from "@testing-library/react";
+import { screen, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { afterEach, describe, expect, it, vi } from "vitest";
+import { renderWithProviders } from "test-utils";
 import { TrajectoryActions } from "#/components/features/trajectory/trajectory-actions";
 
 describe("TrajectoryActions", () => {
@@ -14,7 +15,7 @@ describe("TrajectoryActions", () => {
   });
 
   it("should render correctly", () => {
-    render(
+    renderWithProviders(
       <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}
@@ -25,10 +26,11 @@ describe("TrajectoryActions", () => {
     const actions = screen.getByTestId("feedback-actions");
     within(actions).getByTestId("positive-feedback");
     within(actions).getByTestId("negative-feedback");
+    within(actions).getByTestId("export-trajectory");
   });
 
   it("should call onPositiveFeedback when positive feedback is clicked", async () => {
-    render(
+    renderWithProviders(
       <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}
@@ -43,7 +45,7 @@ describe("TrajectoryActions", () => {
   });
 
   it("should call onNegativeFeedback when negative feedback is clicked", async () => {
-    render(
+    renderWithProviders(
       <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}
@@ -58,7 +60,7 @@ describe("TrajectoryActions", () => {
   });
 
   it("should call onExportTrajectory when negative feedback is clicked", async () => {
-    render(
+    renderWithProviders(
       <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}

frontend/src/assets/notification.mp3

@@ -9,12 +9,21 @@ import {
   useWsClient,
   WsClientProviderStatus,
 } from "#/context/ws-client-provider";
+import { useNotification } from "#/hooks/useNotification";
+import { browserTab } from "#/utils/browser-tab";
+
+const notificationStates = [
+  AgentState.AWAITING_USER_INPUT,
+  AgentState.FINISHED,
+  AgentState.AWAITING_USER_CONFIRMATION,
+];
 
 export function AgentStatusBar() {
   const { t, i18n } = useTranslation();
   const { curAgentState } = useSelector((state: RootState) => state.agent);
   const { curStatusMessage } = useSelector((state: RootState) => state.status);
   const { status } = useWsClient();
+  const { notify } = useNotification();
 
   const [statusMessage, setStatusMessage] = React.useState<string>("");
 
@@ -45,13 +54,40 @@ export function AgentStatusBar() {
     updateStatusMessage();
   }, [curStatusMessage.id]);
 
+  // Handle window focus/blur
+  React.useEffect(() => {
+    if (typeof window === "undefined") return undefined;
+
+    const handleFocus = () => {
+      browserTab.stopNotification();
+    };
+
+    window.addEventListener("focus", handleFocus);
+    return () => {
+      window.removeEventListener("focus", handleFocus);
+      browserTab.stopNotification();
+    };
+  }, []);
+
   React.useEffect(() => {
     if (status === WsClientProviderStatus.DISCONNECTED) {
       setStatusMessage("Connecting...");
     } else {
       setStatusMessage(AGENT_STATUS_MAP[curAgentState].message);
+      if (notificationStates.includes(curAgentState)) {
+        const message = t(AGENT_STATUS_MAP[curAgentState].message);
+        notify(t(AGENT_STATUS_MAP[curAgentState].message), {
+          body: t(`Agent state changed to ${curAgentState}`),
+          playSound: true,
+        });
+
+        // Update browser tab if window exists and is not focused
+        if (typeof document !== "undefined" && !document.hasFocus()) {
+          browserTab.startNotification(message);
+        }
+      }
     }
-  }, [curAgentState]);
+  }, [curAgentState, notify, t]);
 
   return (
     <div className="flex flex-col items-center">

frontend/src/components/features/controls/agent-status-bar.tsx

@@ -1,3 +1,4 @@
+import { useTranslation } from "react-i18next";
 import ThumbsUpIcon from "#/icons/thumbs-up.svg?react";
 import ThumbDownIcon from "#/icons/thumbs-down.svg?react";
 import ExportIcon from "#/icons/export.svg?react";
@@ -14,22 +15,27 @@ export function TrajectoryActions({
   onNegativeFeedback,
   onExportTrajectory,
 }: TrajectoryActionsProps) {
+  const { t } = useTranslation();
+
   return (
     <div data-testid="feedback-actions" className="flex gap-1">
       <TrajectoryActionButton
         testId="positive-feedback"
         onClick={onPositiveFeedback}
         icon={<ThumbsUpIcon width={15} height={15} />}
+        tooltip={t("BUTTON$MARK_HELPFUL")}
       />
       <TrajectoryActionButton
         testId="negative-feedback"
         onClick={onNegativeFeedback}
         icon={<ThumbDownIcon width={15} height={15} />}
+        tooltip={t("BUTTON$MARK_NOT_HELPFUL")}
       />
       <TrajectoryActionButton
         testId="export-trajectory"
         onClick={onExportTrajectory}
         icon={<ExportIcon width={15} height={15} />}
+        tooltip={t("BUTTON$EXPORT_CONVERSATION")}
       />
     </div>
   );

frontend/src/components/features/trajectory/trajectory-actions.tsx

@@ -1,15 +1,19 @@
+import { Tooltip } from "@heroui/react";
+
 interface TrajectoryActionButtonProps {
   testId?: string;
   onClick: () => void;
   icon: React.ReactNode;
+  tooltip?: string;
 }
 
 export function TrajectoryActionButton({
   testId,
   onClick,
   icon,
+  tooltip,
 }: TrajectoryActionButtonProps) {
-  return (
+  const button = (
     <button
       type="button"
       data-testid={testId}
@@ -19,4 +23,14 @@ export function TrajectoryActionButton({
       {icon}
     </button>
   );
+
+  if (tooltip) {
+    return (
+      <Tooltip content={tooltip} closeDelay={100}>
+        {button}
+      </Tooltip>
+    );
+  }
+
+  return button;
 }

frontend/src/components/shared/buttons/trajectory-action-button.tsx

@@ -20,6 +20,7 @@ const saveSettingsMutationFn = async (settings: Partial<PostSettings>) => {
     github_token: settings.github_token,
     unset_github_token: settings.unset_github_token,
     enable_default_condenser: settings.ENABLE_DEFAULT_CONDENSER,
+    enable_sound_notifications: settings.ENABLE_SOUND_NOTIFICATIONS,
     user_consents_to_analytics: settings.user_consents_to_analytics,
   };
 

frontend/src/hooks/mutation/use-save-settings.ts

@@ -20,6 +20,7 @@ const getSettingsQueryFn = async () => {
     REMOTE_RUNTIME_RESOURCE_FACTOR: apiSettings.remote_runtime_resource_factor,
     GITHUB_TOKEN_IS_SET: apiSettings.github_token_is_set,
     ENABLE_DEFAULT_CONDENSER: apiSettings.enable_default_condenser,
+    ENABLE_SOUND_NOTIFICATIONS: apiSettings.enable_sound_notifications,
     USER_CONSENTS_TO_ANALYTICS: apiSettings.user_consents_to_analytics,
   };
 };

frontend/src/hooks/query/use-settings.ts

@@ -0,0 +1,56 @@
+import { useCallback, useRef } from "react";
+import notificationSound from "../assets/notification.mp3";
+import { useCurrentSettings } from "../context/settings-context";
+
+export const useNotification = () => {
+  const { settings } = useCurrentSettings();
+  const audioRef = useRef<HTMLAudioElement | undefined>(undefined);
+
+  // Initialize audio only in browser environment
+  if (typeof window !== "undefined" && !audioRef.current) {
+    audioRef.current = new Audio(notificationSound);
+    audioRef.current.volume = 0.5;
+  }
+
+  const notify = useCallback(
+    async (
+      title: string,
+      options?: NotificationOptions & { playSound?: boolean },
+    ): Promise<Notification | undefined> => {
+      if (typeof window === "undefined") return undefined;
+
+      // Only play sound if:
+      // 1. Explicitly requested via playSound option
+      // 2. Sound notifications are enabled in settings
+      // 3. Audio is available
+      // 4. Not a settings-related notification
+      if (
+        options?.playSound === true && // Must be explicitly true
+        settings?.ENABLE_SOUND_NOTIFICATIONS &&
+        audioRef.current &&
+        !title.includes("BUTTON$") // Don't play for button/settings actions
+      ) {
+        // Reset and play sound
+        audioRef.current.currentTime = 0;
+        audioRef.current.play().catch(() => {
+          // Ignore autoplay errors
+        });
+      }
+
+      if (Notification.permission === "default") {
+        await Notification.requestPermission();
+      }
+
+      if (Notification.permission === "granted") {
+        // Remove playSound from options before passing to Notification
+        const { playSound, ...notificationOptions } = options || {};
+        return new Notification(title, notificationOptions);
+      }
+
+      return undefined;
+    },
+    [settings?.ENABLE_SOUND_NOTIFICATIONS],
+  );
+
+  return { notify };
+};