CodeActAgent: Delegate to BrowsingAgent for browsing tasks

Author: li-boxuan

agenthub/browsing_agent/browsing_agent.py

@@ -88,6 +88,8 @@ def step(self, state: State) -> Action:
         - AgentFinishAction() - end the interaction
         """
         goal = state.get_current_user_intent()
+        if goal is None:
+            goal = state.inputs['task']
         messages = []
         prev_actions = ''
         cur_axtree_txt = ''
@@ -101,7 +103,7 @@ def step(self, state: State) -> Action:
                 isinstance(prev_action, MessageAction) and prev_action.source != 'user'
             ):
                 # agent has responded, task finish.
-                return AgentFinishAction()
+                return AgentFinishAction(outputs={'content': prev_action.content})
 
         if isinstance(last_obs, BrowserOutputObservation):
             if last_obs.error:

agenthub/codeact_agent/codeact_agent.py

@@ -11,13 +11,15 @@
 from opendevin.controller.state.state import State
 from opendevin.events.action import (
     Action,
+    AgentDelegateAction,
     AgentFinishAction,
     BrowseInteractiveAction,
     CmdRunAction,
     IPythonRunCellAction,
     MessageAction,
 )
 from opendevin.events.observation import (
+    AgentDelegateObservation,
     BrowserOutputObservation,
     CmdOutputObservation,
     IPythonRunCellObservation,
@@ -88,6 +90,9 @@ def get_observation_message(obs) -> dict[str, str] | None:
     elif isinstance(obs, BrowserOutputObservation):
         content = 'OBSERVATION:\n' + truncate_observation(obs.content)
         return {'role': 'user', 'content': content}
+    elif isinstance(obs, AgentDelegateObservation):
+        content = 'OBSERVATION:\n' + truncate_observation(str(obs.outputs))
+        return {'role': 'user', 'content': content}
     return None
 
 
@@ -106,7 +111,7 @@ def truncate_observation(observation: str, max_chars: int = 10_000) -> str:
 
 
 class CodeActAgent(Agent):
-    VERSION = '1.5'
+    VERSION = '1.6'
     """
     The Code Act Agent is a minimalist agent.
     The agent works by passing the model a list of action-observation pairs and prompting the model to take the next step.
@@ -188,7 +193,7 @@ def step(self, state: State) -> Action:
         Returns:
         - CmdRunAction(command) - bash command to run
         - IPythonRunCellAction(code) - IPython code to run
-        - BrowseInteractiveAction(browsergym_command) - BrowserGym commands to run
+        - AgentDelegateAction(agent, inputs) - delegate action for (sub)task
         - MessageAction(content) - Message action to run (e.g. ask for clarification)
         - AgentFinishAction() - end the interaction
         """
@@ -260,12 +265,10 @@ def step(self, state: State) -> Action:
         elif browse_command := re.search(
             r'<execute_browse>(.*)</execute_browse>', action_str, re.DOTALL
         ):
-            # BrowserGym actions was found
-            browse_actions = browse_command.group(1).strip()
             thought = action_str.replace(browse_command.group(0), '').strip()
-            return BrowseInteractiveAction(
-                browser_actions=browse_actions, thought=thought
-            )
+            browse_actions = browse_command.group(1).strip()
+            task = f'{thought}. I should start with: {browse_actions}'
+            return AgentDelegateAction(agent='BrowsingAgent', inputs={'task': task})
         else:
             # We assume the LLM is GOOD enough that when it returns pure natural language
             # it want to talk to the user

agenthub/codeact_agent/prompt.py

@@ -16,7 +16,7 @@
 The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
 For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
 The assistant can browse the Internet with commands on behalf of the user by wrapping them with <execute_browse> and </execute_browse>.
-For example, you can browse a given URL by <execute_browse> goto("<URL>") </execute_browse>.
+For example, you can browse a given URL by <execute_browse> Tell me the usa's president using google search </execute_browse>.
 The assistant should attempt fewer things at a time instead of putting too much commands OR code in one "execute" block.
 The assistant can install Python packages using the %pip magic command in an IPython environment by using the following syntax: <execute_ipython> %pip install [package needed] </execute_ipython> and should always import packages and define variables before starting to use them."""
 
@@ -155,7 +155,7 @@ def index():
 ASSISTANT:
 Sure! Let me browse the server's homepage at http://127.0.0.1:5000:
 <execute_browse>
-goto("http://127.0.0.1:5000")
+Get the content on "http://127.0.0.1:5000"
 </execute_browse>
 
 USER: