Migrate multi-line-bash-related sandbox tests into runtime tests and fix multi-line issue

opendevin/core/logger.py

@@ -9,7 +9,7 @@
 from termcolor import colored
 
 DISABLE_COLOR_PRINTING = False
-DEBUG = False
+DEBUG = os.getenv('DEBUG', 'False').lower() in ['true', '1', 'yes']
 
 ColorType = Literal[
     'red',

opendevin/runtime/client/client.py

@@ -46,6 +46,7 @@
     Plugin,
 )
 from opendevin.runtime.server.files import insert_lines, read_lines
+from opendevin.runtime.utils import split_bash_commands
 
 app = FastAPI()
 
@@ -79,14 +80,23 @@ def _init_bash_shell(self, work_dir: str) -> None:
             r'\[PEXPECT_BEGIN\] ([a-z0-9_-]*)@([a-zA-Z0-9.-]*):(.+) \[PEXPECT_END\]'
         )
 
-        self.shell.sendline(f'export PS1="{self.__bash_PS1}"')
+        self.shell.sendline(f'export PS1="{self.__bash_PS1}"; export PS2=""')
         self.shell.expect(self.__bash_expect_regex)
 
         self.shell.sendline(f'cd {work_dir}')
         self.shell.expect(self.__bash_expect_regex)
 
     def _get_bash_prompt(self):
         ps1 = self.shell.after
+
+        # begin at the last occurence of '[PEXPECT_BEGIN]'.
+        # In multi-line bash commands, the prompt will be repeated
+        # and the matched regex captures all of them
+        # - we only want the last one (newest prompt)
+        _begin_pos = ps1.rfind('[PEXPECT_BEGIN]')
+        if _begin_pos != -1:
+            ps1 = ps1[_begin_pos:]
+
         # parse the ps1 to get username, hostname, and working directory
         matched = re.match(self.__bash_expect_regex, ps1)
         assert (
@@ -102,7 +112,7 @@ def _get_bash_prompt(self):
             prompt += '$'
         return prompt + ' '
 
-    def _execute_bash(self, command, keep_prompt: bool = True) -> tuple[str, int]:
+    def _execute_bash(self, command: str, keep_prompt: bool = True) -> tuple[str, int]:
         logger.debug(f'Executing command: {command}')
         self.shell.sendline(command)
         self.shell.expect(self.__bash_expect_regex)
@@ -129,10 +139,22 @@ async def run_action(self, action) -> Observation:
 
     async def run(self, action: CmdRunAction) -> CmdOutputObservation:
         try:
-            output, exit_code = self._execute_bash(action.command)
+            commands = split_bash_commands(action.command)
+            all_output = ''
+            for command in commands:
+                output, exit_code = self._execute_bash(command)
+                if all_output:
+                    # previous output already exists with prompt "user@hostname:working_dir #""
+                    # we need to add the command to the previous output,
+                    # so model knows the following is the output of another action)
+                    all_output = all_output.rstrip() + ' ' + command + '\r\n'
+
+                all_output += str(output) + '\r\n'
+                if exit_code != 0:
+                    break
             return CmdOutputObservation(
                 command_id=-1,
-                content=str(output),
+                content=all_output.rstrip('\r\n'),
                 command=action.command,
                 exit_code=exit_code,
             )

opendevin/runtime/client/runtime.py

@@ -58,7 +58,7 @@ def __init__(
         # TODO: We can switch to aiodocker when `get_od_sandbox_image` is updated to use aiodocker
         self.docker_client: docker.DockerClient = self._init_docker_client()
         self.container_image = (
-            config.sandbox.container_image
+            self.config.sandbox.container_image
             if container_image is None
             else container_image
         )
@@ -103,7 +103,7 @@ def _init_docker_client() -> docker.DockerClient:
     async def _init_container(
         self,
         sandbox_workspace_dir: str,
-        mount_dir: str,
+        mount_dir: str | None = None,
         plugins: list[PluginRequirement] | None = None,
     ):
         try:
@@ -124,6 +124,14 @@ async def _init_container(
             else:
                 port_mapping = {f'{self._port}/tcp': self._port}
 
+            if mount_dir is not None:
+                volumes = {mount_dir: {'bind': sandbox_workspace_dir, 'mode': 'rw'}}
+            else:
+                logger.warn(
+                    'Mount dir is not set, will not mount the workspace directory to the container.'
+                )
+                volumes = None
+
             container = self.docker_client.containers.run(
                 self.container_image,
                 command=(
@@ -139,7 +147,7 @@ async def _init_container(
                 name=self.container_name,
                 detach=True,
                 environment={'DEBUG': 'true'} if self.config.debug else None,
-                volumes={mount_dir: {'bind': sandbox_workspace_dir, 'mode': 'rw'}},
+                volumes=volumes,
             )
             logger.info(f'Container started. Server url: {self.api_url}')
             return container

opendevin/runtime/runtime.py

@@ -33,13 +33,13 @@
 from opendevin.storage import FileStore
 
 
-def _default_env_vars(config: SandboxConfig) -> dict[str, str]:
+def _default_env_vars(sandbox_config: SandboxConfig) -> dict[str, str]:
     ret = {}
     for key in os.environ:
         if key.startswith('SANDBOX_ENV_'):
             sandbox_key = key.removeprefix('SANDBOX_ENV_')
             ret[sandbox_key] = os.environ[key]
-    if config.enable_auto_lint:
+    if sandbox_config.enable_auto_lint:
         ret['ENABLE_AUTO_LINT'] = 'true'
     return ret
 

opendevin/runtime/server/runtime.py

@@ -115,7 +115,7 @@ async def run(self, action: CmdRunAction) -> Observation:
 
     async def run_ipython(self, action: IPythonRunCellAction) -> Observation:
         self._run_command(
-            ("cat > /tmp/opendevin_jupyter_temp.py <<'EOL'\n" f'{action.code}\n' 'EOL'),
+            f"cat > /tmp/opendevin_jupyter_temp.py <<'EOL'\n{action.code}\nEOL"
         )
 
         # run the code

opendevin/runtime/utils/bash.py

@@ -1,87 +1,49 @@
-def split_bash_commands(commands):
-    # States
-    NORMAL = 0
-    IN_SINGLE_QUOTE = 1
-    IN_DOUBLE_QUOTE = 2
-    IN_HEREDOC = 3
-
-    state = NORMAL
-    heredoc_trigger = None
-    result = []
-    current_command: list[str] = []
-
-    i = 0
-    while i < len(commands):
-        char = commands[i]
-
-        if state == NORMAL:
-            if char == "'":
-                state = IN_SINGLE_QUOTE
-            elif char == '"':
-                state = IN_DOUBLE_QUOTE
-            elif char == '\\':
-                # Check if this is escaping a newline
-                if i + 1 < len(commands) and commands[i + 1] == '\n':
-                    i += 1  # Skip the newline
-                    # Continue with the next line as part of the same command
-                    i += 1  # Move to the first character of the next line
-                    continue
-            elif char == '\n':
-                if not heredoc_trigger and current_command:
-                    result.append(''.join(current_command).strip())
-                    current_command = []
-            elif char == '<' and commands[i : i + 2] == '<<':
-                # Detect heredoc
-                state = IN_HEREDOC
-                i += 2  # Skip '<<'
-                while commands[i] == ' ':
-                    i += 1
-                start = i
-                while commands[i] not in [' ', '\n']:
-                    i += 1
-                heredoc_trigger = commands[start:i]
-                current_command.append(commands[start - 2 : i])  # Include '<<'
-                continue  # Skip incrementing i at the end of the loop
-            current_command.append(char)
-
-        elif state == IN_SINGLE_QUOTE:
-            current_command.append(char)
-            if char == "'" and commands[i - 1] != '\\':
-                state = NORMAL
+import bashlex
 
-        elif state == IN_DOUBLE_QUOTE:
-            current_command.append(char)
-            if char == '"' and commands[i - 1] != '\\':
-                state = NORMAL
+from opendevin.core.logger import opendevin_logger as logger
 
-        elif state == IN_HEREDOC:
-            current_command.append(char)
-            if (
-                char == '\n'
-                and heredoc_trigger
-                and commands[i + 1 : i + 1 + len(heredoc_trigger) + 1]
-                == heredoc_trigger + '\n'
-            ):
-                # Check if the next line starts with the heredoc trigger followed by a newline
-                i += (
-                    len(heredoc_trigger) + 1
-                )  # Move past the heredoc trigger and newline
-                current_command.append(
-                    heredoc_trigger + '\n'
-                )  # Include the heredoc trigger and newline
-                result.append(''.join(current_command).strip())
-                current_command = []
-                heredoc_trigger = None
-                state = NORMAL
-                continue
-
-        i += 1
-
-    # Add the last command if any
-    if current_command:
-        result.append(''.join(current_command).strip())
-
-    # Remove any empty strings from the result
-    result = [cmd for cmd in result if cmd]
 
+def split_bash_commands(commands):
+    try:
+        parsed = bashlex.parse(commands)
+    except bashlex.errors.ParsingError as e:
+        logger.error(
+            f'Failed to parse bash commands\n[input]: {commands}\n[error]: {e}'
+        )
+        # If parsing fails, return the original commands
+        return [commands]
+
+    result: list[str] = []
+    last_end = 0
+
+    for node in parsed:
+        start, end = node.pos
+
+        # Include any text between the last command and this one
+        if start > last_end:
+            between = commands[last_end:start]
+            logger.debug(f'BASH PARSING between: {between}')
+            if result:
+                result[-1] += between.rstrip()
+            elif between.strip():
+                # THIS SHOULD NOT HAPPEN
+                result.append(between.rstrip())
+
+        # Extract the command, preserving original formatting
+        command = commands[start:end].rstrip()
+        logger.debug(f'BASH PARSING command: {command}')
+        result.append(command)
+
+        last_end = end
+
+    # Add any remaining text after the last command to the last command
+    remaining = commands[last_end:].rstrip()
+    logger.debug(f'BASH PARSING remaining: {remaining}')
+    if last_end < len(commands) and result:
+        result[-1] += remaining
+        logger.debug(f'BASH PARSING result[-1] += remaining: {result[-1]}')
+    elif last_end < len(commands):
+        if remaining:
+            result.append(remaining)
+            logger.debug(f'BASH PARSING result.append(remaining): {result[-1]}')
     return result

pyproject.toml

@@ -39,6 +39,7 @@ pathspec = "^0.12.1"
 google-cloud-aiplatform = "*"
 grep-ast = "0.3.2"
 tree-sitter = "0.21.3"
+bashlex = "^0.18"
 
 [tool.poetry.group.llama-index.dependencies]
 llama-index = "*"
@@ -72,6 +73,7 @@ reportlab = "*"
 [tool.coverage.run]
 concurrency = ["gevent"]
 
+
 [tool.poetry.group.runtime.dependencies]
 jupyterlab = "*"
 notebook = "*"
@@ -105,6 +107,7 @@ ignore = ["D1"]
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 
+
 [tool.poetry.group.evaluation.dependencies]
 streamlit = "*"
 whatthepatch = "*"