feat!(ContentToolResult): add a new model_format parameter

Author: cpsievert

chatlas/_anthropic.py

@@ -488,12 +488,15 @@ def _as_content_block(content: Content) -> "ContentBlockParam":
                 "input": content.arguments,
             }
         elif isinstance(content, ContentToolResult):
-            return {
+            res: ToolResultBlockParam = {
                 "type": "tool_result",
                 "tool_use_id": content.id,
-                "content": content.get_final_value(),
                 "is_error": content.error is not None,
             }
+            # Anthropic supports non-text contents like ImageBlockParam
+            res["content"] = content.get_formatted_value()  # type: ignore
+            return res
+
         raise ValueError(f"Unknown content type: {type(content)}")
 
     @staticmethod

chatlas/_content.py

@@ -226,6 +226,15 @@ class ContentToolResult(Content):
     ----------
     value
         The value returned by the tool/function (to be sent to the model).
+    model_format
+        The format used for sending the value to the model. The default,
+        `"auto"`, first attempts to format the value as a JSON string. If that
+        fails, it gets converted to a string via `str()`. To force
+        `json.dumps()` or `str()`, set to `"json"` or `"str"`. Finally,
+        `"as_is"` is useful for doing your own formatting and/or passing a
+        non-string value (e.g., a list or dict) straight to the model.
+        Non-string values are useful for tools that return images or other
+        'known' non-text content types.
     error
         An exception that occurred during the tool request. If this is set, the
         error message sent to the model and the value is ignored.
@@ -239,6 +248,7 @@ class ContentToolResult(Content):
 
     # public
     value: Any
+    model_format: Literal["auto", "json", "str", "as_is"] = "auto"
     error: Optional[Exception] = None
     extra: Any = None
 
@@ -266,22 +276,11 @@ def arguments(self):
             )
         return self.request.arguments
 
-    def _get_value(self, pretty: bool = False) -> str:
-        if self.error:
-            return f"Tool call failed with error: '{self.error}'"
-        if not pretty:
-            return str(self.value)
-        try:
-            json_val = json.loads(self.value)  # type: ignore
-            return pformat(json_val, indent=2, sort_dicts=False)
-        except:  # noqa
-            return str(self.value)
-
     # Primarily used for `echo="all"`...
     def __str__(self):
         prefix = "✅ tool result" if not self.error else "❌ tool error"
         comment = f"# {prefix} ({self.id})"
-        value = self._get_value(pretty=True)
+        value = self._get_display_value()
         return f"""```python\n{comment}\n{value}\n```"""
 
     # ... and for displaying in the notebook
@@ -295,9 +294,52 @@ def __repr__(self, indent: int = 0):
             res += f" error='{self.error}'"
         return res + ">"
 
-    # The actual value to send to the model
-    def get_final_value(self) -> str:
-        return self._get_value()
+    # Format the value for display purposes
+    def _get_display_value(self) -> object:
+        if self.error:
+            return f"Tool call failed with error: '{self.error}'"
+
+        val = self.value
+
+        # If value is already a dict or list, format it directly
+        if isinstance(val, (dict, list)):
+            return pformat(val, indent=2, sort_dicts=False)
+
+        # For string values, try to parse as JSON
+        if isinstance(val, str):
+            try:
+                json_val = json.loads(val)
+                return pformat(json_val, indent=2, sort_dicts=False)
+            except json.JSONDecodeError:
+                # Not valid JSON, return as string
+                return val
+
+        return val
+
+    def get_model_value(self) -> object:
+        "Get the actual value sent to the model."
+
+        if self.error:
+            return f"Tool call failed with error: '{self.error}'"
+
+        val, mode = (self.value, self.model_format)
+
+        if isinstance(val, str):
+            return val
+
+        if mode == "auto":
+            try:
+                return json.dumps(val)
+            except Exception:
+                return str(val)
+        elif mode == "json":
+            return json.dumps(val)
+        elif mode == "str":
+            return str(val)
+        elif mode == "as_is":
+            return val
+        else:
+            raise ValueError(f"Unknown format mode: {mode}")
 
     def tagify(self) -> "TagChild":
         """
@@ -317,7 +359,7 @@ def tagify(self) -> "TagChild":
             header = f"❌ Failed to call tool <code>{self.name}</code>"
 
         args = self._arguments_str()
-        content = self._get_value(pretty=True)
+        content = self._get_display_value()
 
         return HTML(
             textwrap.dedent(f"""

chatlas/_google.py

@@ -432,7 +432,7 @@ def _as_part_type(self, content: Content) -> "Part":
             if content.error:
                 resp = {"error": content.error}
             else:
-                resp = {"result": str(content.value)}
+                resp = {"result": content.get_model_value()}
             return Part(
                 # TODO: seems function response parts might need role='tool'???
                 # https://github.com/googleapis/python-genai/blame/c8cfef85c/README.md#L344

chatlas/_openai.py

@@ -499,8 +499,8 @@ def _as_message_param(turns: list[Turn]) -> list["ChatCompletionMessageParam"]:
                     elif isinstance(x, ContentToolResult):
                         tool_results.append(
                             ChatCompletionToolMessageParam(
-                                # TODO: a tool could return an image!?!
-                                content=x.get_final_value(),
+                                # Currently, OpenAI only allows for text content in tool results
+                                content=cast(str, x.get_formatted_value()),
                                 tool_call_id=x.id,
                                 role="tool",
                             )

tests/conftest.py

@@ -256,3 +256,8 @@ def assert_pdf_local(chat_fun: ChatFun):
     wait=wait_exponential(min=1, max=60),
     reraise=True,
 )
+
+
+@pytest.fixture
+def test_images_dir():
+    return Path(__file__).parent / "images"

tests/images/dice.png

@@ -1,5 +1,7 @@
+import base64
+
 import pytest
-from chatlas import ChatAnthropic
+from chatlas import ChatAnthropic, ContentToolResult
 
 from .conftest import (
     assert_data_extraction,
@@ -96,3 +98,33 @@ def test_anthropic_empty_response():
     chat.chat("Respond with only two blank lines")
     resp = chat.chat("What's 1+1? Just give me the number")
     assert "2" == str(resp).strip()
+
+
+def test_anthropic_image_tool(test_images_dir):
+    def get_picture():
+        "Returns an image"
+        # Local copy of https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png
+        with open(test_images_dir / "dice.png", "rb") as image:
+            bytez = image.read()
+        res = [
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": "image/png",
+                    "data": base64.b64encode(bytez).decode("utf-8"),
+                },
+            }
+        ]
+        return ContentToolResult(value=res, model_format="as_is")
+
+    chat = ChatAnthropic()
+    chat.register_tool(get_picture)
+
+    res = chat.chat(
+        "You have a tool called 'get_picture' available to you. "
+        "When called, it returns an image. "
+        "Tell me what you see in the image."
+    )
+
+    assert "dice" in res.get_content()

tests/test_provider_anthropic.py

@@ -0,0 +1,88 @@
+import pytest
+
+from chatlas import ChatSnowflake
+
+from .conftest import assert_data_extraction, assert_turns_existing, assert_turns_system
+
+
+def test_openai_simple_request():
+    chat = ChatSnowflake(
+        connection_name="posit",
+        system_prompt="Be as terse as possible; no punctuation",
+    )
+    chat.chat("What is 1 + 1?")
+    turn = chat.get_last_turn()
+    assert turn is not None
+
+    # No token / finish_reason info available
+    # assert turn.tokens is not None
+    # assert len(turn.tokens) == 2
+    # assert turn.tokens[0] == 27
+    # # Not testing turn.tokens[1] because it's not deterministic. Typically 1 or 2.
+    # assert turn.finish_reason == "stop"
+
+
+@pytest.mark.asyncio
+async def test_openai_simple_streaming_request():
+    chat = ChatSnowflake(
+        connection_name="posit",
+        system_prompt="Be as terse as possible; no punctuation",
+    )
+    res = []
+    async for x in await chat.stream_async("What is 1 + 1?"):
+        res.append(x)
+    assert "2" in "".join(res)
+    turn = chat.get_last_turn()
+
+    # No token / finish_reason info available
+    # assert turn is not None
+    # assert turn.finish_reason == "stop"
+
+
+def test_openai_respects_turns_interface():
+    def chat_fun(**kwargs):
+        return ChatSnowflake(connection_name="posit", **kwargs)
+
+    assert_turns_system(chat_fun)
+    assert_turns_existing(chat_fun)
+
+
+#
+# def test_openai_tool_variations():
+#     def chat_fun(**kwargs):
+#         return ChatSnowflake(connection_name="posit", **kwargs)
+#
+#     assert_tools_simple(chat_fun)
+#     assert_tools_simple_stream_content(chat_fun)
+#     assert_tools_parallel(chat_fun)
+#     assert_tools_sequential(chat_fun, total_calls=6)
+#
+#
+# @pytest.mark.asyncio
+# async def test_openai_tool_variations_async():
+#     def chat_fun(**kwargs):
+#         return ChatSnowflake(connection_name="posit", **kwargs)
+#
+#     await assert_tools_async(chat_fun)
+
+
+def test_data_extraction():
+    def chat_fun():
+        return ChatSnowflake(connection_name="posit")
+
+    assert_data_extraction(chat_fun)
+
+
+# def test_openai_images():
+#     def chat_fun(**kwargs):
+#         return ChatSnowflake(connection_name="posit", **kwargs)
+#
+#     assert_images_inline(chat_fun)
+#     assert_images_remote(chat_fun)
+#
+#
+# def test_openai_pdf():
+#     def chat_fun(**kwargs):
+#         return ChatSnowflake(connection_name="posit", **kwargs)
+#
+#     assert_pdf_local(chat_fun)