Skip to content

Fix issue #7658: [Bug]: BadRequestError from ContentPolicyViolationError #7660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 3, 2025
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions frontend/src/i18n/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -3875,6 +3875,21 @@
"tr": "OpenHands kredileriniz tükendi",
"de": "Ihre OpenHands-Guthaben sind aufgebraucht"
},
"STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION": {
"en": "Content policy violation. The output was blocked by content filtering policy.",
"ja": "コンテンツポリシー違反。出力はコンテンツフィルタリングポリシーによってブロックされました。",
"zh-CN": "内容政策违规。输出被内容过滤策略阻止。",
"zh-TW": "內容政策違規。輸出被內容過濾政策阻止。",
"ko-KR": "콘텐츠 정책 위반. 출력이 콘텐츠 필터링 정책에 의해 차단되었습니다.",
"de": "Verstoß gegen Inhaltsrichtlinien. Die Ausgabe wurde durch die Inhaltsfilterrichtlinie blockiert.",
"no": "Brudd på innholdspolicy. Utdata ble blokkert av innholdsfiltrering.",
"it": "Violazione della policy sui contenuti. L'output è stato bloccato dalla policy di filtraggio dei contenuti.",
"pt": "Violação da política de conteúdo. A saída foi bloqueada pela política de filtragem de conteúdo.",
"es": "Violación de la política de contenido. La salida fue bloqueada por la política de filtrado de contenido.",
"ar": "انتهاك سياسة المحتوى. تم حظر المخرجات بواسطة سياسة تصفية المحتوى.",
"fr": "Violation de la politique de contenu. La sortie a été bloquée par la politique de filtrage de contenu.",
"tr": "İçerik politikası ihlali. Çıktı, içerik filtreleme politikası tarafından engellendi."
},
"STATUS$ERROR_RUNTIME_DISCONNECTED": {
"en": "There was an error while connecting to the runtime. Please refresh the page.",
"zh-CN": "运行时已断开连接",
Expand Down
10 changes: 9 additions & 1 deletion openhands/controller/agent_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
APIError,
AuthenticationError,
BadRequestError,
ContentPolicyViolationError,
ContextWindowExceededError,
InternalServerError,
NotFoundError,
Expand Down Expand Up @@ -255,7 +256,13 @@ async def _react_to_exception(
elif isinstance(e, RateLimitError):
await self.set_agent_state_to(AgentState.RATE_LIMITED)
return
self.status_callback('error', err_id, self.state.last_error)
elif isinstance(e, ContentPolicyViolationError) or (
isinstance(e, BadRequestError)
and 'ContentPolicyViolationError' in str(e)
):
err_id = 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
self.state.last_error = str(e)
await self.status_callback('error', err_id, self.state.last_error)

# Set the agent state to ERROR after storing the reason
await self.set_agent_state_to(AgentState.ERROR)
Expand Down Expand Up @@ -283,6 +290,7 @@ async def _step_with_exception_handling(self):
or isinstance(e, InternalServerError)
or isinstance(e, AuthenticationError)
or isinstance(e, RateLimitError)
or isinstance(e, ContentPolicyViolationError)
or isinstance(e, LLMContextWindowExceedError)
):
reported = e
Expand Down
74 changes: 73 additions & 1 deletion tests/unit/resolver/github/test_issue_handler_error_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import httpx
import pytest
from litellm.exceptions import RateLimitError
from litellm.exceptions import BadRequestError, ContentPolicyViolationError, RateLimitError

from openhands.core.config import LLMConfig
from openhands.events.action.message import MessageAction
Expand Down Expand Up @@ -277,3 +277,75 @@ def test_guess_success_exhausts_retries(mock_completion, default_config):
assert (
mock_completion.call_count == default_config.num_retries
) # Initial call + retries


@patch('openhands.llm.llm.litellm_completion')
def test_guess_success_content_policy_violation_direct(mock_completion, default_config):
"""Test that ContentPolicyViolationError is handled correctly when raised directly."""
# Simulate ContentPolicyViolationError
mock_completion.side_effect = ContentPolicyViolationError(
'Content policy violation', llm_provider='test_provider', model='test_model'
)

# Initialize LLM and handler
llm = LLM(config=default_config)
handler = ServiceContextPR(
GithubPRHandler('test-owner', 'test-repo', 'test-token'), default_config
)
handler.llm = llm

# Mock issue and history
issue = Issue(
owner='test-owner',
repo='test-repo',
number=1,
title='Test Issue',
body='This is a test issue.',
thread_comments=['Please improve error handling'],
)
history = [MessageAction(content='Fixed error handling.')]

# Call guess_success and expect it to raise ContentPolicyViolationError
with pytest.raises(ContentPolicyViolationError):
handler.guess_success(issue, history)

# Assertions
assert (
mock_completion.call_count == default_config.num_retries
) # Initial call + retries


@patch('openhands.llm.llm.litellm_completion')
def test_guess_success_content_policy_violation_bad_request(mock_completion, default_config):
"""Test that ContentPolicyViolationError is handled correctly when wrapped in BadRequestError."""
# Simulate BadRequestError with ContentPolicyViolationError message
mock_completion.side_effect = BadRequestError(
'ContentPolicyViolationError: Content policy violation', llm_provider='test_provider', model='test_model'
)

# Initialize LLM and handler
llm = LLM(config=default_config)
handler = ServiceContextPR(
GithubPRHandler('test-owner', 'test-repo', 'test-token'), default_config
)
handler.llm = llm

# Mock issue and history
issue = Issue(
owner='test-owner',
repo='test-repo',
number=1,
title='Test Issue',
body='This is a test issue.',
thread_comments=['Please improve error handling'],
)
history = [MessageAction(content='Fixed error handling.')]

# Call guess_success and expect it to raise BadRequestError
with pytest.raises(BadRequestError):
handler.guess_success(issue, history)

# Assertions
assert (
mock_completion.call_count == default_config.num_retries
) # Initial call + retries
29 changes: 28 additions & 1 deletion tests/unit/test_agent_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from uuid import uuid4

import pytest
from litellm import ContextWindowExceededError
from litellm import ContentPolicyViolationError, ContextWindowExceededError

from openhands.controller.agent import Agent
from openhands.controller.agent_controller import AgentController
Expand Down Expand Up @@ -166,6 +166,33 @@ async def test_react_to_exception(mock_agent, mock_event_stream, mock_status_cal
await controller.close()


@pytest.mark.asyncio
async def test_react_to_content_policy_violation(
mock_agent, mock_event_stream, mock_status_callback
):
controller = AgentController(
agent=mock_agent,
event_stream=mock_event_stream,
status_callback=mock_status_callback,
max_iterations=10,
sid='test',
confirmation_mode=False,
headless_mode=True,
)
error = ContentPolicyViolationError(
message='Output blocked by content filtering policy',
model='gpt-4',
llm_provider='openai',
)
await controller._react_to_exception(error)
mock_status_callback.assert_called_once_with(
'error', 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION', str(error)
)
assert controller.state.last_error == str(error)
assert controller.state.agent_state == AgentState.ERROR
await controller.close()


@pytest.mark.asyncio
async def test_run_controller_with_fatal_error(test_event_stream, mock_memory):
config = AppConfig()
Expand Down