fix: applied suggestions

Author: jmatejcz

src/rai_bench/rai_bench/tool_calling_agent/benchmark.py

@@ -87,7 +87,7 @@ def run_next(self, agent: CompiledStateGraph, experiment_id: uuid.UUID) -> None:
         )
         callbacks = self.score_tracing_handler.get_callbacks()
         run_id = uuid.uuid4()
-        # NOTE (jmatejcz) reccustion limit calculated as all_nodes_num -> one pass though whole node
+        # NOTE (jmatejcz) recursion limit calculated as all_nodes_num -> one pass though whole node
         # plus (task.max_tool_calls_number-1 because the first pass is already added in)
         # times number of nodes - 2 because we dont cout start and end node
         # this can be to much for larger graphs that dont use all nodes on extra calls

src/rai_bench/rai_bench/tool_calling_agent/predefined/basic_tasks.py

@@ -40,7 +40,7 @@
 )
 from rai_bench.tool_calling_agent.validators import (
     NotOrderedCallsValidator,
-    OptionalValidator,
+    OneFromManyValidator,
     OrderedCallsValidator,
 )
 
@@ -409,7 +409,7 @@
 get_pointcloud_ord_val = OrderedCallsValidator(subtasks=[receive_pointcloud_subtask])
 get_robot_desc_ord_val = OrderedCallsValidator(subtasks=[receive_robot_desc_subtask])
 
-set_param_val = OptionalValidator(
+set_param_val = OneFromManyValidator(
     subtasks=[set_robot_state_params_subtask, set_robot_state_params_atomically_subtask]
 )
 services_ord_val = OrderedCallsValidator(subtasks=[get_services_subtask])
@@ -420,24 +420,24 @@
 )
 spawn_entity_val = OrderedCallsValidator(subtasks=[spawn_entity_subtask])
 
-set_grounded_sam_opt_val_1 = OptionalValidator(
+set_grounded_sam_opt_val_1 = OneFromManyValidator(
     subtasks=[set_grounded_sam_subtask_1, set_grounded_sam_atomically_subtask_1]
 )
-set_grounded_dino_opt_val_1 = OptionalValidator(
+set_grounded_dino_opt_val_1 = OneFromManyValidator(
     subtasks=[set_grounded_dino_subtask_1, set_grounding_dino_atomically_subtask_1]
 )
-set_o3de_fps_opt_val_1 = OptionalValidator(
+set_o3de_fps_opt_val_1 = OneFromManyValidator(
     subtasks=[set_o3de_fps_subtask_1, set_o3de_fps_atomically_subtask_1]
 )
 
 
-set_grounded_sam_opt_val_2 = OptionalValidator(
+set_grounded_sam_opt_val_2 = OneFromManyValidator(
     subtasks=[set_grounded_sam_subtask_2, set_grounded_sam_atomically_subtask_2]
 )
-set_grounded_dino_opt_val_2 = OptionalValidator(
+set_grounded_dino_opt_val_2 = OneFromManyValidator(
     subtasks=[set_grounding_dino_subtask_2, set_grounding_dino_atomically_subtask_2]
 )
-set_o3de_fps_opt_val_2 = OptionalValidator(
+set_o3de_fps_opt_val_2 = OneFromManyValidator(
     subtasks=[set_o3de_fps_subtask_2, set_o3de_fps_atomically_subtask_2]
 )
 

src/rai_bench/rai_bench/tool_calling_agent/validators.py

@@ -146,7 +146,7 @@ def validate(self, tool_calls: List[ToolCall]) -> Tuple[bool, List[ToolCall]]:
         return False, []
 
 
-class OptionalValidator(Validator):
+class OneFromManyValidator(Validator):
     """
     Validator that passes when any one of the given subtasks passes.
     """

tests/rai_bench/tool_calling_agent/test_validators.py

@@ -20,7 +20,7 @@
 from rai_bench.tool_calling_agent.interfaces import SubTaskValidationError, Validator
 from rai_bench.tool_calling_agent.validators import (
     NotOrderedCallsValidator,
-    OptionalValidator,
+    OneFromManyValidator,
     OrderedCallsValidator,
 )
 
@@ -680,11 +680,11 @@ def test_validate_reset(self):
 class TestOptionalValidator:
     def test_init_with_empty_subtasks(self):
         with pytest.raises(ValueError, match="Validator must have at least 1 subtask"):
-            OptionalValidator(subtasks=[])
+            OneFromManyValidator(subtasks=[])
 
     def test_validate_empty_tool_calls(self):
         subtasks = [DummySubTask("task1")]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
 
         success, remaining = validator.validate(tool_calls=[])
 
@@ -709,7 +709,7 @@ def test_validate_successful_first_subtask_matches(self):
             DummySubTask("task1", specific_tool="tool1"),
             DummySubTask("task2", specific_tool="tool2"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall(name="tool1")]
 
         success, remaining = validator.validate(tool_calls=tool_calls)
@@ -737,7 +737,7 @@ def test_validate_successful_second_subtask_matches(self):
             DummySubTask("task1", specific_tool="tool1"),
             DummySubTask("task2", specific_tool="tool2"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall(name="tool2")]
 
         success, remaining = validator.validate(tool_calls=tool_calls)
@@ -765,7 +765,7 @@ def test_validate_successful_with_excess_tool_calls(self):
             DummySubTask("task1", specific_tool="tool1"),
             DummySubTask("task2", specific_tool="tool2"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [
             ToolCall(name="tool1"),
             ToolCall(name="extra_tool"),
@@ -799,7 +799,7 @@ def test_validate_successful_after_failed_attempts(self):
             DummySubTask("task1", specific_tool="tool1"),
             DummySubTask("task2", specific_tool="tool2"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [
             ToolCall(name="wrong_tool"),
             ToolCall(name="another_wrong"),
@@ -835,7 +835,7 @@ def test_validate_failure_no_subtask_matches(self):
             DummySubTask("task1", specific_tool="tool1"),
             DummySubTask("task2", specific_tool="tool2"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [
             ToolCall(name="wrong_tool"),
             ToolCall(name="another_wrong"),
@@ -868,7 +868,7 @@ def test_validate_failure_subtask_validation_error(self):
             DummySubTask("task1", outcomes=[False]),
             DummySubTask("task2", outcomes=[False]),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall()]
 
         success, remaining = validator.validate(tool_calls=tool_calls)
@@ -895,7 +895,7 @@ def test_validate_failure_subtask_validation_error(self):
 
     def test_validate_single_subtask_success(self):
         subtasks = [DummySubTask("task1")]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall()]
 
         success, remaining = validator.validate(tool_calls=tool_calls)
@@ -918,7 +918,7 @@ def test_validate_single_subtask_success(self):
 
     def test_validate_single_subtask_failure(self):
         subtasks = [DummySubTask("task1", outcomes=[False])]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall()]
 
         success, remaining = validator.validate(tool_calls=tool_calls)
@@ -947,7 +947,7 @@ def test_validate_many_subtasks_last_one_succeeds(self):
             DummySubTask("task3", specific_tool="tool3"),
             DummySubTask("task4", specific_tool="tool4"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall(name="tool4")]
 
         success, remaining = validator.validate(tool_calls=tool_calls)
@@ -979,7 +979,7 @@ def test_validate_reset(self):
             DummySubTask("task1", outcomes=4 * [False]),
             DummySubTask("task2", outcomes=4 * [False]),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
         tool_calls = [ToolCall(), ToolCall()]
 
         # First validation call
@@ -1013,7 +1013,7 @@ def test_required_calls_property(self):
             DummySubTask("task2"),
             DummySubTask("task3"),
         ]
-        validator = OptionalValidator(subtasks=subtasks)
+        validator = OneFromManyValidator(subtasks=subtasks)
 
         # OptionalValidator should only require 1 call
         assert validator.required_calls == 1