diff --git a/src/rai_bench/pyproject.toml b/src/rai_bench/pyproject.toml index 33d8f1365..f892b88c8 100644 --- a/src/rai_bench/pyproject.toml +++ b/src/rai_bench/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "rai-bench" -version = "0.1.0" +version = "0.1.1" description = "Package for running and creating benchmarks." authors = ["Jakub Matejczyk ", "Magdalena Kotynia "] readme = "README.md" diff --git a/src/rai_bench/rai_bench/manipulation_o3de/predefined/scenarios.py b/src/rai_bench/rai_bench/manipulation_o3de/predefined/scenarios.py index a00ca9a3f..67ae68244 100644 --- a/src/rai_bench/rai_bench/manipulation_o3de/predefined/scenarios.py +++ b/src/rai_bench/rai_bench/manipulation_o3de/predefined/scenarios.py @@ -72,10 +72,12 @@ def trivial_scenarios(logger: logging.Logger | None) -> List[Scenario]: place_object_tasks.append( PlaceObjectAtCoordTask(obj, coord, disp, logger=logger) ) - easy_place_objects_scenarios = ManipulationO3DEBenchmark.create_scenarios( + place_objects_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=place_object_tasks, scene_configs=scene_configs, scene_configs_paths=scene_configs_paths, + level="trivial", + logger=logger, ) # move objects to the left object_groups = [["carrot"], ["red_cube"], ["tomato"], ["yellow_cube"]] @@ -85,14 +87,15 @@ def trivial_scenarios(logger: logging.Logger | None) -> List[Scenario]: for objects in object_groups ] - easy_move_to_left_scenarios = ManipulationO3DEBenchmark.create_scenarios( + move_to_left_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=move_to_left_tasks, scene_configs=scene_configs, scene_configs_paths=scene_configs_paths, level="trivial", + logger=logger, ) - return [*easy_move_to_left_scenarios, *easy_place_objects_scenarios] + return [*move_to_left_scenarios, *place_objects_scenarios] def easy_scenarios(logger: logging.Logger | None) -> List[Scenario]: @@ -144,10 +147,11 @@ def easy_scenarios(logger: logging.Logger | None) -> List[Scenario]: place_object_tasks.append( PlaceObjectAtCoordTask(obj, coord, disp, logger=logger) ) - easy_place_objects_scenarios = ManipulationO3DEBenchmark.create_scenarios( + place_objects_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=place_object_tasks, scene_configs=scene_configs, scene_configs_paths=scene_configs_paths, + level="easy", logger=logger, ) # move objects to the left @@ -164,25 +168,28 @@ def easy_scenarios(logger: logging.Logger | None) -> List[Scenario]: for objects in object_groups ] - easy_move_to_left_scenarios = ManipulationO3DEBenchmark.create_scenarios( + move_to_left_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=move_to_left_tasks, scene_configs=scene_configs, scene_configs_paths=scene_configs_paths, + level="easy", + logger=logger, ) # place cubes task = PlaceCubesTask(threshold_distance=0.2, logger=logger) - easy_place_cubes_scenarios = ManipulationO3DEBenchmark.create_scenarios( + place_cubes_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=[task], scene_configs=scene_configs, scene_configs_paths=scene_configs_paths, level="easy", + logger=logger, ) return [ - *easy_move_to_left_scenarios, - *easy_place_objects_scenarios, - *easy_place_cubes_scenarios, + *move_to_left_scenarios, + *place_objects_scenarios, + *place_cubes_scenarios, ] @@ -260,7 +267,7 @@ def medium_scenarios(logger: logging.Logger | None) -> List[Scenario]: # place cubes task = PlaceCubesTask(threshold_distance=0.1, logger=logger) - easy_place_cubes_scenarios = ManipulationO3DEBenchmark.create_scenarios( + place_cubes_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=[task], scene_configs=medium_scene_configs, scene_configs_paths=medium_scene_configs_paths, @@ -282,6 +289,8 @@ def medium_scenarios(logger: logging.Logger | None) -> List[Scenario]: tasks=build_tower_tasks, scene_configs=easy_scene_configs, scene_configs_paths=easy_scene_configs_paths, + level="medium", + logger=logger, ) # group object task @@ -302,11 +311,13 @@ def medium_scenarios(logger: logging.Logger | None) -> List[Scenario]: tasks=group_object_tasks, scene_configs=easy_scene_configs, scene_configs_paths=easy_scene_configs_paths, + level="medium", + logger=logger, ) return [ *move_to_left_scenarios, *build_tower_scenarios, - *easy_place_cubes_scenarios, + *place_cubes_scenarios, *group_object_scenarios, ] @@ -381,15 +392,17 @@ def hard_scenarios(logger: logging.Logger | None) -> List[Scenario]: scene_configs=hard_scene_configs, scene_configs_paths=hard_scene_configs_paths, level="hard", + logger=logger, ) # place cubes task = PlaceCubesTask(threshold_distance=0.1, logger=logger) - easy_place_cubes_scenarios = ManipulationO3DEBenchmark.create_scenarios( + place_cubes_scenarios = ManipulationO3DEBenchmark.create_scenarios( tasks=[task], scene_configs=hard_scene_configs, scene_configs_paths=hard_scene_configs_paths, level="hard", + logger=logger, ) # build tower task @@ -407,6 +420,7 @@ def hard_scenarios(logger: logging.Logger | None) -> List[Scenario]: scene_configs=medium_scene_configs, scene_configs_paths=medium_scene_configs_paths, level="hard", + logger=logger, ) # group object task @@ -428,11 +442,13 @@ def hard_scenarios(logger: logging.Logger | None) -> List[Scenario]: tasks=group_object_tasks, scene_configs=medium_scene_configs, scene_configs_paths=medium_scene_configs_paths, + level="hard", + logger=logger, ) return [ *move_to_left_scenarios, *build_tower_scenarios, - *easy_place_cubes_scenarios, + *place_cubes_scenarios, *group_object_scenarios, ] @@ -512,6 +528,7 @@ def very_hard_scenarios(logger: logging.Logger | None) -> List[Scenario]: scene_configs=hard_scene_configs, scene_configs_paths=hard_scene_configs_paths, level="very_hard", + logger=logger, ) return [ *build_tower_scenarios, diff --git a/src/rai_bench/rai_bench/results_processing/data_loading.py b/src/rai_bench/rai_bench/results_processing/data_loading.py index fee969867..429d6fc2d 100644 --- a/src/rai_bench/rai_bench/results_processing/data_loading.py +++ b/src/rai_bench/rai_bench/results_processing/data_loading.py @@ -105,6 +105,7 @@ def convert_row_to_scenario_result(row: pd.Series) -> ScenarioResult: model_name=row["model_name"], scene_config_path=row["scene_config_path"], score=float(row["score"]), + level=row["level"], total_time=float(row["total_time"]), number_of_tool_calls=int(row["number_of_tool_calls"]), )