feat: Track the output path for metrics_table in experiments metadata, if output bucket is specified but no file name is specified, we will generate a unique file name

PiperOrigin-RevId: 742523374

Author: vertex-sdk-bot

tests/unit/vertexai/test_evaluation.py

@@ -2086,18 +2086,34 @@ def test_upload_results(self, mock_storage_blob_from_string):
             mock.ANY,
         )
 
-    def test_upload_results_with_default_file_name(self, mock_storage_blob_from_string):
+    def test_upload_results_with_default_output_file_name(
+        self, mock_storage_blob_from_string
+    ):
+        mock_metric_results = _MOCK_EXACT_MATCH_RESULT
         with mock.patch.object(
             aiplatform_utils, "timestamped_unique_name"
         ) as mock_timestamped_unique_name:
-            mock_timestamped_unique_name.return_value = "2025-02-10-12-00-00-12345"
-            evaluation.utils.upload_evaluation_results(
-                MOCK_EVAL_RESULT,
-                _TEST_BUCKET,
-            )
-
+            with mock.patch.object(
+                target=gapic_evaluation_services.EvaluationServiceClient,
+                attribute="evaluate_instances",
+                side_effect=mock_metric_results,
+            ):
+                mock_timestamped_unique_name.return_value = "2025-02-10-12-00-00-12345"
+                eval_dataset = pd.DataFrame(
+                    {
+                        "response": ["test", "text"],
+                        "reference": ["test", "ref"],
+                    }
+                )
+                test_metrics = ["exact_match"]
+                test_eval_task = EvalTask(
+                    dataset=eval_dataset,
+                    metrics=test_metrics,
+                    output_uri_prefix=_TEST_BUCKET,
+                )
+                _ = test_eval_task.evaluate()
         mock_storage_blob_from_string.assert_any_call(
-            uri="gs://test-bucket/eval_results_2025-02-10-12-00-00-12345/eval_results_2025-02-10-12-00-00-12345.csv",
+            uri="gs://test-bucket/eval_results_2025-02-10-12-00-00-12345/summary_metrics.json",
             client=mock.ANY,
         )
 

vertexai/evaluation/eval_task.py

@@ -22,12 +22,13 @@
 from google.api_core import exceptions
 import vertexai
 from google.cloud.aiplatform import base
+from google.cloud.aiplatform import utils
 from google.cloud.aiplatform.metadata import metadata
 from vertexai import generative_models
 from vertexai.evaluation import _base as eval_base
 from vertexai.evaluation import _evaluation
 from vertexai.evaluation import constants
-from vertexai.evaluation import utils
+from vertexai.evaluation import utils as eval_utils
 from vertexai.evaluation.metrics import (
     _base as metrics_base,
 )
@@ -289,10 +290,10 @@ def __init__(
               evaluation results.
         """
         self._raw_dataset = dataset
-        self._dataset = utils.load_dataset(dataset)
+        self._dataset = eval_utils.load_dataset(dataset)
         self._metrics = metrics
         self._experiment = experiment
-        self._metric_column_mapping = utils.initialize_metric_column_mapping(
+        self._metric_column_mapping = eval_utils.initialize_metric_column_mapping(
             metric_column_mapping, self._dataset
         )
         self.output_uri_prefix = output_uri_prefix
@@ -320,6 +321,7 @@ def _evaluate_with_experiment(
         experiment_run_name: Optional[str] = None,
         evaluation_service_qps: Optional[float] = None,
         retry_timeout: float = 120.0,
+        output_file_name: Optional[str] = None,
     ) -> EvalResult:
         """Runs an evaluation for the EvalTask with an experiment.
 
@@ -336,13 +338,19 @@ def _evaluate_with_experiment(
           evaluation_service_qps: The custom QPS limit for the evaluation service.
           retry_timeout: How long to keep retrying the evaluation requests for
             the whole evaluation dataset, in seconds.
+          output_file_name: The file name with csv suffix to store the output
+            metrics_table to be tracked in the experiment run.
 
         Returns:
           The evaluation result.
         """
         self._validate_experiment_run()
         with vertexai.preview.start_run(experiment_run_name):
-            self._log_eval_experiment_param(model, prompt_template)
+            self._log_eval_experiment_param(
+                model=model,
+                prompt_template=prompt_template,
+                output_file_name=output_file_name,
+            )
             eval_result = _evaluation.evaluate(
                 dataset=self._dataset,
                 metrics=self._metrics,
@@ -413,6 +421,8 @@ def evaluate(
                 "`vertexai.init(experiment='experiment_name')`for logging this"
                 " evaluation run."
             )
+        if self.output_uri_prefix and not output_file_name:
+            output_file_name = f"eval_results_{utils.timestamped_unique_name()}.csv"
         self._verify_and_set_response_column_name(
             response_column_name=response_column_name,
             metric_column_mapping_key=constants.Dataset.MODEL_RESPONSE_COLUMN,
@@ -433,6 +443,7 @@ def evaluate(
                 experiment_run_name=experiment_run_name,
                 evaluation_service_qps=evaluation_service_qps,
                 retry_timeout=retry_timeout,
+                output_file_name=output_file_name,
             )
             metadata._experiment_tracker.set_experiment(
                 experiment=global_experiment_name,
@@ -449,6 +460,7 @@ def evaluate(
                 experiment_run_name=experiment_run_name,
                 evaluation_service_qps=evaluation_service_qps,
                 retry_timeout=retry_timeout,
+                output_file_name=output_file_name,
             )
             metadata._experiment_tracker.reset()
         elif not self._experiment and global_experiment_name:
@@ -458,6 +470,7 @@ def evaluate(
                 experiment_run_name=experiment_run_name,
                 evaluation_service_qps=evaluation_service_qps,
                 retry_timeout=retry_timeout,
+                output_file_name=output_file_name,
             )
         else:
             eval_result = _evaluation.evaluate(
@@ -490,7 +503,7 @@ def evaluate(
         if isinstance(self._raw_dataset, str):
             dataset_uri = self._raw_dataset
 
-        utils.upload_evaluation_results(
+        eval_utils.upload_evaluation_results(
             eval_result,
             self.output_uri_prefix,
             output_file_name,
@@ -513,35 +526,41 @@ def _log_eval_experiment_param(
         self,
         model: Optional[Union[GenerativeModel, Callable[[str], str]]] = None,
         prompt_template: Optional[str] = None,
+        output_file_name: Optional[str] = None,
     ) -> None:
         """Logs variable input parameters of an evaluation to an experiment run."""
-        model_metadata = {}
+        eval_metadata = {}
 
         if prompt_template is not None:
-            model_metadata.update({"prompt_template": prompt_template})
+            eval_metadata.update({"prompt_template": prompt_template})
 
         if isinstance(model, GenerativeModel):
-            model_metadata.update(
+            eval_metadata.update(
                 {
                     "model_name": model._model_name,
                 }
             )
 
             if model._generation_config and isinstance(model._generation_config, dict):
-                model_metadata.update(**model._generation_config)
+                eval_metadata.update(**model._generation_config)
 
             if model._safety_settings and isinstance(model._safety_settings, dict):
                 safety_settings = model._safety_settings
                 safety_settings_as_str = {
                     category.name: threshold.name
                     for category, threshold in safety_settings.items()
                 }
-                model_metadata.update(safety_settings_as_str)
+                eval_metadata.update(safety_settings_as_str)
+
+        if self.output_uri_prefix and output_file_name:
+            eval_metadata.update(
+                {"output_file": self.output_uri_prefix + "/" + output_file_name}
+            )
 
-        if model_metadata:
-            _LOGGER.info(f"Logging Eval Experiment metadata: {model_metadata}")
+        if eval_metadata:
+            _LOGGER.info(f"Logging Eval Experiment metadata: {eval_metadata}")
             try:
-                vertexai.preview.log_params(model_metadata)
+                vertexai.preview.log_params(eval_metadata)
             except (ValueError, TypeError) as e:
                 _LOGGER.warning(f"Experiment metadata logging failed: {str(e)}")
 

vertexai/evaluation/utils.py

@@ -35,8 +35,8 @@
     evaluation_service as gapic_evaluation_services,
 )
 from vertexai.evaluation import _base as eval_base
+from vertexai.evaluation.metrics import _base as metrics_base
 from vertexai.evaluation.metrics import (
-    _base as metrics_base,
     metric_prompt_template as metric_prompt_template_base,
 )
 
@@ -359,26 +359,25 @@ def upload_evaluation_results(
     if eval_result.metrics_table is None:
         return
     if destination_uri_prefix.startswith(_GCS_PREFIX):
-        if not file_name:
-            file_name = f"eval_results_{utils.timestamped_unique_name()}.csv"
-        base_name, extension = os.path.splitext(file_name)
-        file_type = extension.lower()[1:]
-        output_folder = destination_uri_prefix + "/" + base_name
-        metrics_table_path = output_folder + "/" + file_name
-        _upload_pandas_df_to_gcs(
-            eval_result.metrics_table, metrics_table_path, file_type
-        )
-        _upload_evaluation_summary_to_gcs(
-            eval_result.summary_metrics,
-            output_folder + "/summary_metrics.json",
-            candidate_model_name,
-            baseline_model_name,
-            dataset_uri,
-            metrics,
-        )
-        _ipython_utils.display_gen_ai_evaluation_results_button(
-            metrics_table_path.split(_GCS_PREFIX)[1]
-        )
+        if file_name:
+            base_name, extension = os.path.splitext(file_name)
+            file_type = extension.lower()[1:]
+            output_folder = destination_uri_prefix + "/" + base_name
+            metrics_table_path = output_folder + "/" + file_name
+            _upload_pandas_df_to_gcs(
+                eval_result.metrics_table, metrics_table_path, file_type
+            )
+            _upload_evaluation_summary_to_gcs(
+                eval_result.summary_metrics,
+                output_folder + "/summary_metrics.json",
+                candidate_model_name,
+                baseline_model_name,
+                dataset_uri,
+                metrics,
+            )
+            _ipython_utils.display_gen_ai_evaluation_results_button(
+                metrics_table_path.split(_GCS_PREFIX)[1]
+            )
     else:
         raise ValueError(
             f"Unsupported destination URI: {destination_uri_prefix}."

vertexai/preview/evaluation/eval_task.py

@@ -24,6 +24,7 @@
 from google.api_core import exceptions
 import vertexai
 from google.cloud.aiplatform import base
+from google.cloud.aiplatform import utils
 from google.cloud.aiplatform.metadata import metadata
 from vertexai import generative_models
 from vertexai.preview import reasoning_engines
@@ -336,6 +337,7 @@ def _evaluate_with_experiment(
         experiment_run_name: Optional[str] = None,
         evaluation_service_qps: Optional[float] = None,
         retry_timeout: float = 120.0,
+        output_file_name: Optional[str] = None,
     ) -> EvalResult:
         """Runs an evaluation for the EvalTask with an experiment.
 
@@ -355,14 +357,19 @@ def _evaluate_with_experiment(
           evaluation_service_qps: The custom QPS limit for the evaluation service.
           retry_timeout: How long to keep retrying the evaluation requests for
             the whole evaluation dataset, in seconds.
+          output_path: The file name with csv suffix to store the output
+            metrics_table to be tracked in the experiment run.
 
         Returns:
           The evaluation result.
         """
         self._validate_experiment_run()
         with vertexai.preview.start_run(experiment_run_name):
             self._log_eval_experiment_param(
-                model=model, runnable=runnable, prompt_template=prompt_template
+                model=model,
+                runnable=runnable,
+                prompt_template=prompt_template,
+                output_file_name=output_file_name,
             )
             eval_result = _evaluation.evaluate(
                 dataset=self._dataset,
@@ -451,7 +458,8 @@ def evaluate(
             response_column_name=baseline_model_response_column_name,
             metric_column_mapping_key=constants.Dataset.BASELINE_MODEL_RESPONSE_COLUMN,
         )
-
+        if self.output_uri_prefix and not output_file_name:
+            output_file_name = f"eval_results_{utils.timestamped_unique_name()}.csv"
         experiment_run_name = experiment_run_name or f"{uuid.uuid4()}"
         if self._experiment and global_experiment_name:
             metadata._experiment_tracker.set_experiment(  # pylint: disable=protected-access
@@ -464,6 +472,7 @@ def evaluate(
                 experiment_run_name=experiment_run_name,
                 evaluation_service_qps=evaluation_service_qps,
                 retry_timeout=retry_timeout,
+                output_file_name=output_file_name,
             )
             metadata._experiment_tracker.set_experiment(  # pylint: disable=protected-access
                 experiment=global_experiment_name, backing_tensorboard=False
@@ -479,6 +488,7 @@ def evaluate(
                 experiment_run_name=experiment_run_name,
                 evaluation_service_qps=evaluation_service_qps,
                 retry_timeout=retry_timeout,
+                output_file_name=output_file_name,
             )
             metadata._experiment_tracker.reset()  # pylint: disable=protected-access
         elif not self._experiment and global_experiment_name:
@@ -489,6 +499,7 @@ def evaluate(
                 experiment_run_name=experiment_run_name,
                 evaluation_service_qps=evaluation_service_qps,
                 retry_timeout=retry_timeout,
+                output_file_name=output_file_name,
             )
         else:
             eval_result = _evaluation.evaluate(
@@ -503,7 +514,7 @@ def evaluate(
                 autorater_config=self._autorater_config,
             )
         eval_utils.upload_evaluation_results(
-            eval_result.metrics_table, self.output_uri_prefix, output_file_name
+            eval_result, self.output_uri_prefix, output_file_name
         )
         return eval_result
 
@@ -522,6 +533,7 @@ def _log_eval_experiment_param(
         model: _ModelType = None,
         runnable: _RunnableType = None,
         prompt_template: Optional[str] = None,
+        output_file_name: Optional[str] = None,
     ) -> None:
         """Logs variable input parameters of an evaluation to an experiment run."""
         eval_metadata = {}
@@ -568,6 +580,11 @@ def _log_eval_experiment_param(
                     }  # pylint: disable=protected-access
                 )
 
+        if self.output_uri_prefix and output_file_name:
+            eval_metadata.update(
+                {"output_file": self.output_uri_prefix + "/" + output_file_name}
+            )
+
         if eval_metadata:
             _LOGGER.info(
                 f"Logging Eval experiment evaluation metadata: {eval_metadata}"

vertexai/preview/evaluation/utils.py

@@ -368,26 +368,25 @@ def upload_evaluation_results(
     if eval_result.metrics_table is None:
         return
     if destination_uri_prefix.startswith(_GCS_PREFIX):
-        if not file_name:
-            file_name = f"eval_results_{utils.timestamped_unique_name()}.csv"
-        base_name, extension = os.path.splitext(file_name)
-        file_type = extension.lower()[1:]
-        output_folder = destination_uri_prefix + "/" + base_name
-        metrics_table_path = output_folder + "/" + file_name
-        _upload_pandas_df_to_gcs(
-            eval_result.metrics_table, metrics_table_path, file_type
-        )
-        _upload_evaluation_summary_to_gcs(
-            eval_result.summary_metrics,
-            output_folder + "/summary_metrics.json",
-            candidate_model_name,
-            baseline_model_name,
-            dataset_uri,
-            metrics,
-        )
-        _ipython_utils.display_gen_ai_evaluation_results_button(
-            metrics_table_path.split(_GCS_PREFIX)[1]
-        )
+        if file_name:
+            base_name, extension = os.path.splitext(file_name)
+            file_type = extension.lower()[1:]
+            output_folder = destination_uri_prefix + "/" + base_name
+            metrics_table_path = output_folder + "/" + file_name
+            _upload_pandas_df_to_gcs(
+                eval_result.metrics_table, metrics_table_path, file_type
+            )
+            _upload_evaluation_summary_to_gcs(
+                eval_result.summary_metrics,
+                output_folder + "/summary_metrics.json",
+                candidate_model_name,
+                baseline_model_name,
+                dataset_uri,
+                metrics,
+            )
+            _ipython_utils.display_gen_ai_evaluation_results_button(
+                metrics_table_path.split(_GCS_PREFIX)[1]
+            )
     else:
         raise ValueError(
             f"Unsupported destination URI: {destination_uri_prefix}."