feat: Publish GenAI Evaluation SDK GA features to vertexai.evaluation module. Switch GenAI Evaluation Service client to v1 version.

PiperOrigin-RevId: 665153547

Author: jsondai

tests/unit/vertexai/test_evaluation.py

@@ -24,23 +24,23 @@
 import vertexai
 from google.cloud.aiplatform import initializer
 from google.cloud.aiplatform.metadata import metadata
-from google.cloud.aiplatform_v1beta1.services import (
+from google.cloud.aiplatform_v1.services import (
     evaluation_service as gapic_evaluation_services,
 )
-from google.cloud.aiplatform_v1beta1.types import (
+from google.cloud.aiplatform_v1.types import (
     evaluation_service as gapic_evaluation_service_types,
 )
+from vertexai import evaluation
 from vertexai import generative_models
-from vertexai.preview import evaluation
-from vertexai.preview.evaluation import _base as eval_base
-from vertexai.preview.evaluation import _evaluation
-from vertexai.preview.evaluation import utils
-from vertexai.preview.evaluation.metrics import _rouge
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation import _base as eval_base
+from vertexai.evaluation import _evaluation
+from vertexai.evaluation import utils
+from vertexai.evaluation.metrics import _rouge
+from vertexai.evaluation.metrics import (
     metric_prompt_template_examples,
 )
-from vertexai.preview.evaluation.metrics import pairwise_metric
-from vertexai.preview.evaluation.metrics import pointwise_metric
+from vertexai.evaluation.metrics import pairwise_metric
+from vertexai.evaluation.metrics import pointwise_metric
 import numpy as np
 import pandas as pd
 import pytest

vertexai/evaluation/__init__.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Vertex Gen AI Evaluation Service Module."""
+
+from vertexai.evaluation import _base
+from vertexai.evaluation import eval_task
+from vertexai.evaluation import metrics
+from vertexai.evaluation import prompt_template
+
+
+EvalResult = _base.EvalResult
+EvalTask = eval_task.EvalTask
+PairwiseMetric = metrics.PairwiseMetric
+PointwiseMetric = metrics.PointwiseMetric
+CustomMetric = metrics.CustomMetric
+PromptTemplate = prompt_template.PromptTemplate
+PairwiseMetricPromptTemplate = metrics.PairwiseMetricPromptTemplate
+PointwiseMetricPromptTemplate = metrics.PointwiseMetricPromptTemplate
+MetricPromptTemplateExamples = metrics.MetricPromptTemplateExamples
+
+__all__ = [
+    "EvalTask",
+    "EvalResult",
+    "PairwiseMetric",
+    "PointwiseMetric",
+    "CustomMetric",
+    "PromptTemplate",
+    "PairwiseMetricPromptTemplate",
+    "PointwiseMetricPromptTemplate",
+    "MetricPromptTemplateExamples",
+]

vertexai/preview/evaluation/_base.py vertexai/evaluation/_base.py

@@ -20,10 +20,10 @@
 import dataclasses
 from typing import Dict, List, Optional, Union, TYPE_CHECKING
 
-from google.cloud.aiplatform_v1beta1.services import (
+from google.cloud.aiplatform_v1.services import (
     evaluation_service as gapic_evaluation_services,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     _base as metrics_base,
 )
 

vertexai/preview/evaluation/_evaluation.py vertexai/evaluation/_evaluation.py

@@ -28,23 +28,23 @@
     content as gapic_content_types,
 )
 from vertexai import generative_models
-from vertexai.preview.evaluation import _base as evaluation_base
-from vertexai.preview.evaluation import constants
-from vertexai.preview.evaluation import (
+from vertexai.evaluation import _base as evaluation_base
+from vertexai.evaluation import constants
+from vertexai.evaluation import (
     prompt_template as prompt_template_base,
 )
-from vertexai.preview.evaluation import utils
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation import utils
+from vertexai.evaluation.metrics import (
     _base as metrics_base,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     _instance_evaluation,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     metric_prompt_template_examples,
 )
-from vertexai.preview.evaluation.metrics import pairwise_metric
-from vertexai.preview.evaluation.metrics import pointwise_metric
+from vertexai.evaluation.metrics import pairwise_metric
+from vertexai.evaluation.metrics import pointwise_metric
 
 
 try:

vertexai/preview/evaluation/constants.py vertexai/evaluation/constants.py

@@ -22,17 +22,17 @@
 from google.cloud.aiplatform import base
 from google.cloud.aiplatform.metadata import metadata
 from vertexai import generative_models
-from vertexai.preview.evaluation import _base as eval_base
-from vertexai.preview.evaluation import _evaluation
-from vertexai.preview.evaluation import constants
-from vertexai.preview.evaluation import utils
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation import _base as eval_base
+from vertexai.evaluation import _evaluation
+from vertexai.evaluation import constants
+from vertexai.evaluation import utils
+from vertexai.evaluation.metrics import (
     _base as metrics_base,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     pairwise_metric,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     pointwise_metric,
 )
 import numpy as np

vertexai/preview/evaluation/eval_task.py vertexai/evaluation/eval_task.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Evaluation Metrics Module."""
+
+from vertexai.evaluation.metrics import _base
+from vertexai.evaluation.metrics import _rouge
+from vertexai.evaluation.metrics import (
+    metric_prompt_template,
+)
+from vertexai.evaluation.metrics import (
+    metric_prompt_template_examples,
+)
+from vertexai.evaluation.metrics import pairwise_metric
+from vertexai.evaluation.metrics import pointwise_metric
+
+
+PairwiseMetric = pairwise_metric.PairwiseMetric
+PointwiseMetric = pointwise_metric.PointwiseMetric
+CustomMetric = _base.CustomMetric
+PairwiseMetricPromptTemplate = metric_prompt_template.PairwiseMetricPromptTemplate
+PointwiseMetricPromptTemplate = metric_prompt_template.PointwiseMetricPromptTemplate
+MetricPromptTemplateExamples = (
+    metric_prompt_template_examples.MetricPromptTemplateExamples
+)
+Rouge = _rouge.Rouge
+
+
+__all__ = [
+    "CustomMetric",
+    "PairwiseMetric",
+    "PointwiseMetric",
+    "PairwiseMetricPromptTemplate",
+    "PointwiseMetricPromptTemplate",
+    "MetricPromptTemplateExamples",
+    "Rouge",
+]

vertexai/evaluation/metrics/__init__.py

@@ -19,8 +19,8 @@
 import abc
 from typing import Any, Callable, Dict, Literal, Union
 
-from vertexai.preview.evaluation import constants
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation import constants
+from vertexai.evaluation.metrics import (
     metric_prompt_template as metric_prompt_template_base,
 )
 

vertexai/preview/evaluation/metrics/_base.py vertexai/evaluation/metrics/_base.py

@@ -22,24 +22,24 @@
 from google import api_core
 from google.cloud.aiplatform import base
 from google.cloud.aiplatform import initializer
-from google.cloud.aiplatform_v1beta1.services import (
+from google.cloud.aiplatform_v1.services import (
     evaluation_service as gapic_evaluation_services,
 )
-from google.cloud.aiplatform_v1beta1.types import (
+from google.cloud.aiplatform_v1.types import (
     evaluation_service as gapic_eval_service_types,
 )
-from vertexai.preview.evaluation import _base as eval_base
-from vertexai.preview.evaluation import constants
-from vertexai.preview.evaluation import utils
-from vertexai.preview.evaluation import (
+from vertexai.evaluation import _base as eval_base
+from vertexai.evaluation import constants
+from vertexai.evaluation import utils
+from vertexai.evaluation import (
     prompt_template as prompt_template_base,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     _base as metrics_base,
 )
-from vertexai.preview.evaluation.metrics import _rouge
-from vertexai.preview.evaluation.metrics import pairwise_metric
-from vertexai.preview.evaluation.metrics import pointwise_metric
+from vertexai.evaluation.metrics import _rouge
+from vertexai.evaluation.metrics import pairwise_metric
+from vertexai.evaluation.metrics import pointwise_metric
 from google.protobuf import json_format
 
 

vertexai/preview/evaluation/metrics/_default_templates.py vertexai/evaluation/metrics/_default_templates.py

@@ -15,8 +15,8 @@
 # limitations under the License.
 #
 from typing import Literal
-from vertexai.preview.evaluation import constants
-from vertexai.preview.evaluation.metrics import _base
+from vertexai.evaluation import constants
+from vertexai.evaluation.metrics import _base
 
 
 class Rouge(_base._AutomaticMetric):  # pylint: disable=protected-access

vertexai/preview/evaluation/metrics/_instance_evaluation.py vertexai/evaluation/metrics/_instance_evaluation.py

@@ -19,7 +19,7 @@
 from typing import Dict, List, Optional
 
 from google.cloud.aiplatform import base
-from vertexai.preview.evaluation import (
+from vertexai.evaluation import (
     prompt_template,
 )
 

vertexai/preview/evaluation/metrics/_rouge.py vertexai/evaluation/metrics/_rouge.py

@@ -18,12 +18,12 @@
 
 from typing import List
 
-from vertexai.preview.evaluation import constants
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation import constants
+from vertexai.evaluation.metrics import (
     _default_templates,
 )
-from vertexai.preview.evaluation.metrics import pairwise_metric
-from vertexai.preview.evaluation.metrics import pointwise_metric
+from vertexai.evaluation.metrics import pairwise_metric
+from vertexai.evaluation.metrics import pointwise_metric
 import immutabledict
 
 

vertexai/preview/evaluation/metrics/metric_prompt_template.py vertexai/evaluation/metrics/metric_prompt_template.py

@@ -19,8 +19,8 @@
 from typing import Callable, Optional, Union
 
 from vertexai import generative_models
-from vertexai.preview.evaluation.metrics import _base
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import _base
+from vertexai.evaluation.metrics import (
     metric_prompt_template as metric_prompt_template_base,
 )
 

vertexai/preview/evaluation/metrics/metric_prompt_template_examples.py vertexai/evaluation/metrics/metric_prompt_template_examples.py

@@ -18,8 +18,8 @@
 
 from typing import Union
 
-from vertexai.preview.evaluation.metrics import _base
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import _base
+from vertexai.evaluation.metrics import (
     metric_prompt_template as metric_prompt_template_base,
 )
 

vertexai/preview/evaluation/metrics/pairwise_metric.py vertexai/evaluation/metrics/pairwise_metric.py

@@ -28,10 +28,10 @@
 from google.cloud.aiplatform import compat
 from google.cloud.aiplatform import initializer
 from google.cloud.aiplatform import utils
-from google.cloud.aiplatform_v1beta1.services import (
+from google.cloud.aiplatform_v1.services import (
     evaluation_service as gapic_evaluation_services,
 )
-from vertexai.preview.evaluation import constants
+from vertexai.evaluation import constants
 
 
 if TYPE_CHECKING:
@@ -44,10 +44,10 @@
 
 class _EvaluationServiceClientWithOverride(utils.ClientWithOverride):
     _is_temporary = False
-    _default_version = compat.V1BETA1
+    _default_version = compat.V1
     _version_map = (
         (
-            compat.V1BETA1,
+            compat.V1,
             gapic_evaluation_services.EvaluationServiceClient,
         ),
     )

vertexai/preview/evaluation/metrics/pointwise_metric.py vertexai/evaluation/metrics/pointwise_metric.py

@@ -16,10 +16,10 @@
 #
 """Vertex Gen AI Evaluation Service Module."""
 
-from vertexai.preview.evaluation import _base
-from vertexai.preview.evaluation import eval_task
-from vertexai.preview.evaluation import metrics
-from vertexai.preview.evaluation import prompt_template
+from vertexai.evaluation import _base
+from vertexai.evaluation import eval_task
+from vertexai.evaluation import metrics
+from vertexai.evaluation import prompt_template
 
 
 EvalResult = _base.EvalResult

vertexai/preview/evaluation/prompt_template.py vertexai/evaluation/prompt_template.py

@@ -16,16 +16,16 @@
 #
 """Evaluation Metrics Module."""
 
-from vertexai.preview.evaluation.metrics import _base
-from vertexai.preview.evaluation.metrics import _rouge
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import _base
+from vertexai.evaluation.metrics import _rouge
+from vertexai.evaluation.metrics import (
     metric_prompt_template,
 )
-from vertexai.preview.evaluation.metrics import (
+from vertexai.evaluation.metrics import (
     metric_prompt_template_examples,
 )
-from vertexai.preview.evaluation.metrics import pairwise_metric
-from vertexai.preview.evaluation.metrics import pointwise_metric
+from vertexai.evaluation.metrics import pairwise_metric
+from vertexai.evaluation.metrics import pointwise_metric
 
 
 PairwiseMetric = pairwise_metric.PairwiseMetric