feat(vertexai): Add responseModality (#17326)

* Add responseModality

* review comments

Author: cynthiajoan

packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart

@@ -41,7 +41,7 @@ void main() async {
 
   var vertexInstance =
       FirebaseVertexAI.instanceFor(auth: FirebaseAuth.instance);
-  final model = vertexInstance.generativeModel(model: 'gemini-1.5-flash');
+  final model = vertexInstance.generativeModel(model: 'gemini-2.0-flash');
 
   runApp(GenerativeAISample(model: model));
 }

packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart

@@ -95,6 +95,17 @@ class _ChatPageState extends State<ChatPage> {
                   const SizedBox.square(
                     dimension: 15,
                   ),
+                  if (!_loading)
+                    IconButton(
+                      onPressed: () async {
+                        await _imageResponse(_textController.text);
+                      },
+                      icon: Icon(
+                        Icons.image,
+                        color: Theme.of(context).colorScheme.primary,
+                      ),
+                      tooltip: 'Image response',
+                    ),
                   if (!_loading)
                     IconButton(
                       onPressed: () async {
@@ -152,6 +163,58 @@ class _ChatPageState extends State<ChatPage> {
     }
   }
 
+  Future<void> _imageResponse(String message) async {
+    setState(() {
+      _loading = true;
+    });
+
+    try {
+      _messages.add(MessageData(text: message, fromUser: true));
+      var response = await widget.model.generateContent(
+        [Content.text(message)],
+        generationConfig: GenerationConfig(
+          responseModalities: [
+            ResponseModalities.text,
+            ResponseModalities.image,
+          ],
+        ),
+      );
+      var inlineDatas = response.inlineDataParts.toList();
+
+      if (inlineDatas.isEmpty) {
+        _showError('No response from API.');
+        return;
+      } else {
+        for (final inlineData in inlineDatas) {
+          if (inlineData.mimeType.contains('image')) {
+            _messages.add(
+              MessageData(
+                text: response.text,
+                image: Image.memory(inlineData.bytes),
+                fromUser: false,
+              ),
+            );
+          }
+        }
+        setState(() {
+          _loading = false;
+          _scrollDown();
+        });
+      }
+    } catch (e) {
+      _showError(e.toString());
+      setState(() {
+        _loading = false;
+      });
+    } finally {
+      _textController.clear();
+      setState(() {
+        _loading = false;
+      });
+      _textFieldFocus.requestFocus();
+    }
+  }
+
   void _showError(String message) {
     showDialog<void>(
       context: context,

packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart

@@ -27,6 +27,7 @@ export 'src/api.dart'
         HarmProbability,
         HarmBlockMethod,
         PromptFeedback,
+        ResponseModalities,
         SafetyRating,
         SafetySetting,
         // TODO(cynthiajiang) remove in next breaking change.
@@ -72,7 +73,6 @@ export 'src/live_api.dart'
     show
         LiveGenerationConfig,
         SpeechConfig,
-        ResponseModalities,
         LiveServerMessage,
         LiveServerContent,
         LiveServerToolCall,

packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart

@@ -108,6 +108,15 @@ final class GenerateContentResponse {
   Iterable<FunctionCall> get functionCalls =>
       candidates.firstOrNull?.content.parts.whereType<FunctionCall>() ??
       const [];
+
+  /// The inline data parts of the first candidate in [candidates], if any.
+  ///
+  /// Returns an empty list if there are no candidates, or if the first
+  /// candidate has no [InlineDataPart] parts. There is no error thrown if the
+  /// prompt or response were blocked.
+  Iterable<InlineDataPart> get inlineDataParts =>
+      candidates.firstOrNull?.content.parts.whereType<InlineDataPart>() ??
+      const [];
 }
 
 /// Feedback metadata of a prompt specified in a [GenerativeModel] request.
@@ -656,6 +665,24 @@ enum HarmBlockMethod {
   Object toJson() => _jsonString;
 }
 
+/// The available response modalities.
+enum ResponseModalities {
+  /// Text response modality.
+  text('TEXT'),
+
+  /// Image response modality.
+  image('IMAGE'),
+
+  /// Audio response modality.
+  audio('AUDIO');
+
+  const ResponseModalities(this._jsonString);
+  final String _jsonString;
+
+  /// Convert to json format
+  String toJson() => _jsonString;
+}
+
 /// Configuration options for model generation and outputs.
 abstract class BaseGenerationConfig {
   // ignore: public_member_api_docs
@@ -667,6 +694,7 @@ abstract class BaseGenerationConfig {
     this.topK,
     this.presencePenalty,
     this.frequencyPenalty,
+    this.responseModalities,
   });
 
   /// Number of generated responses to return.
@@ -743,6 +771,9 @@ abstract class BaseGenerationConfig {
   /// for more details.
   final double? frequencyPenalty;
 
+  /// The list of desired response modalities.
+  final List<ResponseModalities>? responseModalities;
+
   // ignore: public_member_api_docs
   Map<String, Object?> toJson() => {
         if (candidateCount case final candidateCount?)
@@ -756,6 +787,9 @@ abstract class BaseGenerationConfig {
           'presencePenalty': presencePenalty,
         if (frequencyPenalty case final frequencyPenalty?)
           'frequencyPenalty': frequencyPenalty,
+        if (responseModalities case final responseModalities?)
+          'responseModalities':
+              responseModalities.map((modality) => modality.toJson()).toList(),
       };
 }
 
@@ -771,6 +805,7 @@ final class GenerationConfig extends BaseGenerationConfig {
     super.topK,
     super.presencePenalty,
     super.frequencyPenalty,
+    super.responseModalities,
     this.responseMimeType,
     this.responseSchema,
   });
@@ -996,6 +1031,9 @@ SafetyRating _parseSafetyRating(Object? jsonObject) {
   if (jsonObject is! Map) {
     throw unhandledFormat('SafetyRating', jsonObject);
   }
+  if (jsonObject.isEmpty) {
+    return SafetyRating(HarmCategory.unknown, HarmProbability.unknown);
+  }
   return SafetyRating(HarmCategory._parseValue(jsonObject['category']),
       HarmProbability._parseValue(jsonObject['probability']),
       probabilityScore: jsonObject['probabilityScore'] as double?,

packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart

@@ -71,30 +71,12 @@ class SpeechConfig {
       };
 }
 
-/// The available response modalities.
-enum ResponseModalities {
-  /// Text response modality.
-  text('TEXT'),
-
-  /// Image response modality.
-  image('IMAGE'),
-
-  /// Audio response modality.
-  audio('AUDIO');
-
-  const ResponseModalities(this._jsonString);
-  final String _jsonString;
-
-  /// Convert to json format
-  String toJson() => _jsonString;
-}
-
 /// Configures live generation settings.
 final class LiveGenerationConfig extends BaseGenerationConfig {
   // ignore: public_member_api_docs
   LiveGenerationConfig({
     this.speechConfig,
-    this.responseModalities,
+    super.responseModalities,
     super.candidateCount,
     super.maxOutputTokens,
     super.temperature,
@@ -107,17 +89,11 @@ final class LiveGenerationConfig extends BaseGenerationConfig {
   /// The speech configuration.
   final SpeechConfig? speechConfig;
 
-  /// The list of desired response modalities.
-  final List<ResponseModalities>? responseModalities;
-
   @override
   Map<String, Object?> toJson() => {
         ...super.toJson(),
         if (speechConfig case final speechConfig?)
           'speechConfig': speechConfig.toJson(),
-        if (responseModalities case final responseModalities?)
-          'responseModalities':
-              responseModalities.map((modality) => modality.toJson()).toList(),
       };
 }
 

packages/firebase_vertexai/firebase_vertexai/test/live_test.dart

@@ -13,6 +13,7 @@
 // limitations under the License.
 import 'dart:typed_data';
 
+import 'package:firebase_vertexai/src/api.dart';
 import 'package:firebase_vertexai/src/content.dart';
 import 'package:firebase_vertexai/src/error.dart';
 import 'package:firebase_vertexai/src/live_api.dart';