Merge branch 'main' into ep/bidi-compile-tests

Author: emilypgoogle

firebase-ai/CHANGELOG.md

@@ -13,3 +13,11 @@
 * [fixed] Fixed an issue with `LiveContentResponse` audio data not being present when the model was
   interrupted or the turn completed. (#6870)
 * [fixed] Fixed an issue with `LiveSession` not converting exceptions to `FirebaseVertexAIException`. (#6870)
+* * [changed] **Breaking Change**: Removed the `LiveContentResponse.Status` class, and instead have nested the status
+  fields as properties of `LiveContentResponse`. (#6906)
+* [changed] **Breaking Change**: Removed the `LiveContentResponse` class, and instead have provided subclasses
+  of `LiveServerMessage` that match the responses from the model. (#6910)
+* [feature] Added support for the `id` field on `FunctionResponsePart` and `FunctionCallPart`. (#6910)
+* [feature] Add support for specifying response modalities in `GenerationConfig`. (#6921)
+* [feature] Added a helper field for getting all the `InlineDataPart` from a `GenerateContentResponse`. (#6922)
+

firebase-ai/api.txt

@@ -127,7 +127,7 @@ package com.google.firebase.ai.java {
   @com.google.firebase.ai.type.PublicPreviewAPI public abstract class LiveSessionFutures {
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> close();
     method public static final com.google.firebase.ai.java.LiveSessionFutures from(com.google.firebase.ai.type.LiveSession session);
-    method public abstract org.reactivestreams.Publisher<com.google.firebase.ai.type.LiveContentResponse> receive();
+    method public abstract org.reactivestreams.Publisher<com.google.firebase.ai.type.LiveServerMessage> receive();
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> send(com.google.firebase.ai.type.Content content);
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> send(String text);
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendFunctionResponse(java.util.List<com.google.firebase.ai.type.FunctionResponsePart> functionList);
@@ -293,9 +293,12 @@ package com.google.firebase.ai.type {
 
   public final class FunctionCallPart implements com.google.firebase.ai.type.Part {
     ctor public FunctionCallPart(String name, java.util.Map<java.lang.String,? extends kotlinx.serialization.json.JsonElement> args);
+    ctor public FunctionCallPart(String name, java.util.Map<java.lang.String,? extends kotlinx.serialization.json.JsonElement> args, String? id = null);
     method public java.util.Map<java.lang.String,kotlinx.serialization.json.JsonElement> getArgs();
+    method public String? getId();
     method public String getName();
     property public final java.util.Map<java.lang.String,kotlinx.serialization.json.JsonElement> args;
+    property public final String? id;
     property public final String name;
   }
 
@@ -320,8 +323,11 @@ package com.google.firebase.ai.type {
 
   public final class FunctionResponsePart implements com.google.firebase.ai.type.Part {
     ctor public FunctionResponsePart(String name, kotlinx.serialization.json.JsonObject response);
+    ctor public FunctionResponsePart(String name, kotlinx.serialization.json.JsonObject response, String? id = null);
+    method public String? getId();
     method public String getName();
     method public kotlinx.serialization.json.JsonObject getResponse();
+    property public final String? id;
     property public final String name;
     property public final kotlinx.serialization.json.JsonObject response;
   }
@@ -330,11 +336,13 @@ package com.google.firebase.ai.type {
     ctor public GenerateContentResponse(java.util.List<com.google.firebase.ai.type.Candidate> candidates, com.google.firebase.ai.type.PromptFeedback? promptFeedback, com.google.firebase.ai.type.UsageMetadata? usageMetadata);
     method public java.util.List<com.google.firebase.ai.type.Candidate> getCandidates();
     method public java.util.List<com.google.firebase.ai.type.FunctionCallPart> getFunctionCalls();
+    method public java.util.List<com.google.firebase.ai.type.InlineDataPart> getInlineDataParts();
     method public com.google.firebase.ai.type.PromptFeedback? getPromptFeedback();
     method public String? getText();
     method public com.google.firebase.ai.type.UsageMetadata? getUsageMetadata();
     property public final java.util.List<com.google.firebase.ai.type.Candidate> candidates;
     property public final java.util.List<com.google.firebase.ai.type.FunctionCallPart> functionCalls;
+    property public final java.util.List<com.google.firebase.ai.type.InlineDataPart> inlineDataParts;
     property public final com.google.firebase.ai.type.PromptFeedback? promptFeedback;
     property public final String? text;
     property public final com.google.firebase.ai.type.UsageMetadata? usageMetadata;
@@ -352,6 +360,7 @@ package com.google.firebase.ai.type {
     field public Integer? maxOutputTokens;
     field public Float? presencePenalty;
     field public String? responseMimeType;
+    field public java.util.List<com.google.firebase.ai.type.ResponseModality>? responseModalities;
     field public com.google.firebase.ai.type.Schema? responseSchema;
     field public java.util.List<java.lang.String>? stopSequences;
     field public Float? temperature;
@@ -576,30 +585,6 @@ package com.google.firebase.ai.type {
   public final class InvalidStateException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveContentResponse {
-    method public com.google.firebase.ai.type.Content? getData();
-    method public java.util.List<com.google.firebase.ai.type.FunctionCallPart>? getFunctionCalls();
-    method public int getStatus();
-    method public String? getText();
-    property public final com.google.firebase.ai.type.Content? data;
-    property public final java.util.List<com.google.firebase.ai.type.FunctionCallPart>? functionCalls;
-    property public final int status;
-    property public final String? text;
-  }
-
-  @kotlin.jvm.JvmInline public static final value class LiveContentResponse.Status {
-    field public static final com.google.firebase.ai.type.LiveContentResponse.Status.Companion Companion;
-  }
-
-  public static final class LiveContentResponse.Status.Companion {
-    method public int getINTERRUPTED();
-    method public int getNORMAL();
-    method public int getTURN_COMPLETE();
-    property public final int INTERRUPTED;
-    property public final int NORMAL;
-    property public final int TURN_COMPLETE;
-  }
-
   @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveGenerationConfig {
     field public static final com.google.firebase.ai.type.LiveGenerationConfig.Companion Companion;
   }
@@ -635,9 +620,40 @@ package com.google.firebase.ai.type {
     method public static com.google.firebase.ai.type.LiveGenerationConfig liveGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.LiveGenerationConfig.Builder,kotlin.Unit> init);
   }
 
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveServerContent implements com.google.firebase.ai.type.LiveServerMessage {
+    ctor public LiveServerContent(com.google.firebase.ai.type.Content? content, boolean interrupted, boolean turnComplete, boolean generationComplete);
+    method public com.google.firebase.ai.type.Content? getContent();
+    method public boolean getGenerationComplete();
+    method public boolean getInterrupted();
+    method public boolean getTurnComplete();
+    property public final com.google.firebase.ai.type.Content? content;
+    property public final boolean generationComplete;
+    property public final boolean interrupted;
+    property public final boolean turnComplete;
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public interface LiveServerMessage {
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveServerSetupComplete implements com.google.firebase.ai.type.LiveServerMessage {
+    ctor public LiveServerSetupComplete();
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveServerToolCall implements com.google.firebase.ai.type.LiveServerMessage {
+    ctor public LiveServerToolCall(java.util.List<com.google.firebase.ai.type.FunctionCallPart> functionCalls);
+    method public java.util.List<com.google.firebase.ai.type.FunctionCallPart> getFunctionCalls();
+    property public final java.util.List<com.google.firebase.ai.type.FunctionCallPart> functionCalls;
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveServerToolCallCancellation implements com.google.firebase.ai.type.LiveServerMessage {
+    ctor public LiveServerToolCallCancellation(java.util.List<java.lang.String> functionIds);
+    method public java.util.List<java.lang.String> getFunctionIds();
+    property public final java.util.List<java.lang.String> functionIds;
+  }
+
   @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveSession {
     method public suspend Object? close(kotlin.coroutines.Continuation<? super kotlin.Unit>);
-    method public kotlinx.coroutines.flow.Flow<com.google.firebase.ai.type.LiveContentResponse> receive();
+    method public kotlinx.coroutines.flow.Flow<com.google.firebase.ai.type.LiveServerMessage> receive();
     method public suspend Object? send(com.google.firebase.ai.type.Content content, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? send(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendFunctionResponse(java.util.List<com.google.firebase.ai.type.FunctionResponsePart> functionList, kotlin.coroutines.Continuation<? super kotlin.Unit>);

firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt

@@ -23,7 +23,7 @@ import com.google.common.util.concurrent.ListenableFuture
 import com.google.firebase.ai.type.Content
 import com.google.firebase.ai.type.FunctionCallPart
 import com.google.firebase.ai.type.FunctionResponsePart
-import com.google.firebase.ai.type.LiveContentResponse
+import com.google.firebase.ai.type.LiveServerMessage
 import com.google.firebase.ai.type.LiveSession
 import com.google.firebase.ai.type.MediaData
 import com.google.firebase.ai.type.PublicPreviewAPI
@@ -135,16 +135,16 @@ public abstract class LiveSessionFutures internal constructor() {
    *
    * Call [close] to stop receiving responses from the model.
    *
-   * @return A [Publisher] which will emit [LiveContentResponse] from the model.
+   * @return A [Publisher] which will emit [LiveServerMessage] from the model.
    *
    * @throws [SessionAlreadyReceivingException] when the session is already receiving.
    * @see stopReceiving
    */
-  public abstract fun receive(): Publisher<LiveContentResponse>
+  public abstract fun receive(): Publisher<LiveServerMessage>
 
   private class FuturesImpl(private val session: LiveSession) : LiveSessionFutures() {
 
-    override fun receive(): Publisher<LiveContentResponse> = session.receive().asPublisher()
+    override fun receive(): Publisher<LiveServerMessage> = session.receive().asPublisher()
 
     override fun close(): ListenableFuture<Unit> =
       SuspendToFutureAdapter.launchFuture { session.close() }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerateContentResponse.kt

@@ -44,6 +44,18 @@ public class GenerateContentResponse(
     candidates.first().content.parts.filterIsInstance<FunctionCallPart>()
   }
 
+  /**
+   * Convenience field representing all the [InlineDataPart]s in the first candidate, if they exist.
+   *
+   * This also includes any [ImagePart], but they will be represented as [InlineDataPart] instead.
+   */
+  public val inlineDataParts: List<InlineDataPart> by lazy {
+    candidates.first().content.parts.let { parts ->
+      parts.filterIsInstance<ImagePart>().map { it.toInlineDataPart() } +
+        parts.filterIsInstance<InlineDataPart>()
+    }
+  }
+
   @Serializable
   internal data class Internal(
     val candidates: List<Candidate.Internal>? = null,

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerationConfig.kt

@@ -75,6 +75,8 @@ import kotlinx.serialization.Serializable
  * Refer to the
  * [Control generated output](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/control-generated-output)
  * guide for more details.
+ *
+ * @property responseModalities The format of data in which the model should respond with.
  */
 public class GenerationConfig
 private constructor(
@@ -88,6 +90,7 @@ private constructor(
   internal val stopSequences: List<String>?,
   internal val responseMimeType: String?,
   internal val responseSchema: Schema?,
+  internal val responseModalities: List<ResponseModality>?,
 ) {
 
   /**
@@ -115,6 +118,9 @@ private constructor(
    * @property responseMimeType See [GenerationConfig.responseMimeType].
    *
    * @property responseSchema See [GenerationConfig.responseSchema].
+   *
+   * @property responseModalities See [GenerationConfig.responseModalities].
+   *
    * @see [generationConfig]
    */
   public class Builder {
@@ -128,6 +134,7 @@ private constructor(
     @JvmField public var stopSequences: List<String>? = null
     @JvmField public var responseMimeType: String? = null
     @JvmField public var responseSchema: Schema? = null
+    @JvmField public var responseModalities: List<ResponseModality>? = null
 
     /** Create a new [GenerationConfig] with the attached arguments. */
     public fun build(): GenerationConfig =
@@ -142,6 +149,7 @@ private constructor(
         frequencyPenalty = frequencyPenalty,
         responseMimeType = responseMimeType,
         responseSchema = responseSchema,
+        responseModalities = responseModalities
       )
   }
 
@@ -156,7 +164,8 @@ private constructor(
       frequencyPenalty = frequencyPenalty,
       presencePenalty = presencePenalty,
       responseMimeType = responseMimeType,
-      responseSchema = responseSchema?.toInternal()
+      responseSchema = responseSchema?.toInternal(),
+      responseModalities = responseModalities?.map { it.toInternal() }
     )
 
   @Serializable
@@ -171,6 +180,7 @@ private constructor(
     @SerialName("presence_penalty") val presencePenalty: Float? = null,
     @SerialName("frequency_penalty") val frequencyPenalty: Float? = null,
     @SerialName("response_schema") val responseSchema: Schema.Internal? = null,
+    @SerialName("response_modalities") val responseModalities: List<String>? = null
   )
 
   public companion object {