Bidirectional Streaming Android

firebase-vertexai/CHANGELOG.md

@@ -3,6 +3,7 @@
   `GenerativeModel` or `ImagenModel`.
 * [changed] Added new exception type for quota exceeded scenarios.
 * [feature] `CountTokenRequest` now includes `GenerationConfig` from the model.
+* [feature] Public Preview: Added support for streaming input and output (including audio) using the [Multimodal Live API](/docs/vertex-ai/live-api?platform=android)
 * [changed] **Breaking Change**: `ImagenInlineImage.data` now returns the raw
   image bytes (in JPEG or PNG format, as specified in
   `ImagenInlineImage.mimeType`) instead of Base64-encoded data. (#6800)

firebase-vertexai/api.txt

@@ -29,6 +29,11 @@ package com.google.firebase.vertexai {
     method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null);
     method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null);
     method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.vertexai.type.Tool>? tools = null);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.vertexai.type.Tool>? tools = null, com.google.firebase.vertexai.type.Content? systemInstruction = null);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.vertexai.type.Tool>? tools = null, com.google.firebase.vertexai.type.Content? systemInstruction = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
     property public static final com.google.firebase.vertexai.FirebaseVertexAI instance;
     field public static final com.google.firebase.vertexai.FirebaseVertexAI.Companion Companion;
   }
@@ -63,6 +68,10 @@ package com.google.firebase.vertexai {
     method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>>);
   }
 
+  public final class LiveGenerativeModel {
+    method public suspend Object? connect(kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.LiveSession>);
+  }
+
 }
 
 package com.google.firebase.vertexai.java {
@@ -105,10 +114,43 @@ package com.google.firebase.vertexai.java {
     method public com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
   }
 
+  public abstract class LiveModelFutures {
+    method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.vertexai.type.LiveSession> connect();
+    method public static final com.google.firebase.vertexai.java.LiveModelFutures from(com.google.firebase.vertexai.LiveGenerativeModel model);
+    field public static final com.google.firebase.vertexai.java.LiveModelFutures.Companion Companion;
+  }
+
+  public static final class LiveModelFutures.Companion {
+    method public com.google.firebase.vertexai.java.LiveModelFutures from(com.google.firebase.vertexai.LiveGenerativeModel model);
+  }
+
+  public abstract class LiveSessionFutures {
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> close();
+    method public static final com.google.firebase.vertexai.java.LiveSessionFutures from(com.google.firebase.vertexai.type.LiveSession session);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<org.reactivestreams.Publisher<com.google.firebase.vertexai.type.LiveContentResponse>> receive(java.util.List<com.google.firebase.vertexai.type.ContentModality> outputModalities);
+    method public abstract org.reactivestreams.Publisher<java.util.List<com.google.firebase.vertexai.type.FunctionCallPart>> receiveAudioConversationFunctionCalls();
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> send(com.google.firebase.vertexai.type.Content content);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> send(String text);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendFunctionResponse(java.util.List<com.google.firebase.vertexai.type.FunctionResponsePart> functionList);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendMediaStream(java.util.List<com.google.firebase.vertexai.type.MediaData> mediaChunks);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation();
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> stopAudioConversation();
+    method public abstract void stopReceiving();
+    field public static final com.google.firebase.vertexai.java.LiveSessionFutures.Companion Companion;
+  }
+
+  public static final class LiveSessionFutures.Companion {
+    method public com.google.firebase.vertexai.java.LiveSessionFutures from(com.google.firebase.vertexai.type.LiveSession session);
+  }
+
 }
 
 package com.google.firebase.vertexai.type {
 
+  public final class AudioRecordInitializationFailedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
+    ctor public AudioRecordInitializationFailedException(String message);
+  }
+
   public final class BlockReason {
     method public String getName();
     method public int getOrdinal();
@@ -284,6 +326,10 @@ package com.google.firebase.vertexai.type {
     property public final kotlinx.serialization.json.JsonObject response;
   }
 
+  public final class GeminiConnectionHandshakeFailed extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
+    ctor public GeminiConnectionHandshakeFailed();
+  }
+
   public final class GenerateContentResponse {
     ctor public GenerateContentResponse(java.util.List<com.google.firebase.vertexai.type.Candidate> candidates, com.google.firebase.vertexai.type.PromptFeedback? promptFeedback, com.google.firebase.vertexai.type.UsageMetadata? usageMetadata);
     method public java.util.List<com.google.firebase.vertexai.type.Candidate> getCandidates();
@@ -520,6 +566,86 @@ package com.google.firebase.vertexai.type {
   public final class InvalidStateException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
   }
 
+  public final class LiveContentResponse {
+    method public com.google.firebase.vertexai.type.Content? getData();
+    method public java.util.List<com.google.firebase.vertexai.type.FunctionCallPart>? getFunctionCalls();
+    method public int getStatus();
+    method public String? getText();
+    property public final com.google.firebase.vertexai.type.Content? data;
+    property public final java.util.List<com.google.firebase.vertexai.type.FunctionCallPart>? functionCalls;
+    property public final int status;
+    property public final String? text;
+  }
+
+  @kotlin.jvm.JvmInline public static final value class LiveContentResponse.Status {
+    field public static final com.google.firebase.vertexai.type.LiveContentResponse.Status.Companion Companion;
+  }
+
+  public static final class LiveContentResponse.Status.Companion {
+    method public int getINTERRUPTED();
+    method public int getNORMAL();
+    method public int getTURN_COMPLETE();
+    property public final int INTERRUPTED;
+    property public final int NORMAL;
+    property public final int TURN_COMPLETE;
+  }
+
+  public final class LiveGenerationConfig {
+    field public static final com.google.firebase.vertexai.type.LiveGenerationConfig.Companion Companion;
+  }
+
+  public static final class LiveGenerationConfig.Builder {
+    ctor public LiveGenerationConfig.Builder();
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig build();
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setCandidateCount(Integer? candidateCount);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setFrequencyPenalty(Float? frequencyPenalty);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setMaxOutputTokens(Integer? maxOutputTokens);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setPresencePenalty(Float? presencePenalty);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setResponseModalities(java.util.List<com.google.firebase.vertexai.type.ContentModality>? responseModalities);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setSpeechConfig(com.google.firebase.vertexai.type.SpeechConfig? speechConfig);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setTemperature(Float? temperature);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setTopK(Integer? topK);
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setTopP(Float? topP);
+    field public Integer? candidateCount;
+    field public Float? frequencyPenalty;
+    field public Integer? maxOutputTokens;
+    field public Float? presencePenalty;
+    field public java.util.List<com.google.firebase.vertexai.type.ContentModality>? responseModalities;
+    field public com.google.firebase.vertexai.type.SpeechConfig? speechConfig;
+    field public Float? temperature;
+    field public Integer? topK;
+    field public Float? topP;
+  }
+
+  public static final class LiveGenerationConfig.Companion {
+    method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder builder();
+  }
+
+  public final class LiveGenerationConfigKt {
+    method public static com.google.firebase.vertexai.type.LiveGenerationConfig liveGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.LiveGenerationConfig.Builder,kotlin.Unit> init);
+  }
+
+  public final class LiveSession {
+    method public suspend Object? close(kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? receive(java.util.List<com.google.firebase.vertexai.type.ContentModality> outputModalities, kotlin.coroutines.Continuation<? super kotlinx.coroutines.flow.Flow<? extends com.google.firebase.vertexai.type.LiveContentResponse>>);
+    method public kotlinx.coroutines.flow.Flow<java.util.List<com.google.firebase.vertexai.type.FunctionCallPart>> receiveAudioConversationFunctionCalls();
+    method public suspend Object? send(com.google.firebase.vertexai.type.Content content, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? send(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? sendFunctionResponse(java.util.List<com.google.firebase.vertexai.type.FunctionResponsePart> functionList, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? sendMediaStream(java.util.List<com.google.firebase.vertexai.type.MediaData> mediaChunks, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? startAudioConversation(kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public void stopAudioConversation();
+    method public void stopReceiving();
+  }
+
+  public final class MediaData {
+    ctor public MediaData(String mimeType, byte[] data);
+    method public byte[] getData();
+    method public String getMimeType();
+    property public final byte[] data;
+    property public final String mimeType;
+  }
+
   public final class ModalityTokenCount {
     method public operator com.google.firebase.vertexai.type.ContentModality component1();
     method public operator int component2();
@@ -682,6 +808,16 @@ package com.google.firebase.vertexai.type {
   public final class ServiceDisabledException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
   }
 
+  public final class SessionAlreadyReceivingException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
+    ctor public SessionAlreadyReceivingException();
+  }
+
+  public final class SpeechConfig {
+    ctor public SpeechConfig(com.google.firebase.vertexai.type.Voices voice);
+    method public com.google.firebase.vertexai.type.Voices getVoice();
+    property public final com.google.firebase.vertexai.type.Voices voice;
+  }
+
   public abstract class StringFormat {
   }
 
@@ -728,5 +864,20 @@ package com.google.firebase.vertexai.type {
     property public final int totalTokenCount;
   }
 
+  public final class Voices {
+    method public int getOrdinal();
+    property public final int ordinal;
+    field public static final com.google.firebase.vertexai.type.Voices AOEDE;
+    field public static final com.google.firebase.vertexai.type.Voices CHARON;
+    field public static final com.google.firebase.vertexai.type.Voices.Companion Companion;
+    field public static final com.google.firebase.vertexai.type.Voices FENRIR;
+    field public static final com.google.firebase.vertexai.type.Voices KORE;
+    field public static final com.google.firebase.vertexai.type.Voices PUCK;
+    field public static final com.google.firebase.vertexai.type.Voices UNSPECIFIED;
+  }
+
+  public static final class Voices.Companion {
+  }
+
 }
 

firebase-vertexai/firebase-vertexai.gradle.kts

@@ -63,7 +63,10 @@ android {
       isReturnDefaultValues = true
     }
   }
-  lint { targetSdk = targetSdkVersion }
+  lint {
+    targetSdk = targetSdkVersion
+    baseline = file("lint-baseline.xml")
+  }
 }
 
 // Enable Kotlin "Explicit API Mode". This causes the Kotlin compiler to fail if any
@@ -83,6 +86,7 @@ tasks.withType<KotlinCompile>().all {
 dependencies {
   implementation(libs.ktor.client.okhttp)
   implementation(libs.ktor.client.core)
+  implementation(libs.ktor.client.websockets)
   implementation(libs.ktor.client.content.negotiation)
   implementation(libs.ktor.serialization.kotlinx.json)
   implementation(libs.ktor.client.logging)

firebase-vertexai/lint-baseline.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<issues format="6" by="lint 8.3.2" type="baseline" client="gradle" dependencies="false" name="AGP (8.3.2)" variant="all" version="8.3.2">
+
+    <issue
+        id="MissingPermission"
+        message="Missing permissions required by AudioRecord.AudioRecord: android.permission.RECORD_AUDIO"
+        errorLine1="      AudioRecord("
+        errorLine2="      ^">
+        <location
+            file="src/main/kotlin/com/google/firebase/vertexai/type/AudioHelper.kt"
+            line="85"
+            column="7"/>
+    </issue>
+
+    <issue
+        id="UnsafeOptInUsageWarning"
+        message="This declaration is opt-in and its usage should be marked with `@kotlinx.serialization.ExperimentalSerializationApi` or `@OptIn(markerClass = kotlinx.serialization.ExperimentalSerializationApi.class)`">
+        <location
+            file="src/main/kotlin/com/google/firebase/vertexai/type/LiveSession.kt"/>
+    </issue>
+
+</issues>

firebase-vertexai/src/main/AndroidManifest.xml

@@ -20,7 +20,11 @@
   <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
   <uses-permission android:name="android.permission.INTERNET" />
 
-  <application>
+  <!-- To use the audio conversation feature of Live Sessions, update your app's manifest -->
+  <!-- to request audio recording permission -->
+  <!-- <uses-permission android:name="android.permission.RECORD_AUDIO" /> -->
+
+    <application>
     <service android:name="com.google.firebase.components.ComponentDiscoveryService"
       android:exported="false">
       <meta-data