Reject image input for Count Token on device

common/api-review/vertexai.api.md

@@ -42,7 +42,8 @@ export class BooleanSchema extends Schema {
 // @public
 export class ChatSession {
     // Warning: (ae-forgotten-export) The symbol "ApiSettings" needs to be exported by the entry point index.d.ts
-    constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
+    // Warning: (ae-forgotten-export) The symbol "ChromeAdapter" needs to be exported by the entry point index.d.ts
+    constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapter, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
     getHistory(): Promise<Content[]>;
     // (undocumented)
     model: string;
@@ -324,8 +325,9 @@ export interface GenerativeContentBlob {
 
 // @public
 export class GenerativeModel extends VertexAIModel {
-    constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions);
+    constructor(vertexAI: VertexAI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions);
     countTokens(request: CountTokensRequest | string | Array<string | Part>): Promise<CountTokensResponse>;
+    static DEFAULT_HYBRID_IN_CLOUD_MODEL: string;
     generateContent(request: GenerateContentRequest | string | Array<string | Part>): Promise<GenerateContentResult>;
     generateContentStream(request: GenerateContentRequest | string | Array<string | Part>): Promise<GenerateContentStreamResult>;
     // (undocumented)
@@ -344,7 +346,7 @@ export class GenerativeModel extends VertexAIModel {
 }
 
 // @public
-export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel;
+export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel;
 
 // @beta
 export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
@@ -416,6 +418,14 @@ export enum HarmSeverity {
     HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE"
 }
 
+// @public
+export interface HybridParams {
+    inCloudParams?: ModelParams;
+    mode: InferenceMode;
+    // Warning: (ae-forgotten-export) The symbol "LanguageModelCreateOptions" needs to be exported by the entry point index.d.ts
+    onDeviceParams?: LanguageModelCreateOptions;
+}
+
 // @beta
 export enum ImagenAspectRatio {
     LANDSCAPE_16x9 = "16:9",
@@ -500,6 +510,9 @@ export interface ImagenSafetySettings {
     safetyFilterLevel?: ImagenSafetyFilterLevel;
 }
 
+// @public
+export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud';
+
 // @public
 export interface InlineDataPart {
     // (undocumented)

docs-devsite/_toc.yaml

@@ -536,6 +536,8 @@ toc:
     path: /docs/reference/js/vertexai.groundingattribution.md
   - title: GroundingMetadata
     path: /docs/reference/js/vertexai.groundingmetadata.md
+  - title: HybridParams
+    path: /docs/reference/js/vertexai.hybridparams.md
   - title: ImagenGCSImage
     path: /docs/reference/js/vertexai.imagengcsimage.md
   - title: ImagenGenerationConfig

docs-devsite/vertexai.chatsession.md

@@ -22,7 +22,7 @@ export declare class ChatSession
 
 |  Constructor | Modifiers | Description |
 |  --- | --- | --- |
-|  [(constructor)(apiSettings, model, params, requestOptions)](./vertexai.chatsession.md#chatsessionconstructor) |  | Constructs a new instance of the <code>ChatSession</code> class |
+|  [(constructor)(apiSettings, model, chromeAdapter, params, requestOptions)](./vertexai.chatsession.md#chatsessionconstructor) |  | Constructs a new instance of the <code>ChatSession</code> class |
 
 ## Properties
 
@@ -47,7 +47,7 @@ Constructs a new instance of the `ChatSession` class
 <b>Signature:</b>
 
 ```typescript
-constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
+constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapter, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
 ```
 
 #### Parameters
@@ -56,6 +56,7 @@ constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams |
 |  --- | --- | --- |
 |  apiSettings | ApiSettings |  |
 |  model | string |  |
+|  chromeAdapter | ChromeAdapter |  |
 |  params | [StartChatParams](./vertexai.startchatparams.md#startchatparams_interface) \| undefined |  |
 |  requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined |  |
 

docs-devsite/vertexai.generativemodel.md

@@ -23,12 +23,13 @@ export declare class GenerativeModel extends VertexAIModel
 
 |  Constructor | Modifiers | Description |
 |  --- | --- | --- |
-|  [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.generativemodel.md#generativemodelconstructor) |  | Constructs a new instance of the <code>GenerativeModel</code> class |
+|  [(constructor)(vertexAI, modelParams, chromeAdapter, requestOptions)](./vertexai.generativemodel.md#generativemodelconstructor) |  | Constructs a new instance of the <code>GenerativeModel</code> class |
 
 ## Properties
 
 |  Property | Modifiers | Type | Description |
 |  --- | --- | --- | --- |
+|  [DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL](./vertexai.generativemodel.md#generativemodeldefault_hybrid_in_cloud_model) | <code>static</code> | string | Defines the name of the default in-cloud model to use for hybrid inference. |
 |  [generationConfig](./vertexai.generativemodel.md#generativemodelgenerationconfig) |  | [GenerationConfig](./vertexai.generationconfig.md#generationconfig_interface) |  |
 |  [requestOptions](./vertexai.generativemodel.md#generativemodelrequestoptions) |  | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) |  |
 |  [safetySettings](./vertexai.generativemodel.md#generativemodelsafetysettings) |  | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface)<!-- -->\[\] |  |
@@ -52,7 +53,7 @@ Constructs a new instance of the `GenerativeModel` class
 <b>Signature:</b>
 
 ```typescript
-constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions);
+constructor(vertexAI: VertexAI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions);
 ```
 
 #### Parameters
@@ -61,8 +62,19 @@ constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: Reque
 |  --- | --- | --- |
 |  vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) |  |
 |  modelParams | [ModelParams](./vertexai.modelparams.md#modelparams_interface) |  |
+|  chromeAdapter | ChromeAdapter |  |
 |  requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) |  |
 
+## GenerativeModel.DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL
+
+Defines the name of the default in-cloud model to use for hybrid inference.
+
+<b>Signature:</b>
+
+```typescript
+static DEFAULT_HYBRID_IN_CLOUD_MODEL: string;
+```
+
 ## GenerativeModel.generationConfig
 
 <b>Signature:</b>

docs-devsite/vertexai.hybridparams.md

@@ -0,0 +1,57 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# HybridParams interface
+Toggles hybrid inference.
+
+<b>Signature:</b>
+
+```typescript
+export interface HybridParams 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [inCloudParams](./vertexai.hybridparams.md#hybridparamsincloudparams) | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Optional. Specifies advanced params for in-cloud inference. |
+|  [mode](./vertexai.hybridparams.md#hybridparamsmode) | [InferenceMode](./vertexai.md#inferencemode) | Specifies on-device or in-cloud inference. Defaults to prefer on-device. |
+|  [onDeviceParams](./vertexai.hybridparams.md#hybridparamsondeviceparams) | LanguageModelCreateOptions | Optional. Specifies advanced params for on-device inference. |
+
+## HybridParams.inCloudParams
+
+Optional. Specifies advanced params for in-cloud inference.
+
+<b>Signature:</b>
+
+```typescript
+inCloudParams?: ModelParams;
+```
+
+## HybridParams.mode
+
+Specifies on-device or in-cloud inference. Defaults to prefer on-device.
+
+<b>Signature:</b>
+
+```typescript
+mode: InferenceMode;
+```
+
+## HybridParams.onDeviceParams
+
+Optional. Specifies advanced params for on-device inference.
+
+<b>Signature:</b>
+
+```typescript
+onDeviceParams?: LanguageModelCreateOptions;
+```

docs-devsite/vertexai.md

@@ -19,7 +19,7 @@ The Vertex AI in Firebase Web SDK.
 |  <b>function(app, ...)</b> |
 |  [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. |
 |  <b>function(vertexAI, ...)</b> |
-|  [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. |
+|  [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_8dbc150) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. |
 |  [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | <b><i>(Public Preview)</i></b> Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.<!-- -->Only Imagen 3 models (named <code>imagen-3.0-*</code>) are supported. |
 
 ## Classes
@@ -91,6 +91,7 @@ The Vertex AI in Firebase Web SDK.
 |  [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. |
 |  [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) |  |
 |  [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. |
+|  [HybridParams](./vertexai.hybridparams.md#hybridparams_interface) | Toggles hybrid inference. |
 |  [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.<!-- -->This feature is not available yet. |
 |  [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | <b><i>(Public Preview)</i></b> Configuration options for generating images with Imagen.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. |
 |  [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | <b><i>(Public Preview)</i></b> The response from a request to generate images with Imagen. |
@@ -99,10 +100,10 @@ The Vertex AI in Firebase Web SDK.
 |  [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | <b><i>(Public Preview)</i></b> Settings for controlling the aggressiveness of filtering out sensitive content.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. |
 |  [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. |
 |  [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. |
-|  [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9)<!-- -->. |
+|  [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_8dbc150)<!-- -->. |
 |  [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. |
 |  [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with <code>blockReason</code> and the relevant <code>safetyRatings</code>. |
-|  [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9)<!-- -->. |
+|  [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_8dbc150)<!-- -->. |
 |  [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) |  |
 |  [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) |
 |  [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. |
@@ -130,6 +131,7 @@ The Vertex AI in Firebase Web SDK.
 
 |  Type Alias | Description |
 |  --- | --- |
+|  [InferenceMode](./vertexai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. |
 |  [Part](./vertexai.md#part) | Content part - includes text, image/video, or function call/response part types. |
 |  [Role](./vertexai.md#role) | Role is the producer of the content. |
 |  [Tool](./vertexai.md#tool) | Defines a tool that model can call to access external knowledge. |
@@ -160,22 +162,22 @@ export declare function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions
 
 ## function(vertexAI, ...)
 
-### getGenerativeModel(vertexAI, modelParams, requestOptions) {:#getgenerativemodel_e3037c9}
+### getGenerativeModel(vertexAI, modelParams, requestOptions) {:#getgenerativemodel_8dbc150}
 
 Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality.
 
 <b>Signature:</b>
 
 ```typescript
-export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel;
+export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel;
 ```
 
 #### Parameters
 
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) |  |
-|  modelParams | [ModelParams](./vertexai.modelparams.md#modelparams_interface) |  |
+|  modelParams | [ModelParams](./vertexai.modelparams.md#modelparams_interface) \| [HybridParams](./vertexai.hybridparams.md#hybridparams_interface) |  |
 |  requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) |  |
 
 <b>Returns:</b>
@@ -223,6 +225,16 @@ Possible roles.
 POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]
 ```
 
+## InferenceMode
+
+Determines whether inference happens on-device or in-cloud.
+
+<b>Signature:</b>
+
+```typescript
+export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud';
+```
+
 ## Part
 
 Content part - includes text, image/video, or function call/response part types.

docs-devsite/vertexai.modelparams.md

@@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
 # ModelParams interface
-Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9)<!-- -->.
+Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_8dbc150)<!-- -->.
 
 <b>Signature:</b>
 

docs-devsite/vertexai.requestoptions.md

@@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
 # RequestOptions interface
-Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9)<!-- -->.
+Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_8dbc150)<!-- -->.
 
 <b>Signature:</b>
 

e2e/sample-apps/modular.js

@@ -58,7 +58,7 @@ import {
   onValue,
   off
 } from 'firebase/database';
-import { getGenerativeModel, getVertexAI, VertexAI } from 'firebase/vertexai';
+import { getGenerativeModel, getVertexAI } from 'firebase/vertexai';
 import { getDataConnect, DataConnect } from 'firebase/data-connect';
 
 /**
@@ -313,9 +313,15 @@ function callPerformance(app) {
 async function callVertexAI(app) {
   console.log('[VERTEXAI] start');
   const vertexAI = getVertexAI(app);
-  const model = getGenerativeModel(vertexAI, { model: 'gemini-1.5-flash' });
-  const result = await model.countTokens('abcdefg');
-  console.log(`[VERTEXAI] counted tokens: ${result.totalTokens}`);
+  const model = getGenerativeModel(vertexAI, {
+    mode: 'prefer_on_device'
+  });
+  const singleResult = await model.generateContent([
+    { text: 'describe the following:' },
+    { text: 'the mojave desert' }
+  ]);
+  console.log(`Generated text: ${singleResult.response.text()}`);
+  console.log(`[VERTEXAI] end`);
 }
 
 /**
@@ -341,18 +347,18 @@ async function main() {
   const app = initializeApp(config);
   setLogLevel('warn');
 
-  callAppCheck(app);
-  await authLogin(app);
-  await callStorage(app);
-  await callFirestore(app);
-  await callDatabase(app);
-  await callMessaging(app);
-  callAnalytics(app);
-  callPerformance(app);
-  await callFunctions(app);
+  // callAppCheck(app);
+  // await authLogin(app);
+  // await callStorage(app);
+  // await callFirestore(app);
+  // await callDatabase(app);
+  // await callMessaging(app);
+  // callAnalytics(app);
+  // callPerformance(app);
+  // await callFunctions(app);
   await callVertexAI(app);
-  callDataConnect(app);
-  await authLogout(app);
+  // callDataConnect(app);
+  // await authLogout(app);
   console.log('DONE');
 }
 

packages/vertexai/src/api.test.ts

@@ -101,6 +101,21 @@ describe('Top level API', () => {
     expect(genModel).to.be.an.instanceOf(GenerativeModel);
     expect(genModel.model).to.equal('publishers/google/models/my-model');
   });
+  it('getGenerativeModel with HybridParams sets a default model', () => {
+    const genModel = getGenerativeModel(fakeVertexAI, {
+      mode: 'only_on_device'
+    });
+    expect(genModel.model).to.equal(
+      `publishers/google/models/${GenerativeModel.DEFAULT_HYBRID_IN_CLOUD_MODEL}`
+    );
+  });
+  it('getGenerativeModel with HybridParams honors a model override', () => {
+    const genModel = getGenerativeModel(fakeVertexAI, {
+      mode: 'prefer_on_device',
+      inCloudParams: { model: 'my-model' }
+    });
+    expect(genModel.model).to.equal('publishers/google/models/my-model');
+  });
   it('getImagenModel throws if no model is provided', () => {
     try {
       getImagenModel(fakeVertexAI, {} as ImagenModelParams);