diff --git a/e2e/sample-apps/modular.js b/e2e/sample-apps/modular.js index aeebe19a4b1..abcf829856b 100644 --- a/e2e/sample-apps/modular.js +++ b/e2e/sample-apps/modular.js @@ -314,13 +314,22 @@ async function callVertexAI(app) { console.log('[VERTEXAI] start'); const vertexAI = getVertexAI(app); const model = getGenerativeModel(vertexAI, { - mode: 'only_on_device' + mode: 'prefer_on_device' }); const singleResult = await model.generateContent([ - { text: 'describe the following:' }, - { text: 'the mojave desert' } + { text: 'describe this 20 x 20 px image in two words' }, + { + inlineData: { + mimeType: 'image/heic', + data: 'AAAAGGZ0eXBoZWljAAAAAGhlaWNtaWYxAAAB7G1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAHBpY3QAAAAAAAAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAADnBpdG0AAAAAAAEAAAA4aWluZgAAAAAAAgAAABVpbmZlAgAAAAABAABodmMxAAAAABVpbmZlAgAAAQACAABFeGlmAAAAABppcmVmAAAAAAAAAA5jZHNjAAIAAQABAAABD2lwcnAAAADtaXBjbwAAABNjb2xybmNseAACAAIABoAAAAAMY2xsaQDLAEAAAAAUaXNwZQAAAAAAAAAUAAAADgAAAChjbGFwAAAAFAAAAAEAAAANAAAAAQAAAAAAAAAB/8AAAACAAAAAAAAJaXJvdAAAAAAQcGl4aQAAAAADCAgIAAAAcWh2Y0MBA3AAAACwAAAAAAAe8AD8/fj4AAALA6AAAQAXQAEMAf//A3AAAAMAsAAAAwAAAwAecCShAAEAI0IBAQNwAAADALAAAAMAAAMAHqAUIEHAjw1iHuRZVNwICBgCogABAAlEAcBhcshAUyQAAAAaaXBtYQAAAAAAAAABAAEHgQIDhIUGhwAAACxpbG9jAAAAAEQAAAIAAQAAAAEAAAJsAAABDAACAAAAAQAAAhQAAABYAAAAAW1kYXQAAAAAAAABdAAAAAZFeGlmAABNTQAqAAAACAAEARIAAwAAAAEAAQAAARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAAAAAAAAAAEgAAAABAAAASAAAAAEAAAEIKAGvoR8wDimTiRYUbALiHkU3ZdZ8DXAcSrRB9GARtVQHvnCE0LEyBGAyb5P4eYr6JAK5UxNX10WNlARq3ZpcGeVD+Xom6LodYasuZKKtDHCz/xnswOtC/ksZzVKhtWQqGvkXcsJnLYqWevNkacnccQ95jbHJBg9nXub69jAAN3xhNOXxjGSxaG9QvES5R7sYICEojRjLF5OB5K3v+okQAwfgWpz/u21ayideOgOZQLAyBkKOv7ymLNCagiPWTlHAuy/3qR1Q7m2ERFaxKIAbLSkIVO/P8m8+anKxhzhC//L8NMAUoF+Sf3aEH9O41fwLc+PlcbrDrjgY2EboD3cn9DyN32Rum2Ym' + } + } ]); console.log(`Generated text: ${singleResult.response.text()}`); + const chat = model.startChat(); + let chatResult = await chat.sendMessage('describe red in two words'); + chatResult = await chat.sendMessage('describe blue'); + console.log('Chat history:', await chat.getHistory()); console.log(`[VERTEXAI] end`); } @@ -345,7 +354,7 @@ function callDataConnect(app) { async function main() { console.log('FIREBASE VERSION', SDK_VERSION); const app = initializeApp(config); - setLogLevel('warn'); + setLogLevel('debug'); // callAppCheck(app); // await authLogin(app); diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 9ac8f350a02..9ba674937a8 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -16,6 +16,7 @@ */ import { AIError } from '../errors'; +import { logger } from '../logger'; import { CountTokensRequest, GenerateContentRequest, @@ -65,6 +66,9 @@ export class ChromeAdapter { */ async isAvailable(request: GenerateContentRequest): Promise { if (this.mode === 'only_in_cloud') { + logger.debug( + `On-device inference unavailable because mode is "only_in_cloud".` + ); return false; } @@ -76,10 +80,20 @@ export class ChromeAdapter { } // Applies prefer_on_device logic. - return ( - availability === Availability.available && - ChromeAdapter.isOnDeviceRequest(request) - ); + if (availability !== Availability.available) { + logger.debug( + `On-device inference unavailable because availability is "${availability}".` + ); + return false; + } + if (!ChromeAdapter.isOnDeviceRequest(request)) { + logger.debug( + `On-device inference unavailable because request is incompatible.` + ); + return false; + } + + return true; } /** @@ -135,6 +149,7 @@ export class ChromeAdapter { private static isOnDeviceRequest(request: GenerateContentRequest): boolean { // Returns false if the prompt is empty. if (request.contents.length === 0) { + logger.debug('Empty prompt rejected for on-device inference.'); return false; } @@ -142,6 +157,9 @@ export class ChromeAdapter { // Returns false if the request contains multiple roles, eg a chat history. // TODO: remove this guard once LanguageModelMessage is supported. if (content.role !== 'user') { + logger.debug( + `Non-user role "${content.role}" rejected for on-device inference.` + ); return false; } @@ -153,6 +171,9 @@ export class ChromeAdapter { part.inlineData.mimeType ) === -1 ) { + logger.debug( + `Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.` + ); return false; } }