Merge 78d1cc2 into 9968791

Author: erikeldridge

e2e/sample-apps/modular.js

@@ -314,13 +314,22 @@ async function callVertexAI(app) {
   console.log('[VERTEXAI] start');
   const vertexAI = getVertexAI(app);
   const model = getGenerativeModel(vertexAI, {
-    mode: 'only_on_device'
+    mode: 'prefer_on_device'
   });
   const singleResult = await model.generateContent([
-    { text: 'describe the following:' },
-    { text: 'the mojave desert' }
+    { text: 'describe this 20 x 20 px image in two words' },
+    {
+      inlineData: {
+        mimeType: 'image/heic',
+        data: 'AAAAGGZ0eXBoZWljAAAAAGhlaWNtaWYxAAAB7G1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAHBpY3QAAAAAAAAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAADnBpdG0AAAAAAAEAAAA4aWluZgAAAAAAAgAAABVpbmZlAgAAAAABAABodmMxAAAAABVpbmZlAgAAAQACAABFeGlmAAAAABppcmVmAAAAAAAAAA5jZHNjAAIAAQABAAABD2lwcnAAAADtaXBjbwAAABNjb2xybmNseAACAAIABoAAAAAMY2xsaQDLAEAAAAAUaXNwZQAAAAAAAAAUAAAADgAAAChjbGFwAAAAFAAAAAEAAAANAAAAAQAAAAAAAAAB/8AAAACAAAAAAAAJaXJvdAAAAAAQcGl4aQAAAAADCAgIAAAAcWh2Y0MBA3AAAACwAAAAAAAe8AD8/fj4AAALA6AAAQAXQAEMAf//A3AAAAMAsAAAAwAAAwAecCShAAEAI0IBAQNwAAADALAAAAMAAAMAHqAUIEHAjw1iHuRZVNwICBgCogABAAlEAcBhcshAUyQAAAAaaXBtYQAAAAAAAAABAAEHgQIDhIUGhwAAACxpbG9jAAAAAEQAAAIAAQAAAAEAAAJsAAABDAACAAAAAQAAAhQAAABYAAAAAW1kYXQAAAAAAAABdAAAAAZFeGlmAABNTQAqAAAACAAEARIAAwAAAAEAAQAAARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAAAAAAAAAAEgAAAABAAAASAAAAAEAAAEIKAGvoR8wDimTiRYUbALiHkU3ZdZ8DXAcSrRB9GARtVQHvnCE0LEyBGAyb5P4eYr6JAK5UxNX10WNlARq3ZpcGeVD+Xom6LodYasuZKKtDHCz/xnswOtC/ksZzVKhtWQqGvkXcsJnLYqWevNkacnccQ95jbHJBg9nXub69jAAN3xhNOXxjGSxaG9QvES5R7sYICEojRjLF5OB5K3v+okQAwfgWpz/u21ayideOgOZQLAyBkKOv7ymLNCagiPWTlHAuy/3qR1Q7m2ERFaxKIAbLSkIVO/P8m8+anKxhzhC//L8NMAUoF+Sf3aEH9O41fwLc+PlcbrDrjgY2EboD3cn9DyN32Rum2Ym'
+      }
+    }
   ]);
   console.log(`Generated text: ${singleResult.response.text()}`);
+  const chat = model.startChat();
+  let chatResult = await chat.sendMessage('describe red in two words');
+  chatResult = await chat.sendMessage('describe blue');
+  console.log('Chat history:', await chat.getHistory());
   console.log(`[VERTEXAI] end`);
 }
 
@@ -345,7 +354,7 @@ function callDataConnect(app) {
 async function main() {
   console.log('FIREBASE VERSION', SDK_VERSION);
   const app = initializeApp(config);
-  setLogLevel('warn');
+  setLogLevel('debug');
 
   // callAppCheck(app);
   // await authLogin(app);

packages/vertexai/src/methods/chrome-adapter.ts

@@ -16,6 +16,7 @@
  */
 
 import { AIError } from '../errors';
+import { logger } from '../logger';
 import {
   CountTokensRequest,
   GenerateContentRequest,
@@ -65,6 +66,9 @@ export class ChromeAdapter {
    */
   async isAvailable(request: GenerateContentRequest): Promise<boolean> {
     if (this.mode === 'only_in_cloud') {
+      logger.debug(
+        `On-device inference unavailable because mode is "only_in_cloud".`
+      );
       return false;
     }
 
@@ -76,10 +80,20 @@ export class ChromeAdapter {
     }
 
     // Applies prefer_on_device logic.
-    return (
-      availability === Availability.available &&
-      ChromeAdapter.isOnDeviceRequest(request)
-    );
+    if (availability !== Availability.available) {
+      logger.debug(
+        `On-device inference unavailable because availability is "${availability}".`
+      );
+      return false;
+    }
+    if (!ChromeAdapter.isOnDeviceRequest(request)) {
+      logger.debug(
+        `On-device inference unavailable because request is incompatible.`
+      );
+      return false;
+    }
+
+    return true;
   }
 
   /**
@@ -141,13 +155,17 @@ export class ChromeAdapter {
   private static isOnDeviceRequest(request: GenerateContentRequest): boolean {
     // Returns false if the prompt is empty.
     if (request.contents.length === 0) {
+      logger.debug('Empty prompt rejected for on-device inference.');
       return false;
     }
 
     for (const content of request.contents) {
       // Returns false if the request contains multiple roles, eg a chat history.
       // TODO: remove this guard once LanguageModelMessage is supported.
       if (content.role !== 'user') {
+        logger.debug(
+          `Non-user role "${content.role}" rejected for on-device inference.`
+        );
         return false;
       }
 
@@ -159,6 +177,9 @@ export class ChromeAdapter {
             part.inlineData.mimeType
           ) === -1
         ) {
+          logger.debug(
+            `Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.`
+          );
           return false;
         }
       }