Add support for GCS FileData upload in inference

Author: dlarocque

packages/rules-unit-testing/functions/yarn.lock

@@ -78,7 +78,7 @@
   dependencies:
     tslib "^2.1.0"
 
-"@google-cloud/firestore@^6.6.0":
+"@google-cloud/firestore@^6.8.0":
   version "6.8.0"
   resolved "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-6.8.0.tgz#d8c852844c381afaf62592796606c10e178400b5"
   integrity sha512-JRpk06SmZXLGz0pNx1x7yU3YhkUXheKgH5hbDZ4kMsdhtfV5qPLJLRI4wv69K0cZorIk+zTMOwptue7hizo0eA==
@@ -803,10 +803,10 @@ finalhandler@~1.1.2:
     statuses "~1.5.0"
     unpipe "~1.0.0"
 
-firebase-admin@11.10.1:
-  version "11.10.1"
-  resolved "https://registry.npmjs.org/firebase-admin/-/firebase-admin-11.10.1.tgz#5f0f83a44627e89938d350c5e4bbac76596c963e"
-  integrity sha512-atv1E6GbuvcvWaD3eHwrjeP5dAVs+EaHEJhu9CThMzPY6In8QYDiUR6tq5SwGl4SdA/GcAU0nhwWc/FSJsAzfQ==
+firebase-admin@11.11.1:
+  version "11.11.1"
+  resolved "https://registry.npmjs.org/firebase-admin/-/firebase-admin-11.11.1.tgz#6712923de70d218c9f514d73005d976d03339605"
+  integrity sha512-UyEbq+3u6jWzCYbUntv/HuJiTixwh36G1R9j0v71mSvGAx/YZEWEW7uSGLYxBYE6ckVRQoKMr40PYUEzrm/4dg==
   dependencies:
     "@fastify/busboy" "^1.2.1"
     "@firebase/database-compat" "^0.3.4"
@@ -817,7 +817,7 @@ [email protected]:
     node-forge "^1.3.1"
     uuid "^9.0.0"
   optionalDependencies:
-    "@google-cloud/firestore" "^6.6.0"
+    "@google-cloud/firestore" "^6.8.0"
     "@google-cloud/storage" "^6.9.5"
 
 [email protected]:

packages/vertexai/src/requests/request-helpers.test.ts

@@ -170,6 +170,33 @@ describe('request formatting methods', () => {
         ],
         systemInstruction: { role: 'system', parts: [{ text: 'be excited' }] }
       });
-    });
+    }),
+      it('formats fileData as part if provided as part', () => {
+        const result = formatGenerateContentInput([
+          'What is this?',
+          {
+            fileData: {
+              mimeType: 'image/jpeg',
+              fileUri: 'gs://sample.appspot.com/image.jpeg'
+            }
+          }
+        ]);
+        expect(result).to.be.deep.equal({
+          contents: [
+            {
+              role: 'user',
+              parts: [
+                {
+                  fileData: {
+                    mimeType: 'image/jpeg',
+                    fileUri: 'gs://sample.appspot.com/image.jpeg'
+                  }
+                },
+                { text: 'What is this?' }
+              ]
+            }
+          ]
+        });
+      });
   });
 });

packages/vertexai/src/types/content.ts

@@ -34,7 +34,8 @@ export type Part =
   | TextPart
   | InlineDataPart
   | FunctionCallPart
-  | FunctionResponsePart;
+  | FunctionResponsePart
+  | FileDataPart;
 
 /**
  * Content part interface if the part represents a text string.
@@ -101,6 +102,18 @@ export interface FunctionResponsePart {
   functionResponse: FunctionResponse;
 }
 
+/**
+ * Content part interface if the part represents {@link FileData}
+ * @public
+ */
+export interface FileDataPart {
+  text?: never;
+  inlineData?: never;
+  functionCall?: never;
+  functionResponse?: never;
+  fileData: FileData;
+}
+
 /**
  * A predicted [FunctionCall] returned from the model
  * that contains a string representing the [FunctionDeclaration.name]
@@ -137,3 +150,12 @@ export interface GenerativeContentBlob {
    */
   data: string;
 }
+
+/**
+ * Data pointing to a file uploaded on Google Cloud Storage.
+ * @public
+ */
+export interface FileData {
+  mimeType: string;
+  fileUri: string;
+}