diff --git a/packages/vertexai/src/methods/count-tokens.test.ts b/packages/vertexai/src/methods/count-tokens.test.ts index a3d7c99b4ba..9eccbf702fe 100644 --- a/packages/vertexai/src/methods/count-tokens.test.ts +++ b/packages/vertexai/src/methods/count-tokens.test.ts @@ -45,7 +45,10 @@ describe('countTokens()', () => { restore(); }); it('total tokens', async () => { - const mockResponse = getMockResponse('unary-success-total-tokens.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-total-tokens.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); @@ -69,6 +72,7 @@ describe('countTokens()', () => { }); it('total tokens with modality details', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-detailed-token-response.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -96,6 +100,7 @@ describe('countTokens()', () => { }); it('total tokens no billable characters', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-no-billable-characters.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -120,7 +125,10 @@ describe('countTokens()', () => { ); }); it('model not found', async () => { - const mockResponse = getMockResponse('unary-failure-model-not-found.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-failure-model-not-found.json' + ); const mockFetch = stub(globalThis, 'fetch').resolves({ ok: false, status: 404, diff --git a/packages/vertexai/src/methods/generate-content.test.ts b/packages/vertexai/src/methods/generate-content.test.ts index 426bd5176db..1d15632f828 100644 --- a/packages/vertexai/src/methods/generate-content.test.ts +++ b/packages/vertexai/src/methods/generate-content.test.ts @@ -61,6 +61,7 @@ describe('generateContent()', () => { }); it('short response', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -84,7 +85,10 @@ describe('generateContent()', () => { ); }); it('long response', async () => { - const mockResponse = getMockResponse('unary-success-basic-reply-long.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-basic-reply-long.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); @@ -105,6 +109,7 @@ describe('generateContent()', () => { }); it('long response with token details', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-response-long-usage-metadata.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -138,7 +143,10 @@ describe('generateContent()', () => { ); }); it('citations', async () => { - const mockResponse = getMockResponse('unary-success-citations.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-citations.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); @@ -163,6 +171,7 @@ describe('generateContent()', () => { }); it('blocked prompt', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-prompt-blocked-safety.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -184,6 +193,7 @@ describe('generateContent()', () => { }); it('finishReason safety', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-finish-reason-safety.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -204,7 +214,10 @@ describe('generateContent()', () => { ); }); it('empty content', async () => { - const mockResponse = getMockResponse('unary-failure-empty-content.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-failure-empty-content.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); @@ -224,6 +237,7 @@ describe('generateContent()', () => { }); it('unknown enum - should ignore', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-unknown-enum-safety-ratings.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -244,7 +258,10 @@ describe('generateContent()', () => { ); }); it('image rejected (400)', async () => { - const mockResponse = getMockResponse('unary-failure-image-rejected.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-failure-image-rejected.json' + ); const mockFetch = stub(globalThis, 'fetch').resolves({ ok: false, status: 400, @@ -257,6 +274,7 @@ describe('generateContent()', () => { }); it('api not enabled (403)', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-firebasevertexai-api-not-enabled.json' ); const mockFetch = stub(globalThis, 'fetch').resolves({ diff --git a/packages/vertexai/src/models/generative-model.test.ts b/packages/vertexai/src/models/generative-model.test.ts index 26dff4e04c6..987f9b115e2 100644 --- a/packages/vertexai/src/models/generative-model.test.ts +++ b/packages/vertexai/src/models/generative-model.test.ts @@ -60,6 +60,7 @@ describe('GenerativeModel', () => { ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -89,6 +90,7 @@ describe('GenerativeModel', () => { }); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -129,6 +131,7 @@ describe('GenerativeModel', () => { ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -177,6 +180,7 @@ describe('GenerativeModel', () => { ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -206,6 +210,7 @@ describe('GenerativeModel', () => { }); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -239,6 +244,7 @@ describe('GenerativeModel', () => { ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-basic-reply-short.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -277,7 +283,10 @@ describe('GenerativeModel', () => { }); it('calls countTokens', async () => { const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' }); - const mockResponse = getMockResponse('unary-success-total-tokens.json'); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-total-tokens.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index c566a88e5b0..9e534f2195a 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -47,6 +47,7 @@ const fakeVertexAI: VertexAI = { describe('ImagenModel', () => { it('generateImages makes a request to predict with default parameters', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-generate-images-base64.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -90,6 +91,7 @@ describe('ImagenModel', () => { }); const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-generate-images-base64.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( @@ -133,6 +135,7 @@ describe('ImagenModel', () => { }); it('throws if prompt blocked', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-generate-images-prompt-blocked.json' ); diff --git a/packages/vertexai/src/requests/request.test.ts b/packages/vertexai/src/requests/request.test.ts index 499f06c848b..cd39a0f8ae5 100644 --- a/packages/vertexai/src/requests/request.test.ts +++ b/packages/vertexai/src/requests/request.test.ts @@ -414,6 +414,7 @@ describe('request methods', () => { }); it('Network error, API not enabled', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-firebasevertexai-api-not-enabled.json' ); const fetchStub = stub(globalThis, 'fetch').resolves( diff --git a/packages/vertexai/src/requests/response-helpers.test.ts b/packages/vertexai/src/requests/response-helpers.test.ts index 4cab8cde047..5371d040253 100644 --- a/packages/vertexai/src/requests/response-helpers.test.ts +++ b/packages/vertexai/src/requests/response-helpers.test.ts @@ -257,6 +257,7 @@ describe('response-helpers methods', () => { describe('handlePredictResponse', () => { it('returns base64 images', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-generate-images-base64.json' ) as Response; const res = await handlePredictResponse(mockResponse); @@ -270,6 +271,7 @@ describe('response-helpers methods', () => { }); it('returns GCS images', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-success-generate-images-gcs.json' ) as Response; const res = await handlePredictResponse(mockResponse); @@ -284,6 +286,7 @@ describe('response-helpers methods', () => { }); it('has filtered reason and no images if all images were filtered', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-generate-images-all-filtered.json' ) as Response; const res = await handlePredictResponse(mockResponse); @@ -294,6 +297,7 @@ describe('response-helpers methods', () => { }); it('has filtered reason and no images if all base64 images were filtered', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-generate-images-base64-some-filtered.json' ) as Response; const res = await handlePredictResponse(mockResponse); @@ -308,6 +312,7 @@ describe('response-helpers methods', () => { }); it('has filtered reason and no images if all GCS images were filtered', async () => { const mockResponse = getMockResponse( + 'vertexAI', 'unary-failure-generate-images-gcs-some-filtered.json' ) as Response; const res = await handlePredictResponse(mockResponse); diff --git a/packages/vertexai/src/requests/stream-reader.test.ts b/packages/vertexai/src/requests/stream-reader.test.ts index b68c2423066..bf959276a93 100644 --- a/packages/vertexai/src/requests/stream-reader.test.ts +++ b/packages/vertexai/src/requests/stream-reader.test.ts @@ -72,6 +72,7 @@ describe('processStream', () => { }); it('streaming response - short', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-basic-reply-short.txt' ); const result = processStream(fakeResponse as Response); @@ -83,6 +84,7 @@ describe('processStream', () => { }); it('streaming response - long', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-basic-reply-long.txt' ); const result = processStream(fakeResponse as Response); @@ -95,6 +97,7 @@ describe('processStream', () => { }); it('streaming response - long - big chunk', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-basic-reply-long.txt', 1e6 ); @@ -107,7 +110,10 @@ describe('processStream', () => { expect(aggregatedResponse.text()).to.include('to their owners.'); }); it('streaming response - utf8', async () => { - const fakeResponse = getMockResponseStreaming('streaming-success-utf8.txt'); + const fakeResponse = getMockResponseStreaming( + 'vertexAI', + 'streaming-success-utf8.txt' + ); const result = processStream(fakeResponse as Response); for await (const response of result.stream) { expect(response.text()).to.not.be.empty; @@ -118,6 +124,7 @@ describe('processStream', () => { }); it('streaming response - functioncall', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-function-call-short.txt' ); const result = processStream(fakeResponse as Response); @@ -141,6 +148,7 @@ describe('processStream', () => { }); it('candidate had finishReason', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-failure-finish-reason-safety.txt' ); const result = processStream(fakeResponse as Response); @@ -153,6 +161,7 @@ describe('processStream', () => { }); it('prompt was blocked', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-failure-prompt-blocked-safety.txt' ); const result = processStream(fakeResponse as Response); @@ -165,6 +174,7 @@ describe('processStream', () => { }); it('empty content', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-failure-empty-content.txt' ); const result = processStream(fakeResponse as Response); @@ -176,6 +186,7 @@ describe('processStream', () => { }); it('unknown enum - should ignore', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-unknown-safety-enum.txt' ); const result = processStream(fakeResponse as Response); @@ -187,6 +198,7 @@ describe('processStream', () => { }); it('recitation ending with a missing content field', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-failure-recitation-no-content.txt' ); const result = processStream(fakeResponse as Response); @@ -205,6 +217,7 @@ describe('processStream', () => { }); it('handles citations', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-citations.txt' ); const result = processStream(fakeResponse as Response); @@ -224,6 +237,7 @@ describe('processStream', () => { }); it('removes empty text parts', async () => { const fakeResponse = getMockResponseStreaming( + 'vertexAI', 'streaming-success-empty-text-part.txt' ); const result = processStream(fakeResponse as Response); diff --git a/packages/vertexai/test-utils/convert-mocks.ts b/packages/vertexai/test-utils/convert-mocks.ts index c306bec312f..851d6017b6d 100644 --- a/packages/vertexai/test-utils/convert-mocks.ts +++ b/packages/vertexai/test-utils/convert-mocks.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,42 +15,111 @@ * limitations under the License. */ +/* eslint-disable @typescript-eslint/no-require-imports */ +const { readdirSync, readFileSync, writeFileSync } = require('node:fs'); +const { join } = require('node:path'); + +const MOCK_RESPONSES_DIR_PATH = join( + __dirname, + 'vertexai-sdk-test-data', + 'mock-responses' +); +const MOCK_LOOKUP_OUTPUT_PATH = join(__dirname, 'mocks-lookup.ts'); + +type BackendName = 'vertexAI' | 'googleAI'; + +const mockDirs: Record = { + vertexAI: join(MOCK_RESPONSES_DIR_PATH, 'vertexai'), + // Note: the dirname is developerapi is legacy. It should be updated to googleai. + googleAI: join(MOCK_RESPONSES_DIR_PATH, 'developerapi') +}; + /** - * Converts mock text files into a js file that karma can read without - * using fs. + * Generates a JS file that exports maps from filenames to JSON mock responses (as strings) + * for each backend. + * + * This allows tests that run in a browser to access the mock responses without having to + * read from local disk and requiring 'fs'. */ +function generateMockLookupFile(): void { + console.log('Generating mock lookup file...'); + const vertexAIMockLookupText = generateMockLookup('vertexAI'); + const googleAIMockLookupText = generateMockLookup('googleAI'); -// eslint-disable-next-line @typescript-eslint/no-require-imports -const fs = require('fs'); -// eslint-disable-next-line @typescript-eslint/no-require-imports -const { join } = require('path'); + const fileText = ` +/** + * DO NOT EDIT - This file was generated by the packages/vertexai/test-utils/convert-mocks.ts script. + * + * These objects map mock response filenames to their JSON contents. + * + * Mock response files are pulled from https://github.com/FirebaseExtended/vertexai-sdk-test-data. + */ -const mockResponseDir = join( - __dirname, - 'vertexai-sdk-test-data/mock-responses/vertexai' -); +// Automatically generated at: ${new Date().toISOString()} + +${vertexAIMockLookupText} -async function main(): Promise { - const list = fs.readdirSync(mockResponseDir); +${googleAIMockLookupText} +`; + try { + writeFileSync(MOCK_LOOKUP_OUTPUT_PATH, fileText, 'utf-8'); + console.log( + `Successfully generated mock lookup file at: ${MOCK_LOOKUP_OUTPUT_PATH}` + ); + } catch (err) { + console.error( + `Error writing mock lookup file to ${MOCK_LOOKUP_OUTPUT_PATH}:`, + err + ); + process.exit(1); + } +} + +/** + * Given a directory that contains mock response files, reads through all the files, + * maps file names to file contents, and returns a string of typescript code + * that exports that map as an object. + */ +function generateMockLookup(backendName: BackendName): string { const lookup: Record = {}; - // eslint-disable-next-line guard-for-in - for (const fileName of list) { - const fullText = fs.readFileSync(join(mockResponseDir, fileName), 'utf-8'); - lookup[fileName] = fullText; + const mockDir = mockDirs[backendName]; + let mockFilenames: string[]; + + console.log( + `Processing mocks for "${backendName}" from directory: ${mockDir}` + ); + + try { + mockFilenames = readdirSync(mockDir); + } catch (err) { + console.error( + `Error reading directory ${mockDir} for ${backendName}:`, + err + ); + return `export const ${backendName}MocksLookup: Record = {};`; } - let fileText = `// Generated from mocks text files.`; - - fileText += '\n\n'; - fileText += `export const mocksLookup: Record = ${JSON.stringify( - lookup, - null, - 2 - )}`; - fileText += ';\n'; - fs.writeFileSync(join(__dirname, 'mocks-lookup.ts'), fileText, 'utf-8'); + + if (mockFilenames.length === 0) { + console.warn(`No .json files found in ${mockDir} for ${backendName}.`); + } + + for (const mockFilename of mockFilenames) { + const mockFilepath = `${mockDir}/${mockFilename}`; + try { + const fullText = readFileSync(mockFilepath, 'utf-8'); + lookup[mockFilename] = fullText; + } catch (err) { + console.error( + `Error reading mock file ${mockFilepath} for ${backendName}:`, + err + ); + } + } + + // Use JSON.stringify with indentation for readable output in the generated file + const lookupJsonString = JSON.stringify(lookup, null, 2); + + return `export const ${backendName}MocksLookup: Record = ${lookupJsonString};`; } -main().catch(e => { - console.error(e); - process.exit(1); -}); +generateMockLookupFile(); diff --git a/packages/vertexai/test-utils/mock-response.ts b/packages/vertexai/test-utils/mock-response.ts index 9b42c93427b..5128ddabe74 100644 --- a/packages/vertexai/test-utils/mock-response.ts +++ b/packages/vertexai/test-utils/mock-response.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,12 @@ * limitations under the License. */ -import { mocksLookup } from './mocks-lookup'; +import { vertexAIMocksLookup, googleAIMocksLookup } from './mocks-lookup'; + +const mockSetMaps: Record> = { + 'vertexAI': vertexAIMocksLookup, + 'googleAI': googleAIMocksLookup +}; /** * Mock native Response.body @@ -45,25 +50,33 @@ export function getChunkedStream( return stream; } + export function getMockResponseStreaming( + backendName: BackendName, filename: string, chunkLength: number = 20 ): Partial { - if (!(filename in mocksLookup)) { - throw Error(`Mock response file '${filename}' not found.`); + const mocksMap = mockSetMaps[backendName]; + if (!(filename in mocksMap)) { + throw Error(`${backendName} mock response file '${filename}' not found.`); } - const fullText = mocksLookup[filename]; + const fullText = mocksMap[filename]; return { body: getChunkedStream(fullText, chunkLength) }; } -export function getMockResponse(filename: string): Partial { +export function getMockResponse( + backendName: BackendName, + filename: string +): Partial { + const mocksLookup = mockSetMaps[backendName]; if (!(filename in mocksLookup)) { - throw Error(`Mock response file '${filename}' not found.`); + throw Error(`${backendName} mock response file '${filename}' not found.`); } const fullText = mocksLookup[filename]; + return { ok: true, json: () => Promise.resolve(JSON.parse(fullText))