From 47c48b32c9d248fd9840cffb92add3bb7ad2992c Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Mon, 24 Mar 2025 14:12:35 -0700 Subject: [PATCH 01/29] Define HybridParams --- packages/vertexai/src/types/ai.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/vertexai/src/types/ai.ts diff --git a/packages/vertexai/src/types/ai.ts b/packages/vertexai/src/types/ai.ts new file mode 100644 index 00000000000..30b20863373 --- /dev/null +++ b/packages/vertexai/src/types/ai.ts @@ -0,0 +1,21 @@ +/** + * Shims @types/dom-chromium-ai + * TODO: replace with @types/dom-chromium-ai once we can use es2020.intl. + */ +interface AILanguageModelCreateOptions { + topK?: number; + temperature?: number; +} + +export interface AILanguageModelCreateOptionsWithSystemPrompt + extends AILanguageModelCreateOptions { + systemPrompt?: string; + initialPrompts?: AILanguageModelPrompt[]; +} + +type AILanguageModelPromptRole = 'user' | 'assistant'; + +interface AILanguageModelPrompt { + role: AILanguageModelPromptRole; + content: string; +} From 4b2b2a6e0c3c6041e2a19c6f048d65bda618f02e Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Mon, 24 Mar 2025 16:25:35 -0700 Subject: [PATCH 02/29] Copy over most types from @types package --- packages/vertexai/src/types/ai.ts | 378 +++++++++++++++++++++++++++++- 1 file changed, 369 insertions(+), 9 deletions(-) diff --git a/packages/vertexai/src/types/ai.ts b/packages/vertexai/src/types/ai.ts index 30b20863373..86ce44b4d98 100644 --- a/packages/vertexai/src/types/ai.ts +++ b/packages/vertexai/src/types/ai.ts @@ -2,20 +2,380 @@ * Shims @types/dom-chromium-ai * TODO: replace with @types/dom-chromium-ai once we can use es2020.intl. */ +interface AI { + readonly languageModel: AILanguageModelFactory; + readonly summarizer: AISummarizerFactory; + readonly writer: AIWriterFactory; + readonly rewriter: AIRewriterFactory; + readonly translator: AITranslatorFactory; + readonly languageDetector: AILanguageDetectorFactory; +} + +interface AICreateMonitor extends EventTarget { + ondownloadprogress: ((this: AICreateMonitor, ev: DownloadProgressEvent) => any) | null; + + addEventListener( + type: K, + listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, + options?: boolean | AddEventListenerOptions, + ): void; + addEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions, + ): void; + removeEventListener( + type: K, + listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, + options?: boolean | EventListenerOptions, + ): void; + removeEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | EventListenerOptions, + ): void; +} + +interface DownloadProgressEvent extends Event { + readonly loaded: number; + readonly total: number; +} + +interface AICreateMonitorEventMap { + downloadprogress: DownloadProgressEvent; +} + +type AICreateMonitorCallback = (monitor: AICreateMonitor) => void; + +type AICapabilityAvailability = "readily" | "after-download" | "no"; + +// Language Model +// https://github.com/explainers-by-googlers/prompt-api/#full-api-surface-in-web-idl + +interface AILanguageModelFactory { + create( + options?: AILanguageModelCreateOptionsWithSystemPrompt | AILanguageModelCreateOptionsWithoutSystemPrompt, + ): Promise; + capabilities(): Promise; +} + interface AILanguageModelCreateOptions { - topK?: number; - temperature?: number; + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; + + topK?: number; + temperature?: number; } -export interface AILanguageModelCreateOptionsWithSystemPrompt - extends AILanguageModelCreateOptions { - systemPrompt?: string; - initialPrompts?: AILanguageModelPrompt[]; +export interface AILanguageModelCreateOptionsWithSystemPrompt extends AILanguageModelCreateOptions { + systemPrompt?: string; + initialPrompts?: AILanguageModelPrompt[]; } -type AILanguageModelPromptRole = 'user' | 'assistant'; +interface AILanguageModelCreateOptionsWithoutSystemPrompt extends AILanguageModelCreateOptions { + systemPrompt?: never; + initialPrompts?: + | [AILanguageModelSystemPrompt, ...AILanguageModelPrompt[]] + | AILanguageModelPrompt[]; +} + +type AILanguageModelPromptRole = "user" | "assistant"; +type AILanguageModelInitialPromptRole = "system" | AILanguageModelPromptRole; interface AILanguageModelPrompt { - role: AILanguageModelPromptRole; - content: string; + role: AILanguageModelPromptRole; + content: string; +} + +interface AILanguageModelInitialPrompt { + role: AILanguageModelInitialPromptRole; + content: string; +} + +interface AILanguageModelSystemPrompt extends AILanguageModelInitialPrompt { + role: "system"; +} + +type AILanguageModelPromptInput = string | AILanguageModelPrompt | AILanguageModelPrompt[]; + +interface AILanguageModel extends EventTarget { + prompt(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; + promptStreaming(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): ReadableStream; + + countPromptTokens(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; + readonly maxTokens: number; + readonly tokensSoFar: number; + readonly tokensLeft: number; + + readonly topK: number; + readonly temperature: number; + + oncontextoverflow: ((this: AILanguageModel, ev: Event) => any) | null; + + addEventListener( + type: K, + listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, + options?: boolean | AddEventListenerOptions, + ): void; + addEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions, + ): void; + removeEventListener( + type: K, + listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, + options?: boolean | EventListenerOptions, + ): void; + removeEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | EventListenerOptions, + ): void; + + clone(options?: AILanguageModelCloneOptions): Promise; + destroy(): void; +} + +interface AILanguageModelEventMap { + contextoverflow: Event; +} + +interface AILanguageModelPromptOptions { + signal?: AbortSignal; +} + +interface AILanguageModelCloneOptions { + signal?: AbortSignal; +} + +interface AILanguageModelCapabilities { + readonly available: AICapabilityAvailability; + languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; + + readonly defaultTopK: number | null; + readonly maxTopK: number | null; + readonly defaultTemperature: number | null; + readonly maxTemperature: number | null; +} + +// Summarizer +// https://github.com/explainers-by-googlers/writing-assistance-apis/#full-api-surface-in-web-idl + +interface AISummarizerFactory { + create(options?: AISummarizerCreateOptions): Promise; + capabilities(): Promise; +} + +interface AISummarizerCreateOptions { + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; + + sharedContext?: string; + type?: AISummarizerType; + format?: AISummarizerFormat; + length?: AISummarizerLength; +} + +type AISummarizerType = "tl;dr" | "key-points" | "teaser" | "headline"; +type AISummarizerFormat = "plain-text" | "markdown"; +type AISummarizerLength = "short" | "medium" | "long"; + +interface AISummarizer { + summarize(input: string, options?: AISummarizerSummarizeOptions): Promise; + summarizeStreaming(input: string, options?: AISummarizerSummarizeOptions): ReadableStream; + + readonly sharedContext: string; + readonly type: AISummarizerType; + readonly format: AISummarizerFormat; + readonly length: AISummarizerLength; + + destroy(): void; +} + +interface AISummarizerSummarizeOptions { + signal?: AbortSignal; + context?: string; +} + +interface AISummarizerCapabilities { + readonly available: AICapabilityAvailability; + + supportsType(type: AISummarizerType): AICapabilityAvailability; + supportsFormat(format: AISummarizerFormat): AICapabilityAvailability; + supportsLength(length: AISummarizerLength): AICapabilityAvailability; + + languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; +} + +// Writer +// https://github.com/explainers-by-googlers/writing-assistance-apis/#full-api-surface-in-web-idl + +interface AIWriterFactory { + create(options?: AIWriterCreateOptions): Promise; + capabilities(): Promise; +} + +interface AIWriterCreateOptions { + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; + + sharedContext?: string; + tone?: AIWriterTone; + format?: AIWriterFormat; + length?: AIWriterLength; +} + +type AIWriterTone = "formal" | "neutral" | "casual"; +type AIWriterFormat = "plain-text" | "markdown"; +type AIWriterLength = "short" | "medium" | "long"; + +interface AIWriter { + write(writingTask: string, options?: AIWriterWriteOptions): Promise; + writeStreaming(writingTask: string, options?: AIWriterWriteOptions): ReadableStream; + + readonly sharedContext: string; + readonly tone: AIWriterTone; + readonly format: AIWriterFormat; + readonly length: AIWriterLength; + + destroy(): void; +} + +interface AIWriterWriteOptions { + signal?: AbortSignal; + context?: string; +} + +interface AIWriterCapabilities { + readonly available: AICapabilityAvailability; + + supportsTone(tone: AIWriterTone): AICapabilityAvailability; + supportsFormat(format: AIWriterFormat): AICapabilityAvailability; + supportsLength(length: AIWriterLength): AICapabilityAvailability; + + languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; +} + +// Rewriter +// https://github.com/explainers-by-googlers/writing-assistance-apis/#full-api-surface-in-web-idl + +interface AIRewriterFactory { + create(options?: AIRewriterCreateOptions): Promise; + capabilities(): Promise; +} + +interface AIRewriterCreateOptions { + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; + + sharedContext?: string; + tone?: AIRewriterTone; + format?: AIRewriterFormat; + length?: AIRewriterLength; +} + +type AIRewriterTone = "as-is" | "more-formal" | "more-casual"; +type AIRewriterFormat = "as-is" | "plain-text" | "markdown"; +type AIRewriterLength = "as-is" | "shorter" | "longer"; + +interface AIRewriter { + rewrite(input: string, options?: AIRewriterRewriteOptions): Promise; + rewriteStreaming(input: string, options?: AIRewriterRewriteOptions): ReadableStream; + + readonly sharedContext: string; + readonly tone: AIRewriterTone; + readonly format: AIRewriterFormat; + readonly length: AIRewriterLength; + + destroy(): void; +} + +interface AIRewriterRewriteOptions { + signal?: AbortSignal; + context?: string; +} + +interface AIRewriterCapabilities { + readonly available: AICapabilityAvailability; + + supportsTone(tone: AIRewriterTone): AICapabilityAvailability; + supportsFormat(format: AIRewriterFormat): AICapabilityAvailability; + supportsLength(length: AIRewriterLength): AICapabilityAvailability; + + languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; } + +// Translator +// https://github.com/WICG/translation-api?tab=readme-ov-file#full-api-surface-in-web-idl + +interface AITranslatorFactory { + create(options: AITranslatorCreateOptions): Promise; + capabilities(): Promise; +} + +interface AITranslator { + translate(input: string, options?: AITranslatorTranslateOptions): Promise; + translateStreaming(input: string, options?: AITranslatorTranslateOptions): ReadableStream; + + readonly sourceLanguage: Intl.UnicodeBCP47LocaleIdentifier; + readonly targetLanguage: Intl.UnicodeBCP47LocaleIdentifier; + + destroy(): void; +} + +interface AITranslatorCapabilities { + readonly available: AICapabilityAvailability; + + languagePairAvailable( + sourceLanguage: Intl.UnicodeBCP47LocaleIdentifier, + targetLanguage: Intl.UnicodeBCP47LocaleIdentifier, + ): AICapabilityAvailability; +} + +interface AITranslatorCreateOptions { + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; + + sourceLanguage: Intl.UnicodeBCP47LocaleIdentifier; + targetLanguage: Intl.UnicodeBCP47LocaleIdentifier; +} + +interface AITranslatorTranslateOptions { + signal?: AbortSignal; +} + +// Language detector +// https://github.com/WICG/translation-api?tab=readme-ov-file#full-api-surface-in-web-idl + +interface AILanguageDetectorFactory { + create(options?: AILanguageDetectorCreateOptions): Promise; + capabilities(): Promise; +} + +interface AILanguageDetector { + detect(input: string, options?: AILanguageDetectorDetectOptions): Promise; + + destroy(): void; +} + +interface AILanguageDetectorCapabilities { + readonly available: AICapabilityAvailability; + + languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; +} + +interface AILanguageDetectorCreateOptions { + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; +} + +interface AILanguageDetectorDetectOptions { + signal?: AbortSignal; +} + +interface LanguageDetectionResult { + /** null represents unknown language */ + detectedLanguage: Intl.UnicodeBCP47LocaleIdentifier | null; + confidence: number; +} \ No newline at end of file From ddcae13be58e330658c8ad62235a6f69450ec1fa Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Mon, 24 Mar 2025 16:34:36 -0700 Subject: [PATCH 03/29] Document HybridParams and InferenceMode --- packages/vertexai/src/types/ai.ts | 381 ------------------------------ 1 file changed, 381 deletions(-) delete mode 100644 packages/vertexai/src/types/ai.ts diff --git a/packages/vertexai/src/types/ai.ts b/packages/vertexai/src/types/ai.ts deleted file mode 100644 index 86ce44b4d98..00000000000 --- a/packages/vertexai/src/types/ai.ts +++ /dev/null @@ -1,381 +0,0 @@ -/** - * Shims @types/dom-chromium-ai - * TODO: replace with @types/dom-chromium-ai once we can use es2020.intl. - */ -interface AI { - readonly languageModel: AILanguageModelFactory; - readonly summarizer: AISummarizerFactory; - readonly writer: AIWriterFactory; - readonly rewriter: AIRewriterFactory; - readonly translator: AITranslatorFactory; - readonly languageDetector: AILanguageDetectorFactory; -} - -interface AICreateMonitor extends EventTarget { - ondownloadprogress: ((this: AICreateMonitor, ev: DownloadProgressEvent) => any) | null; - - addEventListener( - type: K, - listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, - options?: boolean | AddEventListenerOptions, - ): void; - addEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | AddEventListenerOptions, - ): void; - removeEventListener( - type: K, - listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, - options?: boolean | EventListenerOptions, - ): void; - removeEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | EventListenerOptions, - ): void; -} - -interface DownloadProgressEvent extends Event { - readonly loaded: number; - readonly total: number; -} - -interface AICreateMonitorEventMap { - downloadprogress: DownloadProgressEvent; -} - -type AICreateMonitorCallback = (monitor: AICreateMonitor) => void; - -type AICapabilityAvailability = "readily" | "after-download" | "no"; - -// Language Model -// https://github.com/explainers-by-googlers/prompt-api/#full-api-surface-in-web-idl - -interface AILanguageModelFactory { - create( - options?: AILanguageModelCreateOptionsWithSystemPrompt | AILanguageModelCreateOptionsWithoutSystemPrompt, - ): Promise; - capabilities(): Promise; -} - -interface AILanguageModelCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; - - topK?: number; - temperature?: number; -} - -export interface AILanguageModelCreateOptionsWithSystemPrompt extends AILanguageModelCreateOptions { - systemPrompt?: string; - initialPrompts?: AILanguageModelPrompt[]; -} - -interface AILanguageModelCreateOptionsWithoutSystemPrompt extends AILanguageModelCreateOptions { - systemPrompt?: never; - initialPrompts?: - | [AILanguageModelSystemPrompt, ...AILanguageModelPrompt[]] - | AILanguageModelPrompt[]; -} - -type AILanguageModelPromptRole = "user" | "assistant"; -type AILanguageModelInitialPromptRole = "system" | AILanguageModelPromptRole; - -interface AILanguageModelPrompt { - role: AILanguageModelPromptRole; - content: string; -} - -interface AILanguageModelInitialPrompt { - role: AILanguageModelInitialPromptRole; - content: string; -} - -interface AILanguageModelSystemPrompt extends AILanguageModelInitialPrompt { - role: "system"; -} - -type AILanguageModelPromptInput = string | AILanguageModelPrompt | AILanguageModelPrompt[]; - -interface AILanguageModel extends EventTarget { - prompt(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; - promptStreaming(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): ReadableStream; - - countPromptTokens(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; - readonly maxTokens: number; - readonly tokensSoFar: number; - readonly tokensLeft: number; - - readonly topK: number; - readonly temperature: number; - - oncontextoverflow: ((this: AILanguageModel, ev: Event) => any) | null; - - addEventListener( - type: K, - listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, - options?: boolean | AddEventListenerOptions, - ): void; - addEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | AddEventListenerOptions, - ): void; - removeEventListener( - type: K, - listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, - options?: boolean | EventListenerOptions, - ): void; - removeEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | EventListenerOptions, - ): void; - - clone(options?: AILanguageModelCloneOptions): Promise; - destroy(): void; -} - -interface AILanguageModelEventMap { - contextoverflow: Event; -} - -interface AILanguageModelPromptOptions { - signal?: AbortSignal; -} - -interface AILanguageModelCloneOptions { - signal?: AbortSignal; -} - -interface AILanguageModelCapabilities { - readonly available: AICapabilityAvailability; - languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; - - readonly defaultTopK: number | null; - readonly maxTopK: number | null; - readonly defaultTemperature: number | null; - readonly maxTemperature: number | null; -} - -// Summarizer -// https://github.com/explainers-by-googlers/writing-assistance-apis/#full-api-surface-in-web-idl - -interface AISummarizerFactory { - create(options?: AISummarizerCreateOptions): Promise; - capabilities(): Promise; -} - -interface AISummarizerCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; - - sharedContext?: string; - type?: AISummarizerType; - format?: AISummarizerFormat; - length?: AISummarizerLength; -} - -type AISummarizerType = "tl;dr" | "key-points" | "teaser" | "headline"; -type AISummarizerFormat = "plain-text" | "markdown"; -type AISummarizerLength = "short" | "medium" | "long"; - -interface AISummarizer { - summarize(input: string, options?: AISummarizerSummarizeOptions): Promise; - summarizeStreaming(input: string, options?: AISummarizerSummarizeOptions): ReadableStream; - - readonly sharedContext: string; - readonly type: AISummarizerType; - readonly format: AISummarizerFormat; - readonly length: AISummarizerLength; - - destroy(): void; -} - -interface AISummarizerSummarizeOptions { - signal?: AbortSignal; - context?: string; -} - -interface AISummarizerCapabilities { - readonly available: AICapabilityAvailability; - - supportsType(type: AISummarizerType): AICapabilityAvailability; - supportsFormat(format: AISummarizerFormat): AICapabilityAvailability; - supportsLength(length: AISummarizerLength): AICapabilityAvailability; - - languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; -} - -// Writer -// https://github.com/explainers-by-googlers/writing-assistance-apis/#full-api-surface-in-web-idl - -interface AIWriterFactory { - create(options?: AIWriterCreateOptions): Promise; - capabilities(): Promise; -} - -interface AIWriterCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; - - sharedContext?: string; - tone?: AIWriterTone; - format?: AIWriterFormat; - length?: AIWriterLength; -} - -type AIWriterTone = "formal" | "neutral" | "casual"; -type AIWriterFormat = "plain-text" | "markdown"; -type AIWriterLength = "short" | "medium" | "long"; - -interface AIWriter { - write(writingTask: string, options?: AIWriterWriteOptions): Promise; - writeStreaming(writingTask: string, options?: AIWriterWriteOptions): ReadableStream; - - readonly sharedContext: string; - readonly tone: AIWriterTone; - readonly format: AIWriterFormat; - readonly length: AIWriterLength; - - destroy(): void; -} - -interface AIWriterWriteOptions { - signal?: AbortSignal; - context?: string; -} - -interface AIWriterCapabilities { - readonly available: AICapabilityAvailability; - - supportsTone(tone: AIWriterTone): AICapabilityAvailability; - supportsFormat(format: AIWriterFormat): AICapabilityAvailability; - supportsLength(length: AIWriterLength): AICapabilityAvailability; - - languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; -} - -// Rewriter -// https://github.com/explainers-by-googlers/writing-assistance-apis/#full-api-surface-in-web-idl - -interface AIRewriterFactory { - create(options?: AIRewriterCreateOptions): Promise; - capabilities(): Promise; -} - -interface AIRewriterCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; - - sharedContext?: string; - tone?: AIRewriterTone; - format?: AIRewriterFormat; - length?: AIRewriterLength; -} - -type AIRewriterTone = "as-is" | "more-formal" | "more-casual"; -type AIRewriterFormat = "as-is" | "plain-text" | "markdown"; -type AIRewriterLength = "as-is" | "shorter" | "longer"; - -interface AIRewriter { - rewrite(input: string, options?: AIRewriterRewriteOptions): Promise; - rewriteStreaming(input: string, options?: AIRewriterRewriteOptions): ReadableStream; - - readonly sharedContext: string; - readonly tone: AIRewriterTone; - readonly format: AIRewriterFormat; - readonly length: AIRewriterLength; - - destroy(): void; -} - -interface AIRewriterRewriteOptions { - signal?: AbortSignal; - context?: string; -} - -interface AIRewriterCapabilities { - readonly available: AICapabilityAvailability; - - supportsTone(tone: AIRewriterTone): AICapabilityAvailability; - supportsFormat(format: AIRewriterFormat): AICapabilityAvailability; - supportsLength(length: AIRewriterLength): AICapabilityAvailability; - - languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; -} - -// Translator -// https://github.com/WICG/translation-api?tab=readme-ov-file#full-api-surface-in-web-idl - -interface AITranslatorFactory { - create(options: AITranslatorCreateOptions): Promise; - capabilities(): Promise; -} - -interface AITranslator { - translate(input: string, options?: AITranslatorTranslateOptions): Promise; - translateStreaming(input: string, options?: AITranslatorTranslateOptions): ReadableStream; - - readonly sourceLanguage: Intl.UnicodeBCP47LocaleIdentifier; - readonly targetLanguage: Intl.UnicodeBCP47LocaleIdentifier; - - destroy(): void; -} - -interface AITranslatorCapabilities { - readonly available: AICapabilityAvailability; - - languagePairAvailable( - sourceLanguage: Intl.UnicodeBCP47LocaleIdentifier, - targetLanguage: Intl.UnicodeBCP47LocaleIdentifier, - ): AICapabilityAvailability; -} - -interface AITranslatorCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; - - sourceLanguage: Intl.UnicodeBCP47LocaleIdentifier; - targetLanguage: Intl.UnicodeBCP47LocaleIdentifier; -} - -interface AITranslatorTranslateOptions { - signal?: AbortSignal; -} - -// Language detector -// https://github.com/WICG/translation-api?tab=readme-ov-file#full-api-surface-in-web-idl - -interface AILanguageDetectorFactory { - create(options?: AILanguageDetectorCreateOptions): Promise; - capabilities(): Promise; -} - -interface AILanguageDetector { - detect(input: string, options?: AILanguageDetectorDetectOptions): Promise; - - destroy(): void; -} - -interface AILanguageDetectorCapabilities { - readonly available: AICapabilityAvailability; - - languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; -} - -interface AILanguageDetectorCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; -} - -interface AILanguageDetectorDetectOptions { - signal?: AbortSignal; -} - -interface LanguageDetectionResult { - /** null represents unknown language */ - detectedLanguage: Intl.UnicodeBCP47LocaleIdentifier | null; - confidence: number; -} \ No newline at end of file From c46825b1b08d3a0efecc5657ffe17d7e75477977 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 25 Mar 2025 09:09:10 -0700 Subject: [PATCH 04/29] Assert HybridParams sets the model name --- packages/vertexai/src/api.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index 7b25dbdf9e9..d49e5dcc15d 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -14,7 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ImagenModelParams, ModelParams, VertexAIErrorCode } from './types'; +import { + ImagenModelParams, + ModelParams, + VertexAIErrorCode +} from './types'; import { VertexAIError } from './errors'; import { ImagenModel, getGenerativeModel, getImagenModel } from './api'; import { expect } from 'chai'; From 2d5897655408c40081c98a02780d9735d062e483 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Wed, 26 Mar 2025 16:56:42 -0700 Subject: [PATCH 05/29] Use dom-chromium-ai package directly --- packages/vertexai/package.json | 1 + repo-scripts/changelog-generator/tsconfig.json | 3 ++- yarn.lock | 13 ++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/vertexai/package.json b/packages/vertexai/package.json index 9faf562a535..076b6a1bc4a 100644 --- a/packages/vertexai/package.json +++ b/packages/vertexai/package.json @@ -58,6 +58,7 @@ "devDependencies": { "@firebase/app": "0.11.4", "@rollup/plugin-json": "6.1.0", + "@types/dom-chromium-ai": "0.0.6", "rollup": "2.79.2", "rollup-plugin-replace": "2.2.0", "rollup-plugin-typescript2": "0.36.0", diff --git a/repo-scripts/changelog-generator/tsconfig.json b/repo-scripts/changelog-generator/tsconfig.json index 38bdb7035e4..cffe622284d 100644 --- a/repo-scripts/changelog-generator/tsconfig.json +++ b/repo-scripts/changelog-generator/tsconfig.json @@ -3,7 +3,8 @@ "strict": true, "outDir": "dist", "lib": [ - "ESNext" + "ESNext", + "dom" ], "module": "CommonJS", "moduleResolution": "node", diff --git a/yarn.lock b/yarn.lock index 51ede769d03..fbfb49f23e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2938,17 +2938,20 @@ "@types/node" "*" "@types/cors@^2.8.12": - version "2.8.17" - resolved "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz#5d718a5e494a8166f569d986794e49c48b216b2b" - integrity sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA== - dependencies: - "@types/node" "*" + version "2.8.12" + resolved "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz" + integrity sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw== "@types/deep-eql@*": version "4.0.2" resolved "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz#334311971d3a07121e7eb91b684a605e7eea9cbd" integrity sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw== +"@types/dom-chromium-ai@0.0.6": + version "0.0.6" + resolved "https://registry.npmjs.org/@types/dom-chromium-ai/-/dom-chromium-ai-0.0.6.tgz#0c9e5712d8db3d26586cd9f175001b509cd2e514" + integrity sha512-/jUGe9a3BLzsjjg18Olk/Ul64PZ0P4aw8uNxrXeXVTni5PSxyCfyhHb4UohsXNVByOnwYGzlqUcb3vYKVsG4mg== + "@types/eslint-scope@^3.7.7": version "3.7.7" resolved "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz#3108bd5f18b0cdb277c867b3dd449c9ed7079ac5" From 35f03f6d9626e35fd54091e9259b22ff72e677e9 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 1 Apr 2025 17:44:30 -0700 Subject: [PATCH 06/29] Use type for inference mode and update docs --- docs-devsite/vertexai.hybridparams.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs-devsite/vertexai.hybridparams.md b/docs-devsite/vertexai.hybridparams.md index cf847b40fa7..c9b053b09f4 100644 --- a/docs-devsite/vertexai.hybridparams.md +++ b/docs-devsite/vertexai.hybridparams.md @@ -36,6 +36,16 @@ Optional. Specifies advanced params for in-cloud inference. inCloudParams?: ModelParams; ``` +## HybridParams.inCloudParams + +Optional. Specifies advanced params for in-cloud inference. + +Signature: + +```typescript +inCloudParams?: ModelParams; +``` + ## HybridParams.mode Specifies on-device or in-cloud inference. Defaults to prefer on-device. From c1fa837a6e5212f583bd7937b7be20b342625585 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 3 Apr 2025 13:58:15 -0700 Subject: [PATCH 07/29] Run yarn format --- packages/vertexai/src/api.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index d49e5dcc15d..7b25dbdf9e9 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -14,11 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - ImagenModelParams, - ModelParams, - VertexAIErrorCode -} from './types'; +import { ImagenModelParams, ModelParams, VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; import { ImagenModel, getGenerativeModel, getImagenModel } from './api'; import { expect } from 'chai'; From b37b23740368fd5d7a18de45ea21ffd5ccc9f4f5 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Mon, 14 Apr 2025 16:07:16 -0700 Subject: [PATCH 08/29] rebased and updated deps --- package.json | 15 +++++--- packages/vertexai/package.json | 1 + scripts/release/utils/workspace.ts | 6 ++-- yarn.lock | 58 ++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 30b6b09a003..91a1c9316de 100644 --- a/package.json +++ b/package.json @@ -56,11 +56,16 @@ "type": "git", "url": "git+https://github.com/firebase/firebase-js-sdk.git" }, - "workspaces": [ - "packages/*", - "integration/*", - "repo-scripts/*" - ], + "workspaces": { + "packages": [ + "packages/*", + "integration/*", + "repo-scripts/*" + ], + "nohoist": [ + "**/vertexai/@types/dom-chromium-ai" + ] + }, "devDependencies": { "@babel/core": "7.26.8", "@babel/plugin-transform-modules-commonjs": "7.26.3", diff --git a/packages/vertexai/package.json b/packages/vertexai/package.json index 076b6a1bc4a..d75ad262aac 100644 --- a/packages/vertexai/package.json +++ b/packages/vertexai/package.json @@ -52,6 +52,7 @@ "@firebase/component": "0.6.13", "@firebase/logger": "0.4.4", "@firebase/util": "1.11.0", + "@types/dom-chromium-ai": "0.0.6", "tslib": "^2.1.0" }, "license": "Apache-2.0", diff --git a/scripts/release/utils/workspace.ts b/scripts/release/utils/workspace.ts index 077e3124b2a..3dba8651675 100644 --- a/scripts/release/utils/workspace.ts +++ b/scripts/release/utils/workspace.ts @@ -27,8 +27,10 @@ const writeFile = promisify(_writeFile); const { workspaces: rawWorkspaces -}: { workspaces: string[] } = require(`${root}/package.json`); -const workspaces = rawWorkspaces.map(workspace => `${root}/${workspace}`); +}: { workspaces: { packages: string[] } } = require(`${root}/package.json`); +const workspaces = rawWorkspaces.packages.map( + workspace => `${root}/${workspace}` +); export function mapWorkspaceToPackages( workspaces: string[] diff --git a/yarn.lock b/yarn.lock index fbfb49f23e0..6c92b4aeb35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1299,6 +1299,28 @@ resolved "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2" integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q== +"@firebase/app@0.11.4": + version "0.11.4" + resolved "https://registry.npmjs.org/@firebase/app/-/app-0.11.4.tgz#93f2637ed5b8dbc1ddf879c727d66a00c656c959" + integrity sha512-GPREsZjfSaHzwyC6cI/Cqvzf6zxqMzya+25tSpUstdqC2w0IdfxEfOMjfdW7bDfVEf4Rb4Nb6gfoOAgVSp4c4g== + dependencies: + "@firebase/component" "0.6.13" + "@firebase/logger" "0.4.4" + "@firebase/util" "1.11.0" + idb "7.1.1" + tslib "^2.1.0" + +"@firebase/vertexai@1.2.0": + version "1.2.0" + resolved "https://registry.npmjs.org/@firebase/vertexai/-/vertexai-1.2.0.tgz#8f8a4ae75284c3067c0a06c7d0bef922571e6dd7" + integrity sha512-WUYIzFpOipjFXT2i0hT26wivJoIximizQptVs3KAxFAqbVlO8sjKPsMkgz0bh+tdKlqP4SUDda71fMUZXUKHgA== + dependencies: + "@firebase/app-check-interop-types" "0.3.3" + "@firebase/component" "0.6.13" + "@firebase/logger" "0.4.4" + "@firebase/util" "1.11.0" + tslib "^2.1.0" + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -3166,6 +3188,13 @@ dependencies: undici-types "~5.26.4" +"@types/node@18.19.83": + version "18.19.83" + resolved "https://registry.npmjs.org/@types/node/-/node-18.19.83.tgz#44d302cd09364640bdd45d001bc75e596f7da920" + integrity sha512-D69JeR5SfFS5H6FLbUaS0vE4r1dGhmMBbG4Ed6BNS4wkDK8GZjsdCShT5LCN59vOHEUHnFCY9J4aclXlIphMkA== + dependencies: + undici-types "~5.26.4" + "@types/node@^12.7.1": version "12.20.55" resolved "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240" @@ -16931,6 +16960,35 @@ webpack@^5: watchpack "^2.4.1" webpack-sources "^3.2.3" +webpack@5.98.0: + version "5.98.0" + resolved "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz#44ae19a8f2ba97537978246072fb89d10d1fbd17" + integrity sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA== + dependencies: + "@types/eslint-scope" "^3.7.7" + "@types/estree" "^1.0.6" + "@webassemblyjs/ast" "^1.14.1" + "@webassemblyjs/wasm-edit" "^1.14.1" + "@webassemblyjs/wasm-parser" "^1.14.1" + acorn "^8.14.0" + browserslist "^4.24.0" + chrome-trace-event "^1.0.2" + enhanced-resolve "^5.17.1" + es-module-lexer "^1.2.1" + eslint-scope "5.1.1" + events "^3.2.0" + glob-to-regexp "^0.4.1" + graceful-fs "^4.2.11" + json-parse-even-better-errors "^2.3.1" + loader-runner "^4.2.0" + mime-types "^2.1.27" + neo-async "^2.6.2" + schema-utils "^4.3.0" + tapable "^2.1.1" + terser-webpack-plugin "^5.3.11" + watchpack "^2.4.1" + webpack-sources "^3.2.3" + websocket-driver@>=0.5.1: version "0.7.4" resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz#89ad5295bbf64b480abcba31e4953aca706f5760" From 97958ab3b5c9dd9810179020a2955c8e8c61a8ad Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Mon, 14 Apr 2025 16:11:47 -0700 Subject: [PATCH 09/29] revert deps --- package.json | 6 +-- packages/vertexai/package.json | 6 +-- yarn.lock | 87 ++++------------------------------ 3 files changed, 15 insertions(+), 84 deletions(-) diff --git a/package.json b/package.json index 91a1c9316de..43589733b54 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "@types/long": "4.0.2", "@types/mocha": "9.1.1", "@types/mz": "2.7.8", - "@types/node": "18.19.83", + "@types/node": "18.19.75", "@types/request": "2.48.12", "@types/sinon": "9.0.11", "@types/sinon-chai": "3.2.12", @@ -144,7 +144,7 @@ "nyc": "15.1.0", "ora": "5.4.1", "patch-package": "7.0.2", - "playwright": "1.51.1", + "playwright": "1.50.1", "postinstall-postinstall": "2.1.0", "prettier": "2.8.8", "protractor": "5.4.2", @@ -163,7 +163,7 @@ "typedoc": "0.16.11", "typescript": "5.5.4", "watch": "1.0.2", - "webpack": "5.98.0", + "webpack": "5.97.1", "yargs": "17.7.2" } } diff --git a/packages/vertexai/package.json b/packages/vertexai/package.json index d75ad262aac..c53cb5d3f48 100644 --- a/packages/vertexai/package.json +++ b/packages/vertexai/package.json @@ -1,6 +1,6 @@ { "name": "@firebase/vertexai", - "version": "1.2.1", + "version": "1.2.0", "description": "A Firebase SDK for VertexAI", "author": "Firebase (https://firebase.google.com/)", "engines": { @@ -57,7 +57,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@firebase/app": "0.11.4", + "@firebase/app": "0.11.3", "@rollup/plugin-json": "6.1.0", "@types/dom-chromium-ai": "0.0.6", "rollup": "2.79.2", @@ -80,4 +80,4 @@ ], "reportDir": "./coverage/node" } -} +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 6c92b4aeb35..adbe420e7b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1299,28 +1299,6 @@ resolved "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2" integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q== -"@firebase/app@0.11.4": - version "0.11.4" - resolved "https://registry.npmjs.org/@firebase/app/-/app-0.11.4.tgz#93f2637ed5b8dbc1ddf879c727d66a00c656c959" - integrity sha512-GPREsZjfSaHzwyC6cI/Cqvzf6zxqMzya+25tSpUstdqC2w0IdfxEfOMjfdW7bDfVEf4Rb4Nb6gfoOAgVSp4c4g== - dependencies: - "@firebase/component" "0.6.13" - "@firebase/logger" "0.4.4" - "@firebase/util" "1.11.0" - idb "7.1.1" - tslib "^2.1.0" - -"@firebase/vertexai@1.2.0": - version "1.2.0" - resolved "https://registry.npmjs.org/@firebase/vertexai/-/vertexai-1.2.0.tgz#8f8a4ae75284c3067c0a06c7d0bef922571e6dd7" - integrity sha512-WUYIzFpOipjFXT2i0hT26wivJoIximizQptVs3KAxFAqbVlO8sjKPsMkgz0bh+tdKlqP4SUDda71fMUZXUKHgA== - dependencies: - "@firebase/app-check-interop-types" "0.3.3" - "@firebase/component" "0.6.13" - "@firebase/logger" "0.4.4" - "@firebase/util" "1.11.0" - tslib "^2.1.0" - "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -3188,13 +3166,6 @@ dependencies: undici-types "~5.26.4" -"@types/node@18.19.83": - version "18.19.83" - resolved "https://registry.npmjs.org/@types/node/-/node-18.19.83.tgz#44d302cd09364640bdd45d001bc75e596f7da920" - integrity sha512-D69JeR5SfFS5H6FLbUaS0vE4r1dGhmMBbG4Ed6BNS4wkDK8GZjsdCShT5LCN59vOHEUHnFCY9J4aclXlIphMkA== - dependencies: - undici-types "~5.26.4" - "@types/node@^12.7.1": version "12.20.55" resolved "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240" @@ -13244,17 +13215,17 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" -playwright-core@1.51.1: - version "1.51.1" - resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz#d57f0393e02416f32a47cf82b27533656a8acce1" - integrity sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw== +playwright-core@1.50.1: + version "1.50.1" + resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.1.tgz#6a0484f1f1c939168f40f0ab3828c4a1592c4504" + integrity sha512-ra9fsNWayuYumt+NiM069M6OkcRb1FZSK8bgi66AtpFoWkg2+y0bJSNmkFrWhMbEBbVKC/EruAHH3g0zmtwGmQ== -playwright@1.51.1: - version "1.51.1" - resolved "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz#ae1467ee318083968ad28d6990db59f47a55390f" - integrity sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw== +playwright@1.50.1: + version "1.50.1" + resolved "https://registry.npmjs.org/playwright/-/playwright-1.50.1.tgz#2f93216511d65404f676395bfb97b41aa052b180" + integrity sha512-G8rwsOQJ63XG6BbKj2w5rHeavFjy5zynBA9zsJMMtBoe/Uf757oG12NXz6e6OirF7RCrTVAKFXbLmn1RbL7Qaw== dependencies: - playwright-core "1.51.1" + playwright-core "1.50.1" optionalDependencies: fsevents "2.3.2" @@ -15772,17 +15743,6 @@ terser-webpack-plugin@^5.3.10: serialize-javascript "^6.0.2" terser "^5.31.1" -terser-webpack-plugin@^5.3.11: - version "5.3.14" - resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz#9031d48e57ab27567f02ace85c7d690db66c3e06" - integrity sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw== - dependencies: - "@jridgewell/trace-mapping" "^0.3.25" - jest-worker "^27.4.5" - schema-utils "^4.3.0" - serialize-javascript "^6.0.2" - terser "^5.31.1" - terser@5.37.0, terser@^5.17.4, terser@^5.31.1: version "5.37.0" resolved "https://registry.npmjs.org/terser/-/terser-5.37.0.tgz#38aa66d1cfc43d0638fab54e43ff8a4f72a21ba3" @@ -16960,35 +16920,6 @@ webpack@^5: watchpack "^2.4.1" webpack-sources "^3.2.3" -webpack@5.98.0: - version "5.98.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz#44ae19a8f2ba97537978246072fb89d10d1fbd17" - integrity sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA== - dependencies: - "@types/eslint-scope" "^3.7.7" - "@types/estree" "^1.0.6" - "@webassemblyjs/ast" "^1.14.1" - "@webassemblyjs/wasm-edit" "^1.14.1" - "@webassemblyjs/wasm-parser" "^1.14.1" - acorn "^8.14.0" - browserslist "^4.24.0" - chrome-trace-event "^1.0.2" - enhanced-resolve "^5.17.1" - es-module-lexer "^1.2.1" - eslint-scope "5.1.1" - events "^3.2.0" - glob-to-regexp "^0.4.1" - graceful-fs "^4.2.11" - json-parse-even-better-errors "^2.3.1" - loader-runner "^4.2.0" - mime-types "^2.1.27" - neo-async "^2.6.2" - schema-utils "^4.3.0" - tapable "^2.1.1" - terser-webpack-plugin "^5.3.11" - watchpack "^2.4.1" - webpack-sources "^3.2.3" - websocket-driver@>=0.5.1: version "0.7.4" resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz#89ad5295bbf64b480abcba31e4953aca706f5760" From 9e1b32d5b4d65d0db18eac987f17928240972939 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 15 Apr 2025 10:45:48 -0700 Subject: [PATCH 10/29] Remove erroneous type dep --- packages/vertexai/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/vertexai/package.json b/packages/vertexai/package.json index c53cb5d3f48..09f4d80b4d5 100644 --- a/packages/vertexai/package.json +++ b/packages/vertexai/package.json @@ -52,7 +52,6 @@ "@firebase/component": "0.6.13", "@firebase/logger": "0.4.4", "@firebase/util": "1.11.0", - "@types/dom-chromium-ai": "0.0.6", "tslib": "^2.1.0" }, "license": "Apache-2.0", From b516b90efedfd0738f678f22bf7e7e205fbe785b Mon Sep 17 00:00:00 2001 From: Siddharth Gupta Date: Tue, 15 Apr 2025 17:13:10 -0700 Subject: [PATCH 11/29] docgen --- docs-devsite/vertexai.hybridparams.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs-devsite/vertexai.hybridparams.md b/docs-devsite/vertexai.hybridparams.md index c9b053b09f4..1a75be4d01d 100644 --- a/docs-devsite/vertexai.hybridparams.md +++ b/docs-devsite/vertexai.hybridparams.md @@ -26,15 +26,6 @@ export interface HybridParams | [mode](./vertexai.hybridparams.md#hybridparamsmode) | [InferenceMode](./vertexai.md#inferencemode) | Specifies on-device or in-cloud inference. Defaults to prefer on-device. | | [onDeviceParams](./vertexai.hybridparams.md#hybridparamsondeviceparams) | LanguageModelCreateOptions | Optional. Specifies advanced params for on-device inference. | -## HybridParams.inCloudParams - -Optional. Specifies advanced params for in-cloud inference. - -Signature: - -```typescript -inCloudParams?: ModelParams; -``` ## HybridParams.inCloudParams From 21d3aa1a924301faef5a831ab5982f5ddd300faa Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Wed, 16 Apr 2025 10:43:07 -0700 Subject: [PATCH 12/29] Inject LanguageModel provider --- packages/vertexai/src/api.ts | 7 ++++++- packages/vertexai/src/methods/chrome-adapter.ts | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 236ca73ce87..2f6de198608 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -31,6 +31,7 @@ import { import { VertexAIError } from './errors'; import { VertexAIModel, GenerativeModel, ImagenModel } from './models'; import { ChromeAdapter } from './methods/chrome-adapter'; +import { LanguageModel } from './types/language-model'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; @@ -95,7 +96,11 @@ export function getGenerativeModel( return new GenerativeModel( vertexAI, inCloudParams, - new ChromeAdapter(hybridParams.mode, hybridParams.onDeviceParams), + new ChromeAdapter( + window.LanguageModel as LanguageModel, + hybridParams.mode, + hybridParams.onDeviceParams + ), requestOptions ); } diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 26ecd55c2da..a51b4060e26 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -16,7 +16,10 @@ */ import { GenerateContentRequest, InferenceMode } from '../types'; -import { LanguageModelCreateOptions } from '../types/language-model'; +import { + LanguageModel, + LanguageModelCreateOptions +} from '../types/language-model'; /** * Defines an inference "backend" that uses Chrome's on-device model, @@ -24,6 +27,7 @@ import { LanguageModelCreateOptions } from '../types/language-model'; */ export class ChromeAdapter { constructor( + private languageModelProvider?: LanguageModel, private mode?: InferenceMode, private onDeviceParams?: LanguageModelCreateOptions ) {} From 1f720b91b8f5d58bad3ecaab4f5e2cf89398bdd7 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Wed, 16 Apr 2025 10:45:37 -0700 Subject: [PATCH 13/29] Run gendoc --- docs-devsite/vertexai.hybridparams.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs-devsite/vertexai.hybridparams.md b/docs-devsite/vertexai.hybridparams.md index 1a75be4d01d..cf847b40fa7 100644 --- a/docs-devsite/vertexai.hybridparams.md +++ b/docs-devsite/vertexai.hybridparams.md @@ -26,7 +26,6 @@ export interface HybridParams | [mode](./vertexai.hybridparams.md#hybridparamsmode) | [InferenceMode](./vertexai.md#inferencemode) | Specifies on-device or in-cloud inference. Defaults to prefer on-device. | | [onDeviceParams](./vertexai.hybridparams.md#hybridparamsondeviceparams) | LanguageModelCreateOptions | Optional. Specifies advanced params for on-device inference. | - ## HybridParams.inCloudParams Optional. Specifies advanced params for in-cloud inference. From 97631679f6cf94979141b620b1b4e6389f0f1490 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 25 Mar 2025 09:30:30 -0700 Subject: [PATCH 14/29] Define ChromeAdapter class --- packages/vertexai/src/types/ai.ts | 161 ++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 packages/vertexai/src/types/ai.ts diff --git a/packages/vertexai/src/types/ai.ts b/packages/vertexai/src/types/ai.ts new file mode 100644 index 00000000000..98a1e1d35bf --- /dev/null +++ b/packages/vertexai/src/types/ai.ts @@ -0,0 +1,161 @@ +/** + * Shims @types/dom-chromium-ai + * TODO: replace with @types/dom-chromium-ai once we can use es2020.intl. + */ +export interface AI { + readonly languageModel: AILanguageModelFactory; +} + +interface AICreateMonitor extends EventTarget { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ondownloadprogress: ((this: AICreateMonitor, ev: DownloadProgressEvent) => any) | null; + + addEventListener( + type: K, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, + options?: boolean | AddEventListenerOptions, + ): void; + addEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions, + ): void; + removeEventListener( + type: K, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, + options?: boolean | EventListenerOptions, + ): void; + removeEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | EventListenerOptions, + ): void; +} + +interface DownloadProgressEvent extends Event { + readonly loaded: number; + readonly total: number; +} + +interface AICreateMonitorEventMap { + downloadprogress: DownloadProgressEvent; +} + +type AICreateMonitorCallback = (monitor: AICreateMonitor) => void; + +type AICapabilityAvailability = "readily" | "after-download" | "no"; + +// Language Model +// https://github.com/explainers-by-googlers/prompt-api/#full-api-surface-in-web-idl + +interface AILanguageModelFactory { + create( + options?: AILanguageModelCreateOptionsWithSystemPrompt | AILanguageModelCreateOptionsWithoutSystemPrompt, + ): Promise; + capabilities(): Promise; +} + +interface AILanguageModelCreateOptions { + signal?: AbortSignal; + monitor?: AICreateMonitorCallback; + + topK?: number; + temperature?: number; +} + +export interface AILanguageModelCreateOptionsWithSystemPrompt extends AILanguageModelCreateOptions { + systemPrompt?: string; + initialPrompts?: AILanguageModelPrompt[]; +} + +interface AILanguageModelCreateOptionsWithoutSystemPrompt extends AILanguageModelCreateOptions { + systemPrompt?: never; + initialPrompts?: + | [AILanguageModelSystemPrompt, ...AILanguageModelPrompt[]] + | AILanguageModelPrompt[]; +} + +type AILanguageModelPromptRole = "user" | "assistant"; +type AILanguageModelInitialPromptRole = "system" | AILanguageModelPromptRole; + +interface AILanguageModelPrompt { + role: AILanguageModelPromptRole; + content: string; +} + +interface AILanguageModelInitialPrompt { + role: AILanguageModelInitialPromptRole; + content: string; +} + +interface AILanguageModelSystemPrompt extends AILanguageModelInitialPrompt { + role: "system"; +} + +type AILanguageModelPromptInput = string | AILanguageModelPrompt | AILanguageModelPrompt[]; + +interface AILanguageModel extends EventTarget { + prompt(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; + promptStreaming(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): ReadableStream; + + countPromptTokens(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; + readonly maxTokens: number; + readonly tokensSoFar: number; + readonly tokensLeft: number; + + readonly topK: number; + readonly temperature: number; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + oncontextoverflow: ((this: AILanguageModel, ev: Event) => any) | null; + + addEventListener( + type: K, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, + options?: boolean | AddEventListenerOptions, + ): void; + addEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions, + ): void; + removeEventListener( + type: K, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, + options?: boolean | EventListenerOptions, + ): void; + removeEventListener( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | EventListenerOptions, + ): void; + + clone(options?: AILanguageModelCloneOptions): Promise; + destroy(): void; +} + +interface AILanguageModelEventMap { + contextoverflow: Event; +} + +interface AILanguageModelPromptOptions { + signal?: AbortSignal; +} + +interface AILanguageModelCloneOptions { + signal?: AbortSignal; +} + +interface AILanguageModelCapabilities { + readonly available: AICapabilityAvailability; + languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; + + readonly defaultTopK: number | null; + readonly maxTopK: number | null; + readonly defaultTemperature: number | null; + readonly maxTemperature: number | null; +} From bf9de16cf8b0572ce8e44c8ee52bd9a7e8b63924 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 1 Apr 2025 17:14:19 -0700 Subject: [PATCH 15/29] Implement ChromeAdapter class --- common/api-review/util.api.md | 5 + packages/util/src/environment.ts | 6 + .../src/methods/chrome-adapter.test.ts | 60 ++++++ .../vertexai/src/methods/chrome-adapter.ts | 173 +++++++++++++++++- packages/vertexai/src/types/language-model.ts | 6 +- 5 files changed, 241 insertions(+), 9 deletions(-) create mode 100644 packages/vertexai/src/methods/chrome-adapter.test.ts diff --git a/common/api-review/util.api.md b/common/api-review/util.api.md index 8c62ff229ac..fb8afb2319c 100644 --- a/common/api-review/util.api.md +++ b/common/api-review/util.api.md @@ -264,6 +264,11 @@ export function isBrowser(): boolean; // @public (undocumented) export function isBrowserExtension(): boolean; +// Warning: (ae-missing-release-tag) "isChrome" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function isChrome(): boolean; + // Warning: (ae-missing-release-tag) "isCloudflareWorker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/packages/util/src/environment.ts b/packages/util/src/environment.ts index a0467b08c59..50d5f534106 100644 --- a/packages/util/src/environment.ts +++ b/packages/util/src/environment.ts @@ -173,6 +173,12 @@ export function isSafari(): boolean { ); } +export function isChrome(): boolean { + return ( + !isNode() && !!navigator.userAgent && navigator.userAgent.includes('Chrome') + ); +} + /** * This method checks if indexedDB is supported by current browser/service worker context * @return true if indexedDB is supported by current browser/service worker context diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts new file mode 100644 index 00000000000..453720c121d --- /dev/null +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect, use } from 'chai'; +import sinonChai from 'sinon-chai'; +import chaiAsPromised from 'chai-as-promised'; +import { ChromeAdapter } from './chrome-adapter'; +import { Availability, LanguageModel } from '../types/language-model'; + +use(sinonChai); +use(chaiAsPromised); + +describe('ChromeAdapter', () => { + describe('isOnDeviceRequest', () => { + it('returns true for simple text part', async () => { + expect( + ChromeAdapter._isOnDeviceRequest({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.true; + }); + it('returns false if contents empty', async () => { + expect( + ChromeAdapter._isOnDeviceRequest({ + contents: [] + }) + ).to.be.false; + }); + }); + describe('isAvailable', () => { + it('returns true if a model is available', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.available) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.true; + }); + }); +}); diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index a51b4060e26..e5364971479 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -15,41 +15,202 @@ * limitations under the License. */ -import { GenerateContentRequest, InferenceMode } from '../types'; import { + Content, + GenerateContentRequest, + InferenceMode, + Part, + Role, + TextPart +} from '../types'; +import { + Availability, LanguageModel, - LanguageModelCreateOptions + LanguageModelCreateOptions, + LanguageModelMessageRole, + LanguageModelMessageShorthand } from '../types/language-model'; +import { isChrome } from '@firebase/util'; /** * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. */ export class ChromeAdapter { + downloadPromise: Promise | undefined; + oldSession: LanguageModel | undefined; constructor( private languageModelProvider?: LanguageModel, private mode?: InferenceMode, private onDeviceParams?: LanguageModelCreateOptions ) {} - // eslint-disable-next-line @typescript-eslint/no-unused-vars + /** + * Convenience method to check if a given request can be made on-device. + * Encapsulates a few concerns: 1) the mode, 2) API existence, 3) prompt formatting, and + * 4) model availability, including triggering download if necessary. + * Pros: caller needn't be concerned with details of on-device availability. Cons: this method + * spans a few concerns and splits request validation from usage. If instance variables weren't + * already part of the API, we could consider a better separation of concerns. + */ async isAvailable(request: GenerateContentRequest): Promise { - return false; + // Returns false if we should only use in-cloud inference. + if (this.mode === 'only_in_cloud') { + return false; + } + // Returns false because only Chrome's experimental Prompt API is supported. + if (!isChrome()) { + return false; + } + // Returns false if the on-device inference API is undefined.; + if (!this.languageModelProvider) { + return false; + } + // Returns false if the request can't be run on-device. + if (!ChromeAdapter._isOnDeviceRequest(request)) { + return false; + } + const availability = await this.languageModelProvider.availability(); + switch (availability) { + case Availability.available: + // Returns true only if a model is immediately available. + return true; + case Availability.downloadable: + // Triggers async download if model is downloadable. + this.download(); + default: + return false; + } } async generateContentOnDevice( - // eslint-disable-next-line @typescript-eslint/no-unused-vars request: GenerateContentRequest ): Promise { + const initialPrompts = ChromeAdapter.toInitialPrompts(request.contents); + // Assumes validation asserted there is at least one initial prompt. + const prompt = initialPrompts.pop()!; + const systemPrompt = ChromeAdapter.toSystemPrompt( + request.systemInstruction + ); + const session = await this.session({ + initialPrompts, + systemPrompt + }); + const text = await session.prompt(prompt.content); return { json: () => Promise.resolve({ candidates: [ { content: { - parts: [{ text: '' }] + parts: [{ text }] } } ] }) } as Response; } + // Visible for testing + static _isOnDeviceRequest(request: GenerateContentRequest): boolean { + if (request.systemInstruction) { + const systemContent = request.systemInstruction as Content; + // Returns false if the role can't be represented on-device. + if (systemContent.role && systemContent.role === 'function') { + return false; + } + + // Returns false if the system prompt is multi-part. + if (systemContent.parts && systemContent.parts.length > 1) { + return false; + } + + // Returns false if the system prompt isn't text. + const systemText = request.systemInstruction as TextPart; + if (!systemText.text) { + return false; + } + } + + // Returns false if the prompt is empty. + if (request.contents.length === 0) { + return false; + } + + // Applies the same checks as above, but for each content item. + for (const content of request.contents) { + if (content.role === 'function') { + return false; + } + + if (content.parts.length > 1) { + return false; + } + + if (!content.parts[0].text) { + return false; + } + } + + return true; + } + private download(): void { + if (this.downloadPromise) { + return; + } + this.downloadPromise = this.languageModelProvider + ?.create(this.onDeviceParams) + .then((model: LanguageModel) => { + delete this.downloadPromise; + return model; + }); + return; + } + private static toSystemPrompt( + prompt: string | Content | Part | undefined + ): string | undefined { + if (!prompt) { + return undefined; + } + + if (typeof prompt === 'string') { + return prompt; + } + + const systemContent = prompt as Content; + if ( + systemContent.parts && + systemContent.parts[0] && + systemContent.parts[0].text + ) { + return systemContent.parts[0].text; + } + + const systemPart = prompt as Part; + if (systemPart.text) { + return systemPart.text; + } + + return undefined; + } + private static toOnDeviceRole(role: Role): LanguageModelMessageRole { + return role === 'model' ? 'assistant' : 'user'; + } + private static toInitialPrompts( + contents: Content[] + ): LanguageModelMessageShorthand[] { + return contents.map(c => ({ + role: ChromeAdapter.toOnDeviceRole(c.role), + // Assumes contents have been verified to contain only a single TextPart. + content: c.parts[0].text! + })); + } + private async session( + opts: LanguageModelCreateOptions + ): Promise { + const newSession = await this.languageModelProvider!.create(opts); + if (this.oldSession) { + this.oldSession.destroy(); + } + // Holds session reference, so model isn't unloaded from memory. + this.oldSession = newSession; + return newSession; + } } diff --git a/packages/vertexai/src/types/language-model.ts b/packages/vertexai/src/types/language-model.ts index e564ca467b4..f1dbc7eacdc 100644 --- a/packages/vertexai/src/types/language-model.ts +++ b/packages/vertexai/src/types/language-model.ts @@ -32,7 +32,7 @@ export interface LanguageModel extends EventTarget { ): Promise; destroy(): undefined; } -enum Availability { +export enum Availability { 'unavailable', 'downloadable', 'downloading', @@ -67,7 +67,7 @@ interface LanguageModelMessage { role: LanguageModelMessageRole; content: LanguageModelMessageContent[]; } -interface LanguageModelMessageShorthand { +export interface LanguageModelMessageShorthand { role: LanguageModelMessageRole; content: string; } @@ -75,7 +75,7 @@ interface LanguageModelMessageContent { type: LanguageModelMessageType; content: LanguageModelMessageContentValue; } -type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; type LanguageModelMessageType = 'text' | 'image' | 'audio'; type LanguageModelMessageContentValue = | ImageBitmapSource From be75b4f200ca8601ef49d733a7b87ec9aeb2380e Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 1 Apr 2025 17:15:47 -0700 Subject: [PATCH 16/29] Integrate with e2e test app --- e2e/sample-apps/modular.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/e2e/sample-apps/modular.js b/e2e/sample-apps/modular.js index 9e943e04494..2e6544eab6a 100644 --- a/e2e/sample-apps/modular.js +++ b/e2e/sample-apps/modular.js @@ -58,7 +58,7 @@ import { onValue, off } from 'firebase/database'; -import { getGenerativeModel, getVertexAI, VertexAI } from 'firebase/vertexai'; +import { getGenerativeModel, getVertexAI, InferenceMode, VertexAI } from 'firebase/vertexai'; import { getDataConnect, DataConnect } from 'firebase/data-connect'; /** @@ -332,6 +332,15 @@ function callDataConnect(app) { console.log('[DATACONNECT] initialized'); } +async function callVertex(app) { + console.log('[VERTEX] start'); + const vertex = getVertexAI(app) + const model = getGenerativeModel(vertex, {mode: InferenceMode.PREFER_ON_DEVICE}) + const result = await model.generateContent("What is Roko's Basalisk?") + console.log(result.response.text()) + console.log('[VERTEX] initialized'); +} + /** * Run smoke tests for all products. * Comment out any products you want to ignore. @@ -353,6 +362,7 @@ async function main() { await callVertexAI(app); callDataConnect(app); await authLogout(app); + await callVertex(app); console.log('DONE'); } From 50272f4b760929309d292b4e51e9d407c9a21be0 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 3 Apr 2025 13:43:54 -0700 Subject: [PATCH 17/29] Remove stray ai.ts --- packages/vertexai/src/types/ai.ts | 161 ------------------------------ 1 file changed, 161 deletions(-) delete mode 100644 packages/vertexai/src/types/ai.ts diff --git a/packages/vertexai/src/types/ai.ts b/packages/vertexai/src/types/ai.ts deleted file mode 100644 index 98a1e1d35bf..00000000000 --- a/packages/vertexai/src/types/ai.ts +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Shims @types/dom-chromium-ai - * TODO: replace with @types/dom-chromium-ai once we can use es2020.intl. - */ -export interface AI { - readonly languageModel: AILanguageModelFactory; -} - -interface AICreateMonitor extends EventTarget { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ondownloadprogress: ((this: AICreateMonitor, ev: DownloadProgressEvent) => any) | null; - - addEventListener( - type: K, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, - options?: boolean | AddEventListenerOptions, - ): void; - addEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | AddEventListenerOptions, - ): void; - removeEventListener( - type: K, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - listener: (this: AICreateMonitor, ev: AICreateMonitorEventMap[K]) => any, - options?: boolean | EventListenerOptions, - ): void; - removeEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | EventListenerOptions, - ): void; -} - -interface DownloadProgressEvent extends Event { - readonly loaded: number; - readonly total: number; -} - -interface AICreateMonitorEventMap { - downloadprogress: DownloadProgressEvent; -} - -type AICreateMonitorCallback = (monitor: AICreateMonitor) => void; - -type AICapabilityAvailability = "readily" | "after-download" | "no"; - -// Language Model -// https://github.com/explainers-by-googlers/prompt-api/#full-api-surface-in-web-idl - -interface AILanguageModelFactory { - create( - options?: AILanguageModelCreateOptionsWithSystemPrompt | AILanguageModelCreateOptionsWithoutSystemPrompt, - ): Promise; - capabilities(): Promise; -} - -interface AILanguageModelCreateOptions { - signal?: AbortSignal; - monitor?: AICreateMonitorCallback; - - topK?: number; - temperature?: number; -} - -export interface AILanguageModelCreateOptionsWithSystemPrompt extends AILanguageModelCreateOptions { - systemPrompt?: string; - initialPrompts?: AILanguageModelPrompt[]; -} - -interface AILanguageModelCreateOptionsWithoutSystemPrompt extends AILanguageModelCreateOptions { - systemPrompt?: never; - initialPrompts?: - | [AILanguageModelSystemPrompt, ...AILanguageModelPrompt[]] - | AILanguageModelPrompt[]; -} - -type AILanguageModelPromptRole = "user" | "assistant"; -type AILanguageModelInitialPromptRole = "system" | AILanguageModelPromptRole; - -interface AILanguageModelPrompt { - role: AILanguageModelPromptRole; - content: string; -} - -interface AILanguageModelInitialPrompt { - role: AILanguageModelInitialPromptRole; - content: string; -} - -interface AILanguageModelSystemPrompt extends AILanguageModelInitialPrompt { - role: "system"; -} - -type AILanguageModelPromptInput = string | AILanguageModelPrompt | AILanguageModelPrompt[]; - -interface AILanguageModel extends EventTarget { - prompt(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; - promptStreaming(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): ReadableStream; - - countPromptTokens(input: AILanguageModelPromptInput, options?: AILanguageModelPromptOptions): Promise; - readonly maxTokens: number; - readonly tokensSoFar: number; - readonly tokensLeft: number; - - readonly topK: number; - readonly temperature: number; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - oncontextoverflow: ((this: AILanguageModel, ev: Event) => any) | null; - - addEventListener( - type: K, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, - options?: boolean | AddEventListenerOptions, - ): void; - addEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | AddEventListenerOptions, - ): void; - removeEventListener( - type: K, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - listener: (this: AILanguageModel, ev: AILanguageModelEventMap[K]) => any, - options?: boolean | EventListenerOptions, - ): void; - removeEventListener( - type: string, - listener: EventListenerOrEventListenerObject, - options?: boolean | EventListenerOptions, - ): void; - - clone(options?: AILanguageModelCloneOptions): Promise; - destroy(): void; -} - -interface AILanguageModelEventMap { - contextoverflow: Event; -} - -interface AILanguageModelPromptOptions { - signal?: AbortSignal; -} - -interface AILanguageModelCloneOptions { - signal?: AbortSignal; -} - -interface AILanguageModelCapabilities { - readonly available: AICapabilityAvailability; - languageAvailable(languageTag: Intl.UnicodeBCP47LocaleIdentifier): AICapabilityAvailability; - - readonly defaultTopK: number | null; - readonly maxTopK: number | null; - readonly defaultTemperature: number | null; - readonly maxTemperature: number | null; -} From a90440eb51d445d6a438e355ad14996c29353d34 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 3 Apr 2025 14:01:12 -0700 Subject: [PATCH 18/29] Run yarn format --- e2e/sample-apps/modular.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/e2e/sample-apps/modular.js b/e2e/sample-apps/modular.js index 2e6544eab6a..292a11535a3 100644 --- a/e2e/sample-apps/modular.js +++ b/e2e/sample-apps/modular.js @@ -58,7 +58,12 @@ import { onValue, off } from 'firebase/database'; -import { getGenerativeModel, getVertexAI, InferenceMode, VertexAI } from 'firebase/vertexai'; +import { + getGenerativeModel, + getVertexAI, + InferenceMode, + VertexAI +} from 'firebase/vertexai'; import { getDataConnect, DataConnect } from 'firebase/data-connect'; /** @@ -334,10 +339,12 @@ function callDataConnect(app) { async function callVertex(app) { console.log('[VERTEX] start'); - const vertex = getVertexAI(app) - const model = getGenerativeModel(vertex, {mode: InferenceMode.PREFER_ON_DEVICE}) - const result = await model.generateContent("What is Roko's Basalisk?") - console.log(result.response.text()) + const vertex = getVertexAI(app); + const model = getGenerativeModel(vertex, { + mode: InferenceMode.PREFER_ON_DEVICE + }); + const result = await model.generateContent("What is Roko's Basalisk?"); + console.log(result.response.text()); console.log('[VERTEX] initialized'); } From 04b8334e4ab8954093df8d95f486658dd4563f6f Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 3 Apr 2025 17:47:35 -0700 Subject: [PATCH 19/29] Test model download logic --- .../src/methods/chrome-adapter.test.ts | 77 +++++++++++++++++++ .../vertexai/src/methods/chrome-adapter.ts | 14 ++-- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts index 453720c121d..89b140fcebb 100644 --- a/packages/vertexai/src/methods/chrome-adapter.test.ts +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -20,6 +20,7 @@ import sinonChai from 'sinon-chai'; import chaiAsPromised from 'chai-as-promised'; import { ChromeAdapter } from './chrome-adapter'; import { Availability, LanguageModel } from '../types/language-model'; +import { stub } from 'sinon'; use(sinonChai); use(chaiAsPromised); @@ -56,5 +57,81 @@ describe('ChromeAdapter', () => { }) ).to.be.true; }); + it('returns false and triggers download when model is available after download', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.downloadable), + create: () => Promise.resolve({}) + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + {} as LanguageModel + ); + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + expect(createStub).to.have.been.calledOnce; + }); + it('avoids redundant downloads', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.downloadable), + create: () => Promise.resolve({}) + } as LanguageModel; + const downloadPromise = new Promise(() => { + /* never resolves */ + }); + const createStub = stub(languageModelProvider, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapter(languageModelProvider); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledOnce; + }); + it('clears state when download completes', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.downloadable), + create: () => Promise.resolve({}) + } as LanguageModel; + let resolveDownload; + const downloadPromise = new Promise(resolveCallback => { + resolveDownload = resolveCallback; + }); + const createStub = stub(languageModelProvider, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapter(languageModelProvider); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + resolveDownload!(); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledTwice; + }); + it('returns false when model is never available', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.unavailable), + create: () => Promise.resolve({}) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + }); }); }); diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index e5364971479..8c867f09682 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -37,8 +37,9 @@ import { isChrome } from '@firebase/util'; * and encapsulates logic for detecting when on-device is possible. */ export class ChromeAdapter { - downloadPromise: Promise | undefined; - oldSession: LanguageModel | undefined; + private isDownloading = false; + private downloadPromise: Promise | undefined; + private oldSession: LanguageModel | undefined; constructor( private languageModelProvider?: LanguageModel, private mode?: InferenceMode, @@ -152,16 +153,15 @@ export class ChromeAdapter { return true; } private download(): void { - if (this.downloadPromise) { + if (this.isDownloading) { return; } + this.isDownloading = true; this.downloadPromise = this.languageModelProvider ?.create(this.onDeviceParams) - .then((model: LanguageModel) => { - delete this.downloadPromise; - return model; + .then(() => { + this.isDownloading = false; }); - return; } private static toSystemPrompt( prompt: string | Content | Part | undefined From 8b6f48369b5f610d517d61592414e7fa0b2e3f5b Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 3 Apr 2025 18:16:32 -0700 Subject: [PATCH 20/29] Test request-based availability checks --- .../src/methods/chrome-adapter.test.ts | 146 ++++++++++++++++-- .../vertexai/src/methods/chrome-adapter.ts | 43 +++--- 2 files changed, 157 insertions(+), 32 deletions(-) diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts index 89b140fcebb..1a72fad354c 100644 --- a/packages/vertexai/src/methods/chrome-adapter.test.ts +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -21,29 +21,155 @@ import chaiAsPromised from 'chai-as-promised'; import { ChromeAdapter } from './chrome-adapter'; import { Availability, LanguageModel } from '../types/language-model'; import { stub } from 'sinon'; +import * as util from '@firebase/util'; use(sinonChai); use(chaiAsPromised); describe('ChromeAdapter', () => { - describe('isOnDeviceRequest', () => { - it('returns true for simple text part', async () => { + describe('isAvailable', () => { + it('returns false if mode is only cloud', async () => { + const adapter = new ChromeAdapter(undefined, 'only_in_cloud'); expect( - ChromeAdapter._isOnDeviceRequest({ - contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + await adapter.isAvailable({ + contents: [] }) - ).to.be.true; + ).to.be.false; }); - it('returns false if contents empty', async () => { + it('returns false if browser is not Chrome', async () => { + const chromeStub = stub(util, 'isChrome').returns(false); + const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); expect( - ChromeAdapter._isOnDeviceRequest({ + await adapter.isAvailable({ contents: [] }) ).to.be.false; + chromeStub.restore(); }); - }); - describe('isAvailable', () => { - it('returns true if a model is available', async () => { + it('returns false if AI API is undefined', async () => { + const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if LanguageModel API is undefined', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request contents empty', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request content has function role', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'function', + parts: [] + } + ] + }) + ).to.be.false; + }); + it('returns false if request content has multiple parts', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ text: 'a' }, { text: 'b' }] + } + ] + }) + ).to.be.false; + }); + it('returns false if request content has non-text part', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ inlineData: { mimeType: 'a', data: 'b' } }] + } + ] + }) + ).to.be.false; + }); + it('returns false if request system instruction has function role', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [], + systemInstruction: { + role: 'function', + parts: [] + } + }) + ).to.be.false; + }); + it('returns false if request system instruction has multiple parts', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [], + systemInstruction: { + role: 'function', + parts: [{ text: 'a' }, { text: 'b' }] + } + }) + ).to.be.false; + }); + it('returns false if request system instruction has non-text part', async () => { + const adapter = new ChromeAdapter( + {} as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [], + systemInstruction: { + role: 'function', + parts: [{ inlineData: { mimeType: 'a', data: 'b' } }] + } + }) + ).to.be.false; + }); + it('returns true if model is readily available', async () => { const languageModelProvider = { availability: () => Promise.resolve(Availability.available) } as LanguageModel; diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 8c867f09682..ff782cc5d88 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -67,7 +67,7 @@ export class ChromeAdapter { return false; } // Returns false if the request can't be run on-device. - if (!ChromeAdapter._isOnDeviceRequest(request)) { + if (!ChromeAdapter.isOnDeviceRequest(request)) { return false; } const availability = await this.languageModelProvider.availability(); @@ -109,27 +109,7 @@ export class ChromeAdapter { }) } as Response; } - // Visible for testing - static _isOnDeviceRequest(request: GenerateContentRequest): boolean { - if (request.systemInstruction) { - const systemContent = request.systemInstruction as Content; - // Returns false if the role can't be represented on-device. - if (systemContent.role && systemContent.role === 'function') { - return false; - } - - // Returns false if the system prompt is multi-part. - if (systemContent.parts && systemContent.parts.length > 1) { - return false; - } - - // Returns false if the system prompt isn't text. - const systemText = request.systemInstruction as TextPart; - if (!systemText.text) { - return false; - } - } - + private static isOnDeviceRequest(request: GenerateContentRequest): boolean { // Returns false if the prompt is empty. if (request.contents.length === 0) { return false; @@ -150,6 +130,25 @@ export class ChromeAdapter { } } + if (request.systemInstruction) { + const systemContent = request.systemInstruction as Content; + // Returns false if the role can't be represented on-device. + if (systemContent.role && systemContent.role === 'function') { + return false; + } + + // Returns false if the system prompt is multi-part. + if (systemContent.parts && systemContent.parts.length > 1) { + return false; + } + + // Returns false if the system prompt isn't text. + const systemText = request.systemInstruction as TextPart; + if (!systemText.text) { + return false; + } + } + return true; } private download(): void { From 8056bc90a107fd9d9d60ae28f94f3f7f4bee5b5e Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Fri, 4 Apr 2025 16:02:21 -0700 Subject: [PATCH 21/29] Test content generation --- .../src/methods/chrome-adapter.test.ts | 64 +++++++++++++++++- .../vertexai/src/methods/chrome-adapter.ts | 66 +++++++------------ packages/vertexai/src/types/language-model.ts | 8 +-- 3 files changed, 89 insertions(+), 49 deletions(-) diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts index 1a72fad354c..8906bf398bf 100644 --- a/packages/vertexai/src/methods/chrome-adapter.test.ts +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -19,9 +19,14 @@ import { expect, use } from 'chai'; import sinonChai from 'sinon-chai'; import chaiAsPromised from 'chai-as-promised'; import { ChromeAdapter } from './chrome-adapter'; -import { Availability, LanguageModel } from '../types/language-model'; +import { + Availability, + LanguageModel, + LanguageModelCreateOptions +} from '../types/language-model'; import { stub } from 'sinon'; import * as util from '@firebase/util'; +import { GenerateContentRequest } from '../types'; use(sinonChai); use(chaiAsPromised); @@ -191,16 +196,18 @@ describe('ChromeAdapter', () => { const createStub = stub(languageModelProvider, 'create').resolves( {} as LanguageModel ); + const onDeviceParams = {} as LanguageModelCreateOptions; const adapter = new ChromeAdapter( languageModelProvider, - 'prefer_on_device' + 'prefer_on_device', + onDeviceParams ); expect( await adapter.isAvailable({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }) ).to.be.false; - expect(createStub).to.have.been.calledOnce; + expect(createStub).to.have.been.calledOnceWith(onDeviceParams); }); it('avoids redundant downloads', async () => { const languageModelProvider = { @@ -260,4 +267,55 @@ describe('ChromeAdapter', () => { ).to.be.false; }); }); + describe('generateContentOnDevice', () => { + it('generates content', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + prompt: i => Promise.resolve(i) + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const promptOutput = 'hi'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const onDeviceParams = { + systemPrompt: 'be yourself' + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + onDeviceParams + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + const response = await adapter.generateContentOnDevice(request); + // Asserts initialization params are proxied. + expect(createStub).to.have.been.calledOnceWith(onDeviceParams); + // Asserts Vertex input type is mapped to Chrome type. + expect(promptStub).to.have.been.calledOnceWith([ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] + } + ]); + // Asserts expected output. + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text: promptOutput }] + } + } + ] + }); + }); + }); }); diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index ff782cc5d88..59f080468c2 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -27,8 +27,9 @@ import { Availability, LanguageModel, LanguageModelCreateOptions, + LanguageModelMessage, LanguageModelMessageRole, - LanguageModelMessageShorthand + LanguageModelMessageContent } from '../types/language-model'; import { isChrome } from '@firebase/util'; @@ -85,17 +86,13 @@ export class ChromeAdapter { async generateContentOnDevice( request: GenerateContentRequest ): Promise { - const initialPrompts = ChromeAdapter.toInitialPrompts(request.contents); - // Assumes validation asserted there is at least one initial prompt. - const prompt = initialPrompts.pop()!; - const systemPrompt = ChromeAdapter.toSystemPrompt( - request.systemInstruction + const session = await this.session( + // TODO: normalize on-device params during construction. + this.onDeviceParams || {} ); - const session = await this.session({ - initialPrompts, - systemPrompt - }); - const text = await session.prompt(prompt.content); + const messages = ChromeAdapter.toLanguageModelMessages(request.contents); + const text = await session.prompt(messages); + console.log(text); return { json: () => Promise.resolve({ @@ -162,45 +159,30 @@ export class ChromeAdapter { this.isDownloading = false; }); } - private static toSystemPrompt( - prompt: string | Content | Part | undefined - ): string | undefined { - if (!prompt) { - return undefined; - } - - if (typeof prompt === 'string') { - return prompt; - } - - const systemContent = prompt as Content; - if ( - systemContent.parts && - systemContent.parts[0] && - systemContent.parts[0].text - ) { - return systemContent.parts[0].text; - } - - const systemPart = prompt as Part; - if (systemPart.text) { - return systemPart.text; - } - - return undefined; - } private static toOnDeviceRole(role: Role): LanguageModelMessageRole { return role === 'model' ? 'assistant' : 'user'; } - private static toInitialPrompts( + private static toLanguageModelMessages( contents: Content[] - ): LanguageModelMessageShorthand[] { + ): LanguageModelMessage[] { return contents.map(c => ({ role: ChromeAdapter.toOnDeviceRole(c.role), - // Assumes contents have been verified to contain only a single TextPart. - content: c.parts[0].text! + content: c.parts.map(ChromeAdapter.toLanguageModelMessageContent) })); } + private static toLanguageModelMessageContent( + part: Part + ): LanguageModelMessageContent { + if (part.text) { + return { + type: 'text', + content: part.text + }; + } + // Assumes contents have been verified to contain only a single TextPart. + // TODO: support other input types + throw new Error('Not yet implemented'); + } private async session( opts: LanguageModelCreateOptions ): Promise { diff --git a/packages/vertexai/src/types/language-model.ts b/packages/vertexai/src/types/language-model.ts index f1dbc7eacdc..88354d0aeec 100644 --- a/packages/vertexai/src/types/language-model.ts +++ b/packages/vertexai/src/types/language-model.ts @@ -56,22 +56,22 @@ interface LanguageModelExpectedInput { type: LanguageModelMessageType; languages?: string[]; } -type LanguageModelPrompt = +export type LanguageModelPrompt = | LanguageModelMessage[] | LanguageModelMessageShorthand[] | string; type LanguageModelInitialPrompts = | LanguageModelMessage[] | LanguageModelMessageShorthand[]; -interface LanguageModelMessage { +export interface LanguageModelMessage { role: LanguageModelMessageRole; content: LanguageModelMessageContent[]; } -export interface LanguageModelMessageShorthand { +interface LanguageModelMessageShorthand { role: LanguageModelMessageRole; content: string; } -interface LanguageModelMessageContent { +export interface LanguageModelMessageContent { type: LanguageModelMessageType; content: LanguageModelMessageContentValue; } From a32421d512706a74fc29fd231e5aed431035e71d Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Fri, 4 Apr 2025 16:19:44 -0700 Subject: [PATCH 22/29] Remove request.systemInstruction validation We only define system prompts via onDeviceParams initialization. --- .../vertexai/src/methods/chrome-adapter.ts | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 59f080468c2..f1d656159bc 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -20,8 +20,7 @@ import { GenerateContentRequest, InferenceMode, Part, - Role, - TextPart + Role } from '../types'; import { Availability, @@ -127,25 +126,6 @@ export class ChromeAdapter { } } - if (request.systemInstruction) { - const systemContent = request.systemInstruction as Content; - // Returns false if the role can't be represented on-device. - if (systemContent.role && systemContent.role === 'function') { - return false; - } - - // Returns false if the system prompt is multi-part. - if (systemContent.parts && systemContent.parts.length > 1) { - return false; - } - - // Returns false if the system prompt isn't text. - const systemText = request.systemInstruction as TextPart; - if (!systemText.text) { - return false; - } - } - return true; } private download(): void { From d3176c23b35e11bbff442a1335ad006bf08288d6 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 8 Apr 2025 08:51:46 -0700 Subject: [PATCH 23/29] Update content generator to emulate Vertex response --- packages/vertexai/src/methods/chrome-adapter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index f1d656159bc..b3152a8daab 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -91,7 +91,6 @@ export class ChromeAdapter { ); const messages = ChromeAdapter.toLanguageModelMessages(request.contents); const text = await session.prompt(messages); - console.log(text); return { json: () => Promise.resolve({ From dc0628f8eb66e20041f850adefb4ee58e1fb749f Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 17 Apr 2025 11:31:37 -0700 Subject: [PATCH 24/29] Remove redundant callVertex e2e test --- e2e/sample-apps/modular.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/e2e/sample-apps/modular.js b/e2e/sample-apps/modular.js index 292a11535a3..4c5238d44dc 100644 --- a/e2e/sample-apps/modular.js +++ b/e2e/sample-apps/modular.js @@ -337,17 +337,6 @@ function callDataConnect(app) { console.log('[DATACONNECT] initialized'); } -async function callVertex(app) { - console.log('[VERTEX] start'); - const vertex = getVertexAI(app); - const model = getGenerativeModel(vertex, { - mode: InferenceMode.PREFER_ON_DEVICE - }); - const result = await model.generateContent("What is Roko's Basalisk?"); - console.log(result.response.text()); - console.log('[VERTEX] initialized'); -} - /** * Run smoke tests for all products. * Comment out any products you want to ignore. @@ -369,7 +358,6 @@ async function main() { await callVertexAI(app); callDataConnect(app); await authLogout(app); - await callVertex(app); console.log('DONE'); } From 47f00b2770e3f17ee04cd2df6b511f239d5186fe Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 17 Apr 2025 14:17:00 -0700 Subject: [PATCH 25/29] Document ChromeAdapter methods --- .../vertexai/src/methods/chrome-adapter.ts | 76 ++++++++++++++++--- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index b3152a8daab..15d0eb9cdef 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -45,13 +45,21 @@ export class ChromeAdapter { private mode?: InferenceMode, private onDeviceParams?: LanguageModelCreateOptions ) {} + /** - * Convenience method to check if a given request can be made on-device. - * Encapsulates a few concerns: 1) the mode, 2) API existence, 3) prompt formatting, and - * 4) model availability, including triggering download if necessary. - * Pros: caller needn't be concerned with details of on-device availability. Cons: this method - * spans a few concerns and splits request validation from usage. If instance variables weren't - * already part of the API, we could consider a better separation of concerns. + * Checks if a given request can be made on-device. + * + *
    Encapsulates a few concerns: + *
  1. the mode
    2. + *
  2. API existence
    3. + *
  3. prompt formatting
    4. + *
  4. model availability, including triggering download if necessary
    5. + *
+ * + *

Pros: callers needn't be concerned with details of on-device availability.

+ *

Cons: this method spans a few concerns and splits request validation from usage. + * If instance variables weren't already part of the API, we could consider a better + * separation of concerns.

*/ async isAvailable(request: GenerateContentRequest): Promise { // Returns false if we should only use in-cloud inference. @@ -82,10 +90,19 @@ export class ChromeAdapter { return false; } } + + /** + * Generates content on device. + * + *

This is comparable to {@link GenerativeModel.generateContent} for generating content in + * Cloud.

+ * @param request a standard Vertex {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ async generateContentOnDevice( request: GenerateContentRequest ): Promise { - const session = await this.session( + const session = await this.createSession( // TODO: normalize on-device params during construction. this.onDeviceParams || {} ); @@ -104,6 +121,10 @@ export class ChromeAdapter { }) } as Response; } + + /** + * Asserts inference for the given request can be performed by an on-device model. + */ private static isOnDeviceRequest(request: GenerateContentRequest): boolean { // Returns false if the prompt is empty. if (request.contents.length === 0) { @@ -127,6 +148,16 @@ export class ChromeAdapter { return true; } + + /** + * Triggers the download of an on-device model. + * + *

Chrome only downloads models as needed. Chrome knows a model is needed when code calls + * LanguageModel.create.

+ * + *

Since Chrome manages the download, the SDK can only avoid redundant download requests by + * tracking if a download has previously been requested.

+ */ private download(): void { if (this.isDownloading) { return; @@ -138,9 +169,17 @@ export class ChromeAdapter { this.isDownloading = false; }); } + + /** + * Converts a Vertex role string to a Chrome role string. + */ private static toOnDeviceRole(role: Role): LanguageModelMessageRole { return role === 'model' ? 'assistant' : 'user'; } + + /** + * Converts a Vertex Content object to a Chrome LanguageModelMessage object. + */ private static toLanguageModelMessages( contents: Content[] ): LanguageModelMessage[] { @@ -149,6 +188,10 @@ export class ChromeAdapter { content: c.parts.map(ChromeAdapter.toLanguageModelMessageContent) })); } + + /** + * Converts a Vertex Part object to a Chrome LanguageModelMessageContent object. + */ private static toLanguageModelMessageContent( part: Part ): LanguageModelMessageContent { @@ -162,10 +205,23 @@ export class ChromeAdapter { // TODO: support other input types throw new Error('Not yet implemented'); } - private async session( - opts: LanguageModelCreateOptions + + /** + * Abstracts Chrome session creation. + * + *

Chrome uses a multi-turn session for all inference. Vertex uses single-turn for all + * inference. To map the Vertex API to Chrome's API, the SDK creates a new session for all + * inference.

+ * + *

Chrome will remove a model from memory if it's no longer in use, so this method ensures a + * new session is created before an old session is destroyed.

+ */ + private async createSession( + // TODO: define a default value, since these are optional. + options: LanguageModelCreateOptions ): Promise { - const newSession = await this.languageModelProvider!.create(opts); + // TODO: could we use this.onDeviceParams instead of passing in options? + const newSession = await this.languageModelProvider!.create(options); if (this.oldSession) { this.oldSession.destroy(); } From b460b8b9f4da6684ce8c1c9612f89eb3826d952f Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 17 Apr 2025 14:39:34 -0700 Subject: [PATCH 26/29] Update yarn.lock --- yarn.lock | 64 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/yarn.lock b/yarn.lock index adbe420e7b0..6fb62665203 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1299,6 +1299,28 @@ resolved "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2" integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q== +"@firebase/app@0.11.3": + version "0.11.3" + resolved "https://registry.npmjs.org/@firebase/app/-/app-0.11.3.tgz#80abcb65a1dce743d081022d2934fe69e60c1d41" + integrity sha512-QlTZl/RcqPSonYxB87n8KgAUW2L6ZZz0W4D91PVmQ1tJPsKsKPrWAFHL0ii2cQW6FxTxfNjbZ7kucuIcKXk3tw== + dependencies: + "@firebase/component" "0.6.13" + "@firebase/logger" "0.4.4" + "@firebase/util" "1.11.0" + idb "7.1.1" + tslib "^2.1.0" + +"@firebase/vertexai@1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@firebase/vertexai/-/vertexai-1.2.1.tgz#6962b8b389f10b58033b8c700b27d0cfdc9ebd22" + integrity sha512-cukZ5ne2RsOWB4PB1EO6nTXgOLxPMKDJfEn+XnSV5ZKWM0ID5o0DvbyS59XihFaBzmy2SwJldP5ap7/xUnW4jA== + dependencies: + "@firebase/app-check-interop-types" "0.3.3" + "@firebase/component" "0.6.13" + "@firebase/logger" "0.4.4" + "@firebase/util" "1.11.0" + tslib "^2.1.0" + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -3159,6 +3181,13 @@ resolved "https://registry.npmjs.org/@types/node/-/node-10.17.13.tgz#ccebcdb990bd6139cd16e84c39dc2fb1023ca90c" integrity sha512-pMCcqU2zT4TjqYFrWtYHKal7Sl30Ims6ulZ4UFXxI4xbtQqK/qqKwkDoBFCfooRqqmRu9vY3xaJRwxSh673aYg== +"@types/node@18.19.75": + version "18.19.75" + resolved "https://registry.npmjs.org/@types/node/-/node-18.19.75.tgz#be932799d1ab40779ffd16392a2b2300f81b565d" + integrity sha512-UIksWtThob6ZVSyxcOqCLOUNg/dyO1Qvx4McgeuhrEtHTLFTf7BBhEazaE4K806FGTPtzd/2sE90qn4fVr7cyw== + dependencies: + undici-types "~5.26.4" + "@types/node@18.19.83": version "18.19.83" resolved "https://registry.npmjs.org/@types/node/-/node-18.19.83.tgz#44d302cd09364640bdd45d001bc75e596f7da920" @@ -15743,6 +15772,17 @@ terser-webpack-plugin@^5.3.10: serialize-javascript "^6.0.2" terser "^5.31.1" +terser-webpack-plugin@^5.3.11: + version "5.3.14" + resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz#9031d48e57ab27567f02ace85c7d690db66c3e06" + integrity sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw== + dependencies: + "@jridgewell/trace-mapping" "^0.3.25" + jest-worker "^27.4.5" + schema-utils "^4.3.0" + serialize-javascript "^6.0.2" + terser "^5.31.1" + terser@5.37.0, terser@^5.17.4, terser@^5.31.1: version "5.37.0" resolved "https://registry.npmjs.org/terser/-/terser-5.37.0.tgz#38aa66d1cfc43d0638fab54e43ff8a4f72a21ba3" @@ -16862,10 +16902,10 @@ webpack-virtual-modules@0.6.2: resolved "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz#057faa9065c8acf48f24cb57ac0e77739ab9a7e8" integrity sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ== -webpack@5.98.0: - version "5.98.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz#44ae19a8f2ba97537978246072fb89d10d1fbd17" - integrity sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA== +webpack@5.97.1, webpack@^5: + version "5.97.1" + resolved "https://registry.npmjs.org/webpack/-/webpack-5.97.1.tgz#972a8320a438b56ff0f1d94ade9e82eac155fa58" + integrity sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg== dependencies: "@types/eslint-scope" "^3.7.7" "@types/estree" "^1.0.6" @@ -16885,16 +16925,16 @@ webpack@5.98.0: loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^4.3.0" + schema-utils "^3.2.0" tapable "^2.1.1" - terser-webpack-plugin "^5.3.11" + terser-webpack-plugin "^5.3.10" watchpack "^2.4.1" webpack-sources "^3.2.3" -webpack@^5: - version "5.97.1" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.97.1.tgz#972a8320a438b56ff0f1d94ade9e82eac155fa58" - integrity sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg== +webpack@5.98.0: + version "5.98.0" + resolved "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz#44ae19a8f2ba97537978246072fb89d10d1fbd17" + integrity sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA== dependencies: "@types/eslint-scope" "^3.7.7" "@types/estree" "^1.0.6" @@ -16914,9 +16954,9 @@ webpack@^5: loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^3.2.0" + schema-utils "^4.3.0" tapable "^2.1.1" - terser-webpack-plugin "^5.3.10" + terser-webpack-plugin "^5.3.11" watchpack "^2.4.1" webpack-sources "^3.2.3" From 8d9a493a949f692e2beb66ac5ef81d2b246c5682 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 17 Apr 2025 15:15:40 -0700 Subject: [PATCH 27/29] Use package.json files from base branch --- package.json | 21 ++++----- packages/vertexai/package.json | 7 ++- scripts/release/utils/workspace.ts | 6 +-- yarn.lock | 76 +++++++++--------------------- 4 files changed, 34 insertions(+), 76 deletions(-) diff --git a/package.json b/package.json index 43589733b54..30b6b09a003 100644 --- a/package.json +++ b/package.json @@ -56,16 +56,11 @@ "type": "git", "url": "git+https://github.com/firebase/firebase-js-sdk.git" }, - "workspaces": { - "packages": [ - "packages/*", - "integration/*", - "repo-scripts/*" - ], - "nohoist": [ - "**/vertexai/@types/dom-chromium-ai" - ] - }, + "workspaces": [ + "packages/*", + "integration/*", + "repo-scripts/*" + ], "devDependencies": { "@babel/core": "7.26.8", "@babel/plugin-transform-modules-commonjs": "7.26.3", @@ -85,7 +80,7 @@ "@types/long": "4.0.2", "@types/mocha": "9.1.1", "@types/mz": "2.7.8", - "@types/node": "18.19.75", + "@types/node": "18.19.83", "@types/request": "2.48.12", "@types/sinon": "9.0.11", "@types/sinon-chai": "3.2.12", @@ -144,7 +139,7 @@ "nyc": "15.1.0", "ora": "5.4.1", "patch-package": "7.0.2", - "playwright": "1.50.1", + "playwright": "1.51.1", "postinstall-postinstall": "2.1.0", "prettier": "2.8.8", "protractor": "5.4.2", @@ -163,7 +158,7 @@ "typedoc": "0.16.11", "typescript": "5.5.4", "watch": "1.0.2", - "webpack": "5.97.1", + "webpack": "5.98.0", "yargs": "17.7.2" } } diff --git a/packages/vertexai/package.json b/packages/vertexai/package.json index 09f4d80b4d5..9faf562a535 100644 --- a/packages/vertexai/package.json +++ b/packages/vertexai/package.json @@ -1,6 +1,6 @@ { "name": "@firebase/vertexai", - "version": "1.2.0", + "version": "1.2.1", "description": "A Firebase SDK for VertexAI", "author": "Firebase (https://firebase.google.com/)", "engines": { @@ -56,9 +56,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@firebase/app": "0.11.3", + "@firebase/app": "0.11.4", "@rollup/plugin-json": "6.1.0", - "@types/dom-chromium-ai": "0.0.6", "rollup": "2.79.2", "rollup-plugin-replace": "2.2.0", "rollup-plugin-typescript2": "0.36.0", @@ -79,4 +78,4 @@ ], "reportDir": "./coverage/node" } -} \ No newline at end of file +} diff --git a/scripts/release/utils/workspace.ts b/scripts/release/utils/workspace.ts index 3dba8651675..077e3124b2a 100644 --- a/scripts/release/utils/workspace.ts +++ b/scripts/release/utils/workspace.ts @@ -27,10 +27,8 @@ const writeFile = promisify(_writeFile); const { workspaces: rawWorkspaces -}: { workspaces: { packages: string[] } } = require(`${root}/package.json`); -const workspaces = rawWorkspaces.packages.map( - workspace => `${root}/${workspace}` -); +}: { workspaces: string[] } = require(`${root}/package.json`); +const workspaces = rawWorkspaces.map(workspace => `${root}/${workspace}`); export function mapWorkspaceToPackages( workspaces: string[] diff --git a/yarn.lock b/yarn.lock index 6fb62665203..d5ea91a7093 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1299,28 +1299,6 @@ resolved "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2" integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q== -"@firebase/app@0.11.3": - version "0.11.3" - resolved "https://registry.npmjs.org/@firebase/app/-/app-0.11.3.tgz#80abcb65a1dce743d081022d2934fe69e60c1d41" - integrity sha512-QlTZl/RcqPSonYxB87n8KgAUW2L6ZZz0W4D91PVmQ1tJPsKsKPrWAFHL0ii2cQW6FxTxfNjbZ7kucuIcKXk3tw== - dependencies: - "@firebase/component" "0.6.13" - "@firebase/logger" "0.4.4" - "@firebase/util" "1.11.0" - idb "7.1.1" - tslib "^2.1.0" - -"@firebase/vertexai@1.2.1": - version "1.2.1" - resolved "https://registry.npmjs.org/@firebase/vertexai/-/vertexai-1.2.1.tgz#6962b8b389f10b58033b8c700b27d0cfdc9ebd22" - integrity sha512-cukZ5ne2RsOWB4PB1EO6nTXgOLxPMKDJfEn+XnSV5ZKWM0ID5o0DvbyS59XihFaBzmy2SwJldP5ap7/xUnW4jA== - dependencies: - "@firebase/app-check-interop-types" "0.3.3" - "@firebase/component" "0.6.13" - "@firebase/logger" "0.4.4" - "@firebase/util" "1.11.0" - tslib "^2.1.0" - "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -2969,11 +2947,6 @@ resolved "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz#334311971d3a07121e7eb91b684a605e7eea9cbd" integrity sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw== -"@types/dom-chromium-ai@0.0.6": - version "0.0.6" - resolved "https://registry.npmjs.org/@types/dom-chromium-ai/-/dom-chromium-ai-0.0.6.tgz#0c9e5712d8db3d26586cd9f175001b509cd2e514" - integrity sha512-/jUGe9a3BLzsjjg18Olk/Ul64PZ0P4aw8uNxrXeXVTni5PSxyCfyhHb4UohsXNVByOnwYGzlqUcb3vYKVsG4mg== - "@types/eslint-scope@^3.7.7": version "3.7.7" resolved "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz#3108bd5f18b0cdb277c867b3dd449c9ed7079ac5" @@ -3181,13 +3154,6 @@ resolved "https://registry.npmjs.org/@types/node/-/node-10.17.13.tgz#ccebcdb990bd6139cd16e84c39dc2fb1023ca90c" integrity sha512-pMCcqU2zT4TjqYFrWtYHKal7Sl30Ims6ulZ4UFXxI4xbtQqK/qqKwkDoBFCfooRqqmRu9vY3xaJRwxSh673aYg== -"@types/node@18.19.75": - version "18.19.75" - resolved "https://registry.npmjs.org/@types/node/-/node-18.19.75.tgz#be932799d1ab40779ffd16392a2b2300f81b565d" - integrity sha512-UIksWtThob6ZVSyxcOqCLOUNg/dyO1Qvx4McgeuhrEtHTLFTf7BBhEazaE4K806FGTPtzd/2sE90qn4fVr7cyw== - dependencies: - undici-types "~5.26.4" - "@types/node@18.19.83": version "18.19.83" resolved "https://registry.npmjs.org/@types/node/-/node-18.19.83.tgz#44d302cd09364640bdd45d001bc75e596f7da920" @@ -13244,17 +13210,17 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" -playwright-core@1.50.1: - version "1.50.1" - resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.1.tgz#6a0484f1f1c939168f40f0ab3828c4a1592c4504" - integrity sha512-ra9fsNWayuYumt+NiM069M6OkcRb1FZSK8bgi66AtpFoWkg2+y0bJSNmkFrWhMbEBbVKC/EruAHH3g0zmtwGmQ== +playwright-core@1.51.1: + version "1.51.1" + resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz#d57f0393e02416f32a47cf82b27533656a8acce1" + integrity sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw== -playwright@1.50.1: - version "1.50.1" - resolved "https://registry.npmjs.org/playwright/-/playwright-1.50.1.tgz#2f93216511d65404f676395bfb97b41aa052b180" - integrity sha512-G8rwsOQJ63XG6BbKj2w5rHeavFjy5zynBA9zsJMMtBoe/Uf757oG12NXz6e6OirF7RCrTVAKFXbLmn1RbL7Qaw== +playwright@1.51.1: + version "1.51.1" + resolved "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz#ae1467ee318083968ad28d6990db59f47a55390f" + integrity sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw== dependencies: - playwright-core "1.50.1" + playwright-core "1.51.1" optionalDependencies: fsevents "2.3.2" @@ -16902,10 +16868,10 @@ webpack-virtual-modules@0.6.2: resolved "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz#057faa9065c8acf48f24cb57ac0e77739ab9a7e8" integrity sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ== -webpack@5.97.1, webpack@^5: - version "5.97.1" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.97.1.tgz#972a8320a438b56ff0f1d94ade9e82eac155fa58" - integrity sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg== +webpack@5.98.0: + version "5.98.0" + resolved "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz#44ae19a8f2ba97537978246072fb89d10d1fbd17" + integrity sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA== dependencies: "@types/eslint-scope" "^3.7.7" "@types/estree" "^1.0.6" @@ -16925,16 +16891,16 @@ webpack@5.97.1, webpack@^5: loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^3.2.0" + schema-utils "^4.3.0" tapable "^2.1.1" - terser-webpack-plugin "^5.3.10" + terser-webpack-plugin "^5.3.11" watchpack "^2.4.1" webpack-sources "^3.2.3" -webpack@5.98.0: - version "5.98.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz#44ae19a8f2ba97537978246072fb89d10d1fbd17" - integrity sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA== +webpack@^5: + version "5.97.1" + resolved "https://registry.npmjs.org/webpack/-/webpack-5.97.1.tgz#972a8320a438b56ff0f1d94ade9e82eac155fa58" + integrity sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg== dependencies: "@types/eslint-scope" "^3.7.7" "@types/estree" "^1.0.6" @@ -16954,9 +16920,9 @@ webpack@5.98.0: loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^4.3.0" + schema-utils "^3.2.0" tapable "^2.1.1" - terser-webpack-plugin "^5.3.11" + terser-webpack-plugin "^5.3.10" watchpack "^2.4.1" webpack-sources "^3.2.3" From adef2fe537dbcb15ce63ce56381224a88a24cc79 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 17 Apr 2025 16:19:47 -0700 Subject: [PATCH 28/29] Stub chrome in positive test --- packages/vertexai/src/methods/chrome-adapter.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts index 8906bf398bf..42a8d723fab 100644 --- a/packages/vertexai/src/methods/chrome-adapter.test.ts +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -175,6 +175,7 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns true if model is readily available', async () => { + const chromeStub = stub(util, 'isChrome').returns(true); const languageModelProvider = { availability: () => Promise.resolve(Availability.available) } as LanguageModel; @@ -187,6 +188,7 @@ describe('ChromeAdapter', () => { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }) ).to.be.true; + chromeStub.restore(); }); it('returns false and triggers download when model is available after download', async () => { const languageModelProvider = { From 9304e472856584bfd93a113427c8d23f6bb3b88f Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 17 Apr 2025 16:38:12 -0700 Subject: [PATCH 29/29] Defer checking for Chrome --- common/api-review/util.api.md | 5 ----- packages/util/src/environment.ts | 6 ------ .../vertexai/src/methods/chrome-adapter.test.ts | 13 ------------- packages/vertexai/src/methods/chrome-adapter.ts | 5 ----- 4 files changed, 29 deletions(-) diff --git a/common/api-review/util.api.md b/common/api-review/util.api.md index fb8afb2319c..8c62ff229ac 100644 --- a/common/api-review/util.api.md +++ b/common/api-review/util.api.md @@ -264,11 +264,6 @@ export function isBrowser(): boolean; // @public (undocumented) export function isBrowserExtension(): boolean; -// Warning: (ae-missing-release-tag) "isChrome" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export function isChrome(): boolean; - // Warning: (ae-missing-release-tag) "isCloudflareWorker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/packages/util/src/environment.ts b/packages/util/src/environment.ts index 50d5f534106..a0467b08c59 100644 --- a/packages/util/src/environment.ts +++ b/packages/util/src/environment.ts @@ -173,12 +173,6 @@ export function isSafari(): boolean { ); } -export function isChrome(): boolean { - return ( - !isNode() && !!navigator.userAgent && navigator.userAgent.includes('Chrome') - ); -} - /** * This method checks if indexedDB is supported by current browser/service worker context * @return true if indexedDB is supported by current browser/service worker context diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts index 42a8d723fab..b11fb9c937e 100644 --- a/packages/vertexai/src/methods/chrome-adapter.test.ts +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -25,7 +25,6 @@ import { LanguageModelCreateOptions } from '../types/language-model'; import { stub } from 'sinon'; -import * as util from '@firebase/util'; import { GenerateContentRequest } from '../types'; use(sinonChai); @@ -41,16 +40,6 @@ describe('ChromeAdapter', () => { }) ).to.be.false; }); - it('returns false if browser is not Chrome', async () => { - const chromeStub = stub(util, 'isChrome').returns(false); - const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); - expect( - await adapter.isAvailable({ - contents: [] - }) - ).to.be.false; - chromeStub.restore(); - }); it('returns false if AI API is undefined', async () => { const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); expect( @@ -175,7 +164,6 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns true if model is readily available', async () => { - const chromeStub = stub(util, 'isChrome').returns(true); const languageModelProvider = { availability: () => Promise.resolve(Availability.available) } as LanguageModel; @@ -188,7 +176,6 @@ describe('ChromeAdapter', () => { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }) ).to.be.true; - chromeStub.restore(); }); it('returns false and triggers download when model is available after download', async () => { const languageModelProvider = { diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index 15d0eb9cdef..10844079c03 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -30,7 +30,6 @@ import { LanguageModelMessageRole, LanguageModelMessageContent } from '../types/language-model'; -import { isChrome } from '@firebase/util'; /** * Defines an inference "backend" that uses Chrome's on-device model, @@ -66,10 +65,6 @@ export class ChromeAdapter { if (this.mode === 'only_in_cloud') { return false; } - // Returns false because only Chrome's experimental Prompt API is supported. - if (!isChrome()) { - return false; - } // Returns false if the on-device inference API is undefined.; if (!this.languageModelProvider) { return false;