diff --git a/.changeset/violet-planets-impress.md b/.changeset/violet-planets-impress.md new file mode 100644 index 00000000000..62427699f26 --- /dev/null +++ b/.changeset/violet-planets-impress.md @@ -0,0 +1,6 @@ +--- +'firebase': minor +'@firebase/vertexai': minor +--- + +**Public Preview** Added support for generating images using the Imagen 3 model. diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 041bc62451f..4521aaf2b70 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -323,7 +323,7 @@ export interface GenerativeContentBlob { } // @public -export class GenerativeModel { +export class GenerativeModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions); countTokens(request: CountTokensRequest | string | Array): Promise; generateContent(request: GenerateContentRequest | string | Array): Promise; @@ -331,8 +331,6 @@ export class GenerativeModel { // (undocumented) generationConfig: GenerationConfig; // (undocumented) - model: string; - // (undocumented) requestOptions?: RequestOptions; // (undocumented) safetySettings: SafetySetting[]; @@ -348,6 +346,9 @@ export class GenerativeModel { // @public export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +// @beta +export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; + // @public export function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI; @@ -429,6 +430,90 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } +// @beta +export enum ImagenAspectRatio { + LANDSCAPE_16x9 = "16:9", + LANDSCAPE_3x4 = "3:4", + PORTRAIT_4x3 = "4:3", + PORTRAIT_9x16 = "9:16", + SQUARE = "1:1" +} + +// @public +export interface ImagenGCSImage { + gcsURI: string; + mimeType: string; +} + +// @beta +export interface ImagenGenerationConfig { + addWatermark?: boolean; + aspectRatio?: ImagenAspectRatio; + imageFormat?: ImagenImageFormat; + negativePrompt?: string; + numberOfImages?: number; +} + +// @beta +export interface ImagenGenerationResponse { + filteredReason?: string; + images: T[]; +} + +// @beta +export class ImagenImageFormat { + compressionQuality?: number; + static jpeg(compressionQuality?: number): ImagenImageFormat; + mimeType: string; + static png(): ImagenImageFormat; +} + +// @beta +export interface ImagenInlineImage { + bytesBase64Encoded: string; + mimeType: string; +} + +// @beta +export class ImagenModel extends VertexAIModel { + constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); + generateImages(prompt: string): Promise>; + // @internal + generateImagesGCS(prompt: string, gcsURI: string): Promise>; + generationConfig?: ImagenGenerationConfig; + // (undocumented) + requestOptions?: RequestOptions | undefined; + safetySettings?: ImagenSafetySettings; +} + +// @beta +export interface ImagenModelParams { + generationConfig?: ImagenGenerationConfig; + model: string; + safetySettings?: ImagenSafetySettings; +} + +// @beta +export enum ImagenPersonFilterLevel { + ALLOW_ADULT = "allow_adult", + ALLOW_ALL = "allow_all", + BLOCK_ALL = "dont_allow" +} + +// @beta +export enum ImagenSafetyFilterLevel { + BLOCK_LOW_AND_ABOVE = "block_low_and_above", + BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", + BLOCK_NONE = "block_none", + BLOCK_ONLY_HIGH = "block_only_high" +} + +// @beta +export interface ImagenSafetySettings { + personFilterLevel?: ImagenPersonFilterLevel; + safetyFilterLevel?: ImagenSafetyFilterLevel; +} + // @public export interface InlineDataPart { // (undocumented) @@ -718,6 +803,16 @@ export const enum VertexAIErrorCode { RESPONSE_ERROR = "response-error" } +// @public +export abstract class VertexAIModel { + // @internal + protected constructor(vertexAI: VertexAI, modelName: string); + // @internal (undocumented) + protected _apiSettings: ApiSettings; + readonly model: string; + static normalizeModelName(modelName: string): string; +} + // @public export interface VertexAIOptions { // (undocumented) diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index bf0318389ba..d97377e3d3d 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -536,6 +536,22 @@ toc: path: /docs/reference/js/vertexai.groundingattribution.md - title: GroundingMetadata path: /docs/reference/js/vertexai.groundingmetadata.md + - title: ImagenGCSImage + path: /docs/reference/js/vertexai.imagengcsimage.md + - title: ImagenGenerationConfig + path: /docs/reference/js/vertexai.imagengenerationconfig.md + - title: ImagenGenerationResponse + path: /docs/reference/js/vertexai.imagengenerationresponse.md + - title: ImagenImageFormat + path: /docs/reference/js/vertexai.imagenimageformat.md + - title: ImagenInlineImage + path: /docs/reference/js/vertexai.imageninlineimage.md + - title: ImagenModel + path: /docs/reference/js/vertexai.imagenmodel.md + - title: ImagenModelParams + path: /docs/reference/js/vertexai.imagenmodelparams.md + - title: ImagenSafetySettings + path: /docs/reference/js/vertexai.imagensafetysettings.md - title: InlineDataPart path: /docs/reference/js/vertexai.inlinedatapart.md - title: IntegerSchema @@ -584,6 +600,8 @@ toc: path: /docs/reference/js/vertexai.vertexai.md - title: VertexAIError path: /docs/reference/js/vertexai.vertexaierror.md + - title: VertexAIModel + path: /docs/reference/js/vertexai.vertexaimodel.md - title: VertexAIOptions path: /docs/reference/js/vertexai.vertexaioptions.md - title: VideoMetadata diff --git a/docs-devsite/vertexai.generativemodel.md b/docs-devsite/vertexai.generativemodel.md index 7105f9c100b..b734e241e78 100644 --- a/docs-devsite/vertexai.generativemodel.md +++ b/docs-devsite/vertexai.generativemodel.md @@ -15,8 +15,9 @@ Class for generative model APIs. Signature: ```typescript -export declare class GenerativeModel +export declare class GenerativeModel extends VertexAIModel ``` +Extends: [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) ## Constructors @@ -29,7 +30,6 @@ export declare class GenerativeModel | Property | Modifiers | Type | Description | | --- | --- | --- | --- | | [generationConfig](./vertexai.generativemodel.md#generativemodelgenerationconfig) | | [GenerationConfig](./vertexai.generationconfig.md#generationconfig_interface) | | -| [model](./vertexai.generativemodel.md#generativemodelmodel) | | string | | | [requestOptions](./vertexai.generativemodel.md#generativemodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | | [safetySettings](./vertexai.generativemodel.md#generativemodelsafetysettings) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface)\[\] | | | [systemInstruction](./vertexai.generativemodel.md#generativemodelsysteminstruction) | | [Content](./vertexai.content.md#content_interface) | | @@ -71,14 +71,6 @@ constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: Reque generationConfig: GenerationConfig; ``` -## GenerativeModel.model - -Signature: - -```typescript -model: string; -``` - ## GenerativeModel.requestOptions Signature: diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md new file mode 100644 index 00000000000..b094e63c1d8 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -0,0 +1,54 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImage interface +An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. + +This feature is not available yet. + +Signature: + +```typescript +export interface ImagenGCSImage +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The URI of the file stored in a Cloud Storage for Firebase bucket. | +| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +## ImagenGCSImage.gcsURI + +The URI of the file stored in a Cloud Storage for Firebase bucket. + +Signature: + +```typescript +gcsURI: string; +``` + +### Example + +`"gs://bucket-name/path/sample_0.jpg"`. + +## ImagenGCSImage.mimeType + +The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + +To request a different format, set the `imageFormat` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md new file mode 100644 index 00000000000..cee7734f789 --- /dev/null +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -0,0 +1,111 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGenerationConfig interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Configuration options for generating images with Imagen. + +See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. + +Signature: + +```typescript +export interface ImagenGenerationConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [addWatermark](./vertexai.imagengenerationconfig.md#imagengenerationconfigaddwatermark) | boolean | (Public Preview) Whether to add an invisible watermark to generated images.If set to true, an invisible SynthID watermark is embedded in generated images to indicate that they are AI generated. If set to false, watermarking will be disabled.For Imagen 3 models, the default value is true; see the addWatermark documentation for more details. | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) The aspect ratio of the generated images. The default value is square 1:1. Supported aspect ratios depend on the Imagen model, see [ImagenAspectRatio](./vertexai.md#imagenaspectratio) for more details. | +| [imageFormat](./vertexai.imagengenerationconfig.md#imagengenerationconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | (Public Preview) The image format of the generated images. The default is PNG.See [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for more details. | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | (Public Preview) A description of what should be omitted from the generated images.Support for negative prompts depends on the Imagen model.See the [documentation](http://firebase.google.com/docs/vertex-ai/model-parameters#imagen) for more details. | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | (Public Preview) The number of images to generate. The default value is 1.The number of sample images that may be generated in each request depends on the model (typically up to 4); see the sampleCount documentation for more details. | + +## ImagenGenerationConfig.addWatermark + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Whether to add an invisible watermark to generated images. + +If set to `true`, an invisible SynthID watermark is embedded in generated images to indicate that they are AI generated. If set to `false`, watermarking will be disabled. + +For Imagen 3 models, the default value is `true`; see the addWatermark documentation for more details. + +Signature: + +```typescript +addWatermark?: boolean; +``` + +## ImagenGenerationConfig.aspectRatio + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The aspect ratio of the generated images. The default value is square 1:1. Supported aspect ratios depend on the Imagen model, see [ImagenAspectRatio](./vertexai.md#imagenaspectratio) for more details. + +Signature: + +```typescript +aspectRatio?: ImagenAspectRatio; +``` + +## ImagenGenerationConfig.imageFormat + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The image format of the generated images. The default is PNG. + +See [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for more details. + +Signature: + +```typescript +imageFormat?: ImagenImageFormat; +``` + +## ImagenGenerationConfig.negativePrompt + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A description of what should be omitted from the generated images. + +Support for negative prompts depends on the Imagen model. + +See the [documentation](http://firebase.google.com/docs/vertex-ai/model-parameters#imagen) for more details. + +Signature: + +```typescript +negativePrompt?: string; +``` + +## ImagenGenerationConfig.numberOfImages + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The number of images to generate. The default value is 1. + +The number of sample images that may be generated in each request depends on the model (typically up to 4); see the sampleCount documentation for more details. + +Signature: + +```typescript +numberOfImages?: number; +``` diff --git a/docs-devsite/vertexai.imagengenerationresponse.md b/docs-devsite/vertexai.imagengenerationresponse.md new file mode 100644 index 00000000000..32ed69718f9 --- /dev/null +++ b/docs-devsite/vertexai.imagengenerationresponse.md @@ -0,0 +1,61 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGenerationResponse interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The response from a request to generate images with Imagen. + +Signature: + +```typescript +export interface ImagenGenerationResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagengenerationresponse.md#imagengenerationresponsefilteredreason) | string | (Public Preview) The reason that images were filtered out. This property will only be defined if one or more images were filtered.Images may be filtered out due to the [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel), [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel), or filtering included in the model. The filter levels may be adjusted in your [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface).See the [Responsible AI and usage guidelines for Imagen](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen) for more details. | +| [images](./vertexai.imagengenerationresponse.md#imagengenerationresponseimages) | T\[\] | (Public Preview) The images generated by Imagen.The number of images generated may be fewer than the number requested if one or more were filtered out; see filteredReason. | + +## ImagenGenerationResponse.filteredReason + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The reason that images were filtered out. This property will only be defined if one or more images were filtered. + +Images may be filtered out due to the [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel), [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel), or filtering included in the model. The filter levels may be adjusted in your [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface). + +See the [Responsible AI and usage guidelines for Imagen](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen) for more details. + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenGenerationResponse.images + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The images generated by Imagen. + +The number of images generated may be fewer than the number requested if one or more were filtered out; see `filteredReason`. + +Signature: + +```typescript +images: T[]; +``` diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md new file mode 100644 index 00000000000..785c7c726fc --- /dev/null +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -0,0 +1,119 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageFormat class +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Defines the image format for images generated by Imagen. + +Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare class ImagenImageFormat +``` + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | (Public Preview) The level of compression (a number between 0 and 100). | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | (Public Preview) The MIME type. | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | (Public Preview) Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a JPEG image. | +| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | (Public Preview) Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a PNG image. | + +## ImagenImageFormat.compressionQuality + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The level of compression (a number between 0 and 100). + +Signature: + +```typescript +compressionQuality?: number; +``` + +## ImagenImageFormat.mimeType + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The MIME type. + +Signature: + +```typescript +mimeType: string; +``` + +## ImagenImageFormat.jpeg() + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a JPEG image. + +Signature: + +```typescript +static jpeg(compressionQuality?: number): ImagenImageFormat; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| compressionQuality | number | The level of compression (a number between 0 and 100). | + +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) + +An [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) object for a JPEG image. + +## ImagenImageFormat.png() + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a PNG image. + +Signature: + +```typescript +static png(): ImagenImageFormat; +``` +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) + +An [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) object for a PNG image. + +### Example + + +```javascript +const imagenModelParams = { + // ... other ImagenModelParams + imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. +} + +``` + diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md new file mode 100644 index 00000000000..19fe8a67764 --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -0,0 +1,57 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImage interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +An image generated by Imagen, represented as inline data. + +Signature: + +```typescript +export interface ImagenInlineImage +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | (Public Preview) The base64-encoded image data. | +| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | (Public Preview) The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +## ImagenInlineImage.bytesBase64Encoded + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The base64-encoded image data. + +Signature: + +```typescript +bytesBase64Encoded: string; +``` + +## ImagenInlineImage.mimeType + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + +To request a different format, set the `imageFormat` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md new file mode 100644 index 00000000000..63e15ff133a --- /dev/null +++ b/docs-devsite/vertexai.imagenmodel.md @@ -0,0 +1,157 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModel class +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Class for Imagen model APIs. + +This class provides methods for generating images using the Imagen model. + +Signature: + +```typescript +export declare class ImagenModel extends VertexAIModel +``` +Extends: [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) + +## Constructors + +| Constructor | Modifiers | Description | +| --- | --- | --- | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | (Public Preview) Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. | + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [generationConfig](./vertexai.imagenmodel.md#imagenmodelgenerationconfig) | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) The Imagen generation configuration. | +| [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | (Public Preview) | +| [safetySettings](./vertexai.imagenmodel.md#imagenmodelsafetysettings) | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Safety settings for filtering inappropriate content. | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [generateImages(prompt)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | (Public Preview) Generates images using the Imagen model and returns them as base64-encoded strings. | + +## ImagenModel.(constructor) + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. + +Signature: + +```typescript +constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making requests to Imagen. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | Additional options to use when making requests. | + +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. + +## ImagenModel.generationConfig + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The Imagen generation configuration. + +Signature: + +```typescript +generationConfig?: ImagenGenerationConfig; +``` + +## ImagenModel.requestOptions + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Signature: + +```typescript +requestOptions?: RequestOptions | undefined; +``` + +## ImagenModel.safetySettings + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Safety settings for filtering inappropriate content. + +Signature: + +```typescript +safetySettings?: ImagenSafetySettings; +``` + +## ImagenModel.generateImages() + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If the prompt was not blocked, but one or more of the generated images were filtered, the returned object will have a `filteredReason` property. If all images are filtered, the `images` array will be empty. + +Signature: + +```typescript +generateImages(prompt: string): Promise>; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | A text prompt describing the image(s) to generate. | + +Returns: + +Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)>> + +A promise that resolves to an [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) object containing the generated images. + +#### Exceptions + +If the request to generate images fails. This happens if the prompt is blocked. + +### Example + + +```javascript +const imagen = new ImagenModel( + vertexAI, + { + model: 'imagen-3.0-generate-002' + } +); + +const response = await imagen.generateImages('A photo of a cat'); +if (response.images.length > 0) { + console.log(response.images[0].bytesBase64Encoded); +} + +``` + diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md new file mode 100644 index 00000000000..66c4bb0bfe6 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -0,0 +1,73 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelParams interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). + +Signature: + +```typescript +export interface ImagenModelParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [generationConfig](./vertexai.imagenmodelparams.md#imagenmodelparamsgenerationconfig) | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen. | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | (Public Preview) The Imagen model to use for generating images. For example: imagen-3.0-generate-002.Only Imagen 3 models (named imagen-3.0-*) are supported.See [model versions](https://firebase.google.com/docs/vertex-ai/models) for a full list of supported Imagen 3 models. | +| [safetySettings](./vertexai.imagenmodelparams.md#imagenmodelparamssafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Safety settings for filtering potentially inappropriate content. | + +## ImagenModelParams.generationConfig + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Configuration options for generating images with Imagen. + +Signature: + +```typescript +generationConfig?: ImagenGenerationConfig; +``` + +## ImagenModelParams.model + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The Imagen model to use for generating images. For example: `imagen-3.0-generate-002`. + +Only Imagen 3 models (named `imagen-3.0-*`) are supported. + +See [model versions](https://firebase.google.com/docs/vertex-ai/models) for a full list of supported Imagen 3 models. + +Signature: + +```typescript +model: string; +``` + +## ImagenModelParams.safetySettings + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Safety settings for filtering potentially inappropriate content. + +Signature: + +```typescript +safetySettings?: ImagenSafetySettings; +``` diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md new file mode 100644 index 00000000000..3cf7931a959 --- /dev/null +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -0,0 +1,57 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenSafetySettings interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Settings for controlling the aggressiveness of filtering out sensitive content. + +See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. + +Signature: + +```typescript +export interface ImagenSafetySettings +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressive to filter out sensitive content from generated images. | + +## ImagenSafetySettings.personFilterLevel + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling whether generation of images containing people or faces is allowed. + +Signature: + +```typescript +personFilterLevel?: ImagenPersonFilterLevel; +``` + +## ImagenSafetySettings.safetyFilterLevel + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling how aggressive to filter out sensitive content from generated images. + +Signature: + +```typescript +safetyFilterLevel?: ImagenSafetyFilterLevel; +``` diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index d9e26eabc5d..1d284345c39 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -20,6 +20,7 @@ The Vertex AI in Firebase Web SDK. | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | (Public Preview) Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.Only Imagen 3 models (named imagen-3.0-*) are supported. | ## Classes @@ -29,12 +30,15 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | (Public Preview) Defines the image format for images generated by Imagen.Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | (Public Preview) Class for Imagen model APIs.This class provides methods for generating images using the Imagen model. | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | | [Schema](./vertexai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with JSON.stringify() into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) | | [StringSchema](./vertexai.stringschema.md#stringschema_class) | Schema class for "string" types. Can be used with or without enum values. | | [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) | Error class for the Vertex AI in Firebase SDK. | +| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | Base class for Vertex AI in Firebase model APIs. | ## Enumerations @@ -48,6 +52,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface).See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -83,6 +90,12 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | +| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | (Public Preview) An image generated by Imagen, represented as inline data. | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | @@ -167,6 +180,37 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) +### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. + +Only Imagen 3 models (named `imagen-3.0-*`) are supported. + +Signature: + +```typescript +export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making Imagen requests. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Additional options to use when making requests. | + +Returns: + +[ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) + +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. + ## POSSIBLE\_ROLES Possible roles. @@ -363,6 +407,80 @@ export declare enum HarmSeverity | HARM\_SEVERITY\_MEDIUM | "HARM_SEVERITY_MEDIUM" | | | HARM\_SEVERITY\_NEGLIGIBLE | "HARM_SEVERITY_NEGLIGIBLE" | | +## ImagenAspectRatio + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Aspect ratios for Imagen images. + +To specify an aspect ratio for generated images, set the `aspectRatio` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + +See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. + +Signature: + +```typescript +export declare enum ImagenAspectRatio +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| LANDSCAPE\_16x9 | "16:9" | (Public Preview) Landscape (16:9) aspect ratio. | +| LANDSCAPE\_3x4 | "3:4" | (Public Preview) Landscape (3:4) aspect ratio. | +| PORTRAIT\_4x3 | "4:3" | (Public Preview) Portrait (4:3) aspect ratio. | +| PORTRAIT\_9x16 | "9:16" | (Public Preview) Portrait (9:16) aspect ratio. | +| SQUARE | "1:1" | (Public Preview) Square (1:1) aspect ratio. | + +## ImagenPersonFilterLevel + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling whether generation of images containing people or faces is allowed. + +See the personGeneration documentation for more details. + +Signature: + +```typescript +export declare enum ImagenPersonFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ALLOW\_ADULT | "allow_adult" | (Public Preview) Allow generation of images containing adults only; images of children are filtered out.Generation of images containing people or faces may require your use case to be reviewed and approved by Cloud support; see the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) for more details. | +| ALLOW\_ALL | "allow_all" | (Public Preview) Allow generation of images containing adults only; images of children are filtered out.Generation of images containing people or faces may require your use case to be reviewed and approved by Cloud support; see the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) for more details. | +| BLOCK\_ALL | "dont_allow" | (Public Preview) Disallow generation of images containing people or faces; images of people are filtered out. | + +## ImagenSafetyFilterLevel + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling how aggressively to filter sensitive content. + +Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, `violence`, `sexual`, `derogatory`, and `toxic`). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. + +Signature: + +```typescript +export declare enum ImagenSafetyFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | (Public Preview) The most aggressive filtering level; most strict blocking. | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | (Public Preview) Blocks some sensitive prompts and responses. | +| BLOCK\_NONE | "block_none" | (Public Preview) The least aggressive filtering level; blocks very few sensitive prompts and responses.Access to this feature is restricted and may require your case to be reviewed and approved by Cloud support. | +| BLOCK\_ONLY\_HIGH | "block_only_high" | (Public Preview) Blocks few sensitive prompts and responses. | + ## SchemaType Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) diff --git a/docs-devsite/vertexai.vertexaimodel.md b/docs-devsite/vertexai.vertexaimodel.md new file mode 100644 index 00000000000..5c3244fe1e5 --- /dev/null +++ b/docs-devsite/vertexai.vertexaimodel.md @@ -0,0 +1,66 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# VertexAIModel class +Base class for Vertex AI in Firebase model APIs. + +The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `VertexAIModel` class. + +Signature: + +```typescript +export declare abstract class VertexAIModel +``` + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | The fully qualified model resource name to use for generating images (for example, publishers/google/models/imagen-3.0-generate-002). | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [normalizeModelName(modelName)](./vertexai.vertexaimodel.md#vertexaimodelnormalizemodelname) | static | Normalizes the given model name to a fully qualified model resource name. | + +## VertexAIModel.model + +The fully qualified model resource name to use for generating images (for example, `publishers/google/models/imagen-3.0-generate-002`). + +Signature: + +```typescript +readonly model: string; +``` + +## VertexAIModel.normalizeModelName() + +Normalizes the given model name to a fully qualified model resource name. + +Signature: + +```typescript +static normalizeModelName(modelName: string): string; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| modelName | string | The model name to normalize. | + +Returns: + +string + +The fully qualified model resource name. + diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index b6c96923856..c1b2635ce70 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ModelParams, VertexAIErrorCode } from './types'; +import { ImagenModelParams, ModelParams, VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; -import { getGenerativeModel } from './api'; +import { ImagenModel, getGenerativeModel, getImagenModel } from './api'; import { expect } from 'chai'; import { VertexAI } from './public-types'; import { GenerativeModel } from './models/generative-model'; @@ -84,4 +84,54 @@ describe('Top level API', () => { expect(genModel).to.be.an.instanceOf(GenerativeModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); }); + it('getImagenModel throws if no model is provided', () => { + try { + getImagenModel(fakeVertexAI, {} as ImagenModelParams); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_MODEL); + expect((e as VertexAIError).message).includes( + `VertexAI: Must provide a model name. Example: ` + + `getImagenModel({ model: 'my-model-name' }) (vertexAI/${VertexAIErrorCode.NO_MODEL})` + ); + } + }); + it('getImagenModel throws if no apiKey is provided', () => { + const fakeVertexNoApiKey = { + ...fakeVertexAI, + app: { options: { projectId: 'my-project' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoApiKey, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_API_KEY); + expect((e as VertexAIError).message).equals( + `VertexAI: The "apiKey" field is empty in the local ` + + `Firebase config. Firebase VertexAI requires this field to` + + ` contain a valid API key. (vertexAI/${VertexAIErrorCode.NO_API_KEY})` + ); + } + }); + it('getImagenModel throws if no projectId is provided', () => { + const fakeVertexNoProject = { + ...fakeVertexAI, + app: { options: { apiKey: 'my-key' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoProject, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes( + VertexAIErrorCode.NO_PROJECT_ID + ); + expect((e as VertexAIError).message).equals( + `VertexAI: The "projectId" field is empty in the local` + + ` Firebase config. Firebase VertexAI requires this field ` + + `to contain a valid project ID. (vertexAI/${VertexAIErrorCode.NO_PROJECT_ID})` + ); + } + }); + it('getImagenModel gets an ImagenModel', () => { + const genModel = getImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(genModel).to.be.an.instanceOf(ImagenModel); + expect(genModel.model).to.equal('publishers/google/models/my-model'); + }); }); diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 42f33ed2a85..752e75c7e23 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -21,15 +21,19 @@ import { getModularInstance } from '@firebase/util'; import { DEFAULT_LOCATION, VERTEX_TYPE } from './constants'; import { VertexAIService } from './service'; import { VertexAI, VertexAIOptions } from './public-types'; -import { ModelParams, RequestOptions, VertexAIErrorCode } from './types'; +import { + ImagenModelParams, + ModelParams, + RequestOptions, + VertexAIErrorCode +} from './types'; import { VertexAIError } from './errors'; -import { GenerativeModel } from './models/generative-model'; +import { VertexAIModel, GenerativeModel, ImagenModel } from './models'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; - -export { GenerativeModel }; - +export { ImagenImageFormat } from './requests/imagen-image-format'; +export { VertexAIModel, GenerativeModel, ImagenModel }; export { VertexAIError }; declare module '@firebase/component' { @@ -77,3 +81,31 @@ export function getGenerativeModel( } return new GenerativeModel(vertexAI, modelParams, requestOptions); } + +/** + * Returns an {@link ImagenModel} class with methods for using Imagen. + * + * Only Imagen 3 models (named `imagen-3.0-*`) are supported. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelParams - Parameters to use when making Imagen requests. + * @param requestOptions - Additional options to use when making requests. + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + * + * @beta + */ +export function getImagenModel( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + requestOptions?: RequestOptions +): ImagenModel { + if (!modelParams.model) { + throw new VertexAIError( + VertexAIErrorCode.NO_MODEL, + `Must provide a model name. Example: getImagenModel({ model: 'my-model-name' })` + ); + } + return new ImagenModel(vertexAI, modelParams, requestOptions); +} diff --git a/packages/vertexai/src/models/generative-model.test.ts b/packages/vertexai/src/models/generative-model.test.ts index e03f39e8a83..c2dbdfac75c 100644 --- a/packages/vertexai/src/models/generative-model.test.ts +++ b/packages/vertexai/src/models/generative-model.test.ts @@ -37,28 +37,6 @@ const fakeVertexAI: VertexAI = { }; describe('GenerativeModel', () => { - it('handles plain model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles models/ prefixed model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'models/my-model' - }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles full model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'publishers/google/models/my-model' - }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles prefixed tuned model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'tunedModels/my-model' - }); - expect(genModel.model).to.equal('tunedModels/my-model'); - }); it('passes params through to generateContent', async () => { const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model', diff --git a/packages/vertexai/src/models/generative-model.ts b/packages/vertexai/src/models/generative-model.ts index b44dc1131a3..b4cf464f025 100644 --- a/packages/vertexai/src/models/generative-model.ts +++ b/packages/vertexai/src/models/generative-model.ts @@ -33,10 +33,8 @@ import { SafetySetting, StartChatParams, Tool, - ToolConfig, - VertexAIErrorCode + ToolConfig } from '../types'; -import { VertexAIError } from '../errors'; import { ChatSession } from '../methods/chat-session'; import { countTokens } from '../methods/count-tokens'; import { @@ -44,17 +42,13 @@ import { formatSystemInstruction } from '../requests/request-helpers'; import { VertexAI } from '../public-types'; -import { ApiSettings } from '../types/internal'; -import { VertexAIService } from '../service'; -import { _isFirebaseServerApp } from '@firebase/app'; +import { VertexAIModel } from './vertexai-model'; /** * Class for generative model APIs. * @public */ -export class GenerativeModel { - private _apiSettings: ApiSettings; - model: string; +export class GenerativeModel extends VertexAIModel { generationConfig: GenerationConfig; safetySettings: SafetySetting[]; requestOptions?: RequestOptions; @@ -67,53 +61,7 @@ export class GenerativeModel { modelParams: ModelParams, requestOptions?: RequestOptions ) { - if (!vertexAI.app?.options?.apiKey) { - throw new VertexAIError( - VertexAIErrorCode.NO_API_KEY, - `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` - ); - } else if (!vertexAI.app?.options?.projectId) { - throw new VertexAIError( - VertexAIErrorCode.NO_PROJECT_ID, - `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` - ); - } else { - this._apiSettings = { - apiKey: vertexAI.app.options.apiKey, - project: vertexAI.app.options.projectId, - location: vertexAI.location - }; - - if ( - _isFirebaseServerApp(vertexAI.app) && - vertexAI.app.settings.appCheckToken - ) { - const token = vertexAI.app.settings.appCheckToken; - this._apiSettings.getAppCheckToken = () => { - return Promise.resolve({ token }); - }; - } else if ((vertexAI as VertexAIService).appCheck) { - this._apiSettings.getAppCheckToken = () => - (vertexAI as VertexAIService).appCheck!.getToken(); - } - - if ((vertexAI as VertexAIService).auth) { - this._apiSettings.getAuthToken = () => - (vertexAI as VertexAIService).auth!.getToken(); - } - } - if (modelParams.model.includes('/')) { - if (modelParams.model.startsWith('models/')) { - // Add "publishers/google" if the user is only passing in 'models/model-name'. - this.model = `publishers/google/${modelParams.model}`; - } else { - // Any other custom format (e.g. tuned models) must be passed in correctly. - this.model = modelParams.model; - } - } else { - // If path is not included, assume it's a non-tuned model. - this.model = `publishers/google/models/${modelParams.model}`; - } + super(vertexAI, modelParams.model); this.generationConfig = modelParams.generationConfig || {}; this.safetySettings = modelParams.safetySettings || []; this.tools = modelParams.tools; diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts new file mode 100644 index 00000000000..000b2f07f90 --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -0,0 +1,160 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { ImagenModel } from './imagen-model'; +import { + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel, + VertexAI, + VertexAIErrorCode +} from '../public-types'; +import * as request from '../requests/request'; +import sinonChai from 'sinon-chai'; +import { VertexAIError } from '../errors'; +import { getMockResponse } from '../../test-utils/mock-response'; +import { match, restore, stub } from 'sinon'; + +use(sinonChai); + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('ImagenModel', () => { + it('generateImages makes a request to predict with default parameters', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes(`"prompt":"${prompt}"`) && + value.includes(`"sampleCount":1`) + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with generation config and safety settings', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model', + generationConfig: { + negativePrompt: 'do not hallucinate', + numberOfImages: 4, + aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, + imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + addWatermark: true + }, + safetySettings: { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + } + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes( + `"negativePrompt":"${imagenModel.generationConfig?.negativePrompt}"` + ) && + value.includes( + `"sampleCount":${imagenModel.generationConfig?.numberOfImages}` + ) && + value.includes( + `"aspectRatio":"${imagenModel.generationConfig?.aspectRatio}"` + ) && + value.includes( + JSON.stringify(imagenModel.generationConfig?.imageFormat?.mimeType) + ) && + value.includes( + JSON.stringify(imagenModel.generationConfig?.addWatermark) + ) && + value.includes( + JSON.stringify(imagenModel.safetySettings?.safetyFilterLevel) + ) && + value.includes( + JSON.stringify(imagenModel.safetySettings?.personFilterLevel) + ) + ); + }), + undefined + ); + restore(); + }); + it('throws if prompt blocked', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-prompt-blocked.json' + ); + + stub(globalThis, 'fetch').resolves({ + ok: false, + status: 400, + statusText: 'Bad Request', + json: mockResponse.json + } as Response); + + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + try { + await imagenModel.generateImages('some inappropriate prompt.'); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); + expect((e as VertexAIError).message).to.include('400'); + expect((e as VertexAIError).message).to.include( + "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback." + ); + } finally { + restore(); + } + }); +}); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts new file mode 100644 index 00000000000..89c740852a3 --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.ts @@ -0,0 +1,160 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAI } from '../public-types'; +import { Task, makeRequest } from '../requests/request'; +import { createPredictRequestBody } from '../requests/request-helpers'; +import { handlePredictResponse } from '../requests/response-helpers'; +import { + ImagenGCSImage, + ImagenGenerationConfig, + ImagenInlineImage, + RequestOptions, + ImagenModelParams, + ImagenGenerationResponse, + ImagenSafetySettings +} from '../types'; +import { VertexAIModel } from './vertexai-model'; + +/** + * Class for Imagen model APIs. + * + * This class provides methods for generating images using the Imagen model. + * + * @example + * ```javascript + * const imagen = new ImagenModel( + * vertexAI, + * { + * model: 'imagen-3.0-generate-002' + * } + * ); + * + * const response = await imagen.generateImages('A photo of a cat'); + * if (response.images.length > 0) { + * console.log(response.images[0].bytesBase64Encoded); + * } + * ``` + * + * @beta + */ +export class ImagenModel extends VertexAIModel { + /** + * The Imagen generation configuration. + */ + generationConfig?: ImagenGenerationConfig; + /** + * Safety settings for filtering inappropriate content. + */ + safetySettings?: ImagenSafetySettings; + + /** + * Constructs a new instance of the {@link ImagenModel} class. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelParams - Parameters to use when making requests to Imagen. + * @param requestOptions - Additional options to use when making requests. + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + */ + constructor( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + public requestOptions?: RequestOptions + ) { + const { model, generationConfig, safetySettings } = modelParams; + super(vertexAI, model); + this.generationConfig = generationConfig; + this.safetySettings = safetySettings; + } + + /** + * Generates images using the Imagen model and returns them as + * base64-encoded strings. + * + * @param prompt - A text prompt describing the image(s) to generate. + * @returns A promise that resolves to an {@link ImagenGenerationResponse} + * object containing the generated images. + * + * @throws If the request to generate images fails. This happens if the + * prompt is blocked. + * + * @remarks + * If the prompt was not blocked, but one or more of the generated images were filtered, the + * returned object will have a `filteredReason` property. + * If all images are filtered, the `images` array will be empty. + * + * @beta + */ + async generateImages( + prompt: string + ): Promise> { + const body = createPredictRequestBody(prompt, { + ...this.generationConfig, + ...this.safetySettings + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } + + /** + * Generates images to Cloud Storage for Firebase using the Imagen model. + * + * @internal This method is temporarily internal. + * + * @param prompt - A text prompt describing the image(s) to generate. + * @param gcsURI - The URI of file stored in a Cloud Storage for Firebase bucket. + * This should be a directory. For example, `gs://my-bucket/my-directory/`. + * @returns A promise that resolves to an {@link ImagenGenerationResponse} + * object containing the URLs of the generated images. + * + * @throws If the request fails to generate images fails. This happens if + * the prompt is blocked. + * + * @remarks + * If the prompt was not blocked, but one or more of the generated images were filtered, the + * returned object will have a `filteredReason` property. + * If all images are filtered, the `images` array will be empty. + */ + async generateImagesGCS( + prompt: string, + gcsURI: string + ): Promise> { + const body = createPredictRequestBody(prompt, { + gcsURI, + ...this.generationConfig, + ...this.safetySettings + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } +} diff --git a/packages/vertexai/src/models/index.ts b/packages/vertexai/src/models/index.ts new file mode 100644 index 00000000000..aec06be26fd --- /dev/null +++ b/packages/vertexai/src/models/index.ts @@ -0,0 +1,20 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './vertexai-model'; +export * from './generative-model'; +export * from './imagen-model'; diff --git a/packages/vertexai/src/models/vertexai-model.test.ts b/packages/vertexai/src/models/vertexai-model.test.ts new file mode 100644 index 00000000000..2aa36d56f0d --- /dev/null +++ b/packages/vertexai/src/models/vertexai-model.test.ts @@ -0,0 +1,103 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { VertexAI, VertexAIErrorCode } from '../public-types'; +import sinonChai from 'sinon-chai'; +import { VertexAIModel } from './vertexai-model'; +import { VertexAIError } from '../errors'; + +use(sinonChai); + +/** + * A class that extends VertexAIModel that allows us to test the protected constructor. + */ +class TestModel extends VertexAIModel { + /* eslint-disable @typescript-eslint/no-useless-constructor */ + constructor(vertexAI: VertexAI, modelName: string) { + super(vertexAI, modelName); + } +} + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('VertexAIModel', () => { + it('handles plain model name', () => { + const testModel = new TestModel(fakeVertexAI, 'my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles models/ prefixed model name', () => { + const testModel = new TestModel(fakeVertexAI, 'models/my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles full model name', () => { + const testModel = new TestModel( + fakeVertexAI, + 'publishers/google/models/my-model' + ); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles prefixed tuned model name', () => { + const testModel = new TestModel(fakeVertexAI, 'tunedModels/my-model'); + expect(testModel.model).to.equal('tunedModels/my-model'); + }); + it('throws if not passed an api key', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + projectId: 'my-project' + } + }, + location: 'us-central1' + }; + try { + new TestModel(fakeVertexAI, 'my-model'); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); + } + }); + it('throws if not passed a project ID', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key' + } + }, + location: 'us-central1' + }; + try { + new TestModel(fakeVertexAI, 'my-model'); + } catch (e) { + expect((e as VertexAIError).code).to.equal( + VertexAIErrorCode.NO_PROJECT_ID + ); + } + }); +}); diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts new file mode 100644 index 00000000000..4e211c0cf94 --- /dev/null +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -0,0 +1,121 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAIError } from '../errors'; +import { VertexAI, VertexAIErrorCode } from '../public-types'; +import { VertexAIService } from '../service'; +import { ApiSettings } from '../types/internal'; +import { _isFirebaseServerApp } from '@firebase/app'; + +/** + * Base class for Vertex AI in Firebase model APIs. + * + * @public + */ +export abstract class VertexAIModel { + /** + * The fully qualified model resource name to use for generating images + * (for example, `publishers/google/models/imagen-3.0-generate-002`). + */ + readonly model: string; + + /** + * @internal + */ + protected _apiSettings: ApiSettings; + + /** + * Constructs a new instance of the {@link VertexAIModel} class. + * + * This constructor should only be called from subclasses that provide + * a model API. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelName - The name of the model being used. It can be in one of the following formats: + * - `my-model` (short name, will resolve to `publishers/google/models/my-model`) + * - `models/my-model` (will resolve to `publishers/google/models/my-model`) + * - `publishers/my-publisher/models/my-model` (fully qualified model name) + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + * + * @internal + */ + protected constructor(vertexAI: VertexAI, modelName: string) { + this.model = VertexAIModel.normalizeModelName(modelName); + + if (!vertexAI.app?.options?.apiKey) { + throw new VertexAIError( + VertexAIErrorCode.NO_API_KEY, + `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` + ); + } else if (!vertexAI.app?.options?.projectId) { + throw new VertexAIError( + VertexAIErrorCode.NO_PROJECT_ID, + `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` + ); + } else { + this._apiSettings = { + apiKey: vertexAI.app.options.apiKey, + project: vertexAI.app.options.projectId, + location: vertexAI.location + }; + + if ( + _isFirebaseServerApp(vertexAI.app) && + vertexAI.app.settings.appCheckToken + ) { + const token = vertexAI.app.settings.appCheckToken; + this._apiSettings.getAppCheckToken = () => { + return Promise.resolve({ token }); + }; + } else if ((vertexAI as VertexAIService).appCheck) { + this._apiSettings.getAppCheckToken = () => + (vertexAI as VertexAIService).appCheck!.getToken(); + } + + if ((vertexAI as VertexAIService).auth) { + this._apiSettings.getAuthToken = () => + (vertexAI as VertexAIService).auth!.getToken(); + } + } + } + + /** + * Normalizes the given model name to a fully qualified model resource name. + * + * @param modelName - The model name to normalize. + * @returns The fully qualified model resource name. + */ + static normalizeModelName(modelName: string): string { + let model: string; + if (modelName.includes('/')) { + if (modelName.startsWith('models/')) { + // Add 'publishers/google' if the user is only passing in 'models/model-name'. + model = `publishers/google/${modelName}`; + } else { + // Any other custom format (e.g. tuned models) must be passed in correctly. + model = modelName; + } + } else { + // If path is not included, assume it's a non-tuned model. + model = `publishers/google/models/${modelName}`; + } + + return model; + } +} diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts new file mode 100644 index 00000000000..283dc80bfaf --- /dev/null +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -0,0 +1,81 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { logger } from '../logger'; + +/** + * Defines the image format for images generated by Imagen. + * + * Use this class to specify the desired format (JPEG or PNG) and compression quality + * for images generated by Imagen. This is typically included as part of + * {@link ImagenModelParams}. + * + * @example + * ```javascript + * const imagenModelParams = { + * // ... other ImagenModelParams + * imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. + * } + * ``` + * + * @beta + */ +export class ImagenImageFormat { + /** + * The MIME type. + */ + mimeType: string; + /** + * The level of compression (a number between 0 and 100). + */ + compressionQuality?: number; + + private constructor() { + this.mimeType = 'image/png'; + } + + /** + * Creates an {@link ImagenImageFormat} for a JPEG image. + * + * @param compressionQuality - The level of compression (a number between 0 and 100). + * @returns An {@link ImagenImageFormat} object for a JPEG image. + * + * @beta + */ + static jpeg(compressionQuality?: number): ImagenImageFormat { + if ( + compressionQuality && + (compressionQuality < 0 || compressionQuality > 100) + ) { + logger.warn( + `Invalid JPEG compression quality of ${compressionQuality} specified; the supported range is [0, 100].` + ); + } + return { mimeType: 'image/jpeg', compressionQuality }; + } + + /** + * Creates an {@link ImagenImageFormat} for a PNG image. + * + * @returns An {@link ImagenImageFormat} object for a PNG image. + * + * @beta + */ + static png(): ImagenImageFormat { + return { mimeType: 'image/png' }; + } +} diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 76b2f0ca1bf..d8337850925 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -17,8 +17,16 @@ import { expect, use } from 'chai'; import sinonChai from 'sinon-chai'; -import { Content } from '../types'; -import { formatGenerateContentInput } from './request-helpers'; +import { + Content, + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel +} from '../types'; +import { + createPredictRequestBody, + formatGenerateContentInput +} from './request-helpers'; use(sinonChai); @@ -199,4 +207,68 @@ describe('request formatting methods', () => { }); }); }); + describe('createPredictRequestBody', () => { + it('creates body with default request parameters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const body = createPredictRequestBody(prompt, {}); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.sampleCount).to.equal(1); + expect(body.parameters.includeRaiReason).to.be.true; + + // Parameters without default values should be undefined + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.outputOptions).to.be.undefined; + expect(body.parameters.negativePrompt).to.be.undefined; + expect(body.parameters.aspectRatio).to.be.undefined; + expect(body.parameters.addWatermark).to.be.undefined; + expect(body.parameters.safetyFilterLevel).to.be.undefined; + expect(body.parameters.personGeneration).to.be.undefined; + }); + }); + it('creates body with non-default request paramaters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const imageFormat = { mimeType: 'image/jpeg', compressionQuality: 75 }; + const safetySettings = { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_LOW_AND_ABOVE, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + }; + const addWatermark = true; + const numberOfImages = 4; + const negativePrompt = 'do not hallucinate'; + const aspectRatio = ImagenAspectRatio.LANDSCAPE_16x9; + const body = createPredictRequestBody(prompt, { + numberOfImages, + imageFormat, + addWatermark, + negativePrompt, + aspectRatio, + ...safetySettings + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters).deep.equal({ + sampleCount: numberOfImages, + outputOptions: { + mimeType: imageFormat.mimeType, + compressionQuality: imageFormat.compressionQuality + }, + addWatermark, + negativePrompt, + safetyFilterLevel: safetySettings.safetyFilterLevel, + personGeneration: safetySettings.personFilterLevel, + aspectRatio, + includeRaiReason: true, + storageUri: undefined + }); + }); + it('creates body with GCS URI', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const gcsURI = 'gcs-uri'; + const body = createPredictRequestBody(prompt, { + gcsURI + }); + + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.storageUri).to.equal(gcsURI); + }); }); diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 9e525b2a875..f69c88fca92 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -22,6 +22,7 @@ import { VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; +import { ImagenGenerationParams, PredictRequestBody } from '../types/internal'; export function formatSystemInstruction( input?: string | Part | Content @@ -124,3 +125,44 @@ export function formatGenerateContentInput( } return formattedRequest; } + +/** + * Convert the user-defined parameters in {@link ImagenGenerationParams} to the format + * that is expected from the REST API. + * + * @internal + */ +export function createPredictRequestBody( + prompt: string, + { + gcsURI, + imageFormat, + addWatermark, + numberOfImages = 1, + negativePrompt, + aspectRatio, + safetyFilterLevel, + personFilterLevel + }: ImagenGenerationParams +): PredictRequestBody { + // Properties that are undefined will be omitted from the JSON string that is sent in the request. + const body: PredictRequestBody = { + instances: [ + { + prompt + } + ], + parameters: { + storageUri: gcsURI, + negativePrompt, + sampleCount: numberOfImages, + aspectRatio, + outputOptions: imageFormat, + addWatermark, + safetyFilterLevel, + personGeneration: personFilterLevel, + includeRaiReason: true + } + }; + return body; +} diff --git a/packages/vertexai/src/requests/request.ts b/packages/vertexai/src/requests/request.ts index f81b40635e3..9b9465db776 100644 --- a/packages/vertexai/src/requests/request.ts +++ b/packages/vertexai/src/requests/request.ts @@ -30,7 +30,8 @@ import { logger } from '../logger'; export enum Task { GENERATE_CONTENT = 'generateContent', STREAM_GENERATE_CONTENT = 'streamGenerateContent', - COUNT_TOKENS = 'countTokens' + COUNT_TOKENS = 'countTokens', + PREDICT = 'predict' } export class RequestUrl { diff --git a/packages/vertexai/src/requests/response-helpers.test.ts b/packages/vertexai/src/requests/response-helpers.test.ts index 91a60d2cfce..4cab8cde047 100644 --- a/packages/vertexai/src/requests/response-helpers.test.ts +++ b/packages/vertexai/src/requests/response-helpers.test.ts @@ -15,7 +15,11 @@ * limitations under the License. */ -import { addHelpers, formatBlockErrorMessage } from './response-helpers'; +import { + addHelpers, + formatBlockErrorMessage, + handlePredictResponse +} from './response-helpers'; import { expect, use } from 'chai'; import { restore } from 'sinon'; import sinonChai from 'sinon-chai'; @@ -23,8 +27,11 @@ import { BlockReason, Content, FinishReason, - GenerateContentResponse + GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage } from '../types'; +import { getMockResponse } from '../../test-utils/mock-response'; use(sinonChai); @@ -246,4 +253,71 @@ describe('response-helpers methods', () => { ); }); }); + + describe('handlePredictResponse', () => { + it('returns base64 images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded.length).to.be.greaterThan(0); + }); + }); + }); + it('returns GCS images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-gcs.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach((image, i) => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.equal( + `gs://test-project-id-1234.firebasestorage.app/images/1234567890123/sample_${i}.jpg` + ); + }); + }); + it('has filtered reason and no images if all images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-all-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 39322892, 29310472" + ); + expect(res.images.length).to.equal(0); + }); + it('has filtered reason and no images if all base64 images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-base64-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded).to.have.length.greaterThan(0); + }); + }); + it('has filtered reason and no images if all GCS images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-gcs-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.have.length.greaterThan(0); + }); + }); }); diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 27347d10f0d..1ba0986f3f9 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -21,10 +21,13 @@ import { FunctionCall, GenerateContentCandidate, GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage, VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; import { logger } from '../logger'; +import { ImagenResponseInternal } from '../types/internal'; /** * Creates an EnhancedGenerateContentResponse object that has helper functions and @@ -196,3 +199,52 @@ export function formatBlockErrorMessage( } return message; } + +/** + * Convert a generic successful fetch response body to an Imagen response object + * that can be returned to the user. This converts the REST APIs response format to our + * APIs representation of a response. + * + * @internal + */ +export async function handlePredictResponse< + T extends ImagenInlineImage | ImagenGCSImage +>(response: Response): Promise<{ images: T[]; filteredReason?: string }> { + const responseJson: ImagenResponseInternal = await response.json(); + + const images: T[] = []; + let filteredReason: string | undefined = undefined; + + // The backend should always send a non-empty array of predictions if the response was successful. + if (!responseJson.predictions || responseJson.predictions?.length === 0) { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + 'No predictions or filtered reason received from Vertex AI. Please report this issue with the full error details at https://github.com/firebase/firebase-js-sdk/issues.' + ); + } + + for (const prediction of responseJson.predictions) { + if (prediction.raiFilteredReason) { + filteredReason = prediction.raiFilteredReason; + } else if (prediction.mimeType && prediction.bytesBase64Encoded) { + images.push({ + mimeType: prediction.mimeType, + bytesBase64Encoded: prediction.bytesBase64Encoded + } as T); + } else if (prediction.mimeType && prediction.gcsUri) { + images.push({ + mimeType: prediction.mimeType, + gcsURI: prediction.gcsUri + } as T); + } else { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + `Predictions array in response has missing properties. Response: ${JSON.stringify( + responseJson + )}` + ); + } + } + + return { images, filteredReason }; +} diff --git a/packages/vertexai/src/types/imagen/index.ts b/packages/vertexai/src/types/imagen/index.ts new file mode 100644 index 00000000000..546c64f13b1 --- /dev/null +++ b/packages/vertexai/src/types/imagen/index.ts @@ -0,0 +1,19 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './requests'; +export * from './responses'; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts new file mode 100644 index 00000000000..02a8a55e01c --- /dev/null +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -0,0 +1,129 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ImagenGenerationConfig, ImagenSafetySettings } from './requests'; + +/** + * A response from the REST API is expected to look like this in the success case: + * { + * "predictions": [ + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "iVBORw0KG..." + * }, + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "i4BOtw0KG..." + * } + * ] + * } + * + * And like this in the failure case: + * { + * "predictions": [ + * { + * "raiFilteredReason": "..." + * } + * ] + * } + * + * @internal + */ +export interface ImagenResponseInternal { + predictions?: Array<{ + /** + * The MIME type of the generated image. + */ + mimeType?: string; + /** + * The image data encoded as a base64 string. + */ + bytesBase64Encoded?: string; + /** + * The GCS URI where the image was stored. + */ + gcsUri?: string; + /** + * The reason why the image was filtered. + */ + raiFilteredReason?: string; + }>; +} + +/** + * The parameters to be sent in the request body of the HTTP call + * to the Vertex AI backend. + * + * We need a seperate internal-only interface for this because the REST + * API expects different parameter names than what we show to our users. + * + * Sample request body JSON: + * { + * "instances": [ + * { + * "prompt": "Portrait of a golden retriever on a beach." + * } + * ], + * "parameters": { + * "mimeType": "image/png", + * "safetyFilterLevel": "block_low_and_above", + * "personGeneration": "allow_all", + * "sampleCount": 2, + * "includeRaiReason": true, + * "aspectRatio": "9:16" + * } + * } + * + * See the Google Cloud docs: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#-drest + * + * @internal + */ +export interface PredictRequestBody { + instances: [ + { + prompt: string; + } + ]; + parameters: { + sampleCount: number; // Maps to numberOfImages + aspectRatio?: string; + outputOptions?: { + mimeType: string; + compressionQuality?: number; + }; + negativePrompt?: string; + storageUri?: string; // Maps to gcsURI + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; // Maps to personFilterLevel + includeRaiReason: boolean; + }; +} + +/** + * Contains all possible REST API paramaters that are provided by the caller. + * + * @internal + */ +export type ImagenGenerationParams = { + /** + * The Cloud Storage for Firebase bucket URI where the images should be stored + * (for GCS requests only). + */ + gcsURI?: string; +} & ImagenGenerationConfig & + ImagenSafetySettings; diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts new file mode 100644 index 00000000000..ac37488dfb5 --- /dev/null +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -0,0 +1,212 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ImagenImageFormat } from '../../requests/imagen-image-format'; + +/** + * Parameters for configuring an {@link ImagenModel}. + * + * @beta + */ +export interface ImagenModelParams { + /** + * The Imagen model to use for generating images. + * For example: `imagen-3.0-generate-002`. + * + * Only Imagen 3 models (named `imagen-3.0-*`) are supported. + * + * See {@link https://firebase.google.com/docs/vertex-ai/models | model versions} + * for a full list of supported Imagen 3 models. + */ + model: string; + /** + * Configuration options for generating images with Imagen. + */ + generationConfig?: ImagenGenerationConfig; + /** + * Safety settings for filtering potentially inappropriate content. + */ + safetySettings?: ImagenSafetySettings; +} + +/** + * Configuration options for generating images with Imagen. + * + * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images-imagen | documentation} for + * more details. + * + * @beta + */ +export interface ImagenGenerationConfig { + /** + * A description of what should be omitted from the generated images. + * + * Support for negative prompts depends on the Imagen model. + * + * See the {@link http://firebase.google.com/docs/vertex-ai/model-parameters#imagen | documentation} for more details. + */ + negativePrompt?: string; + /** + * The number of images to generate. The default value is 1. + * + * The number of sample images that may be generated in each request depends on the model + * (typically up to 4); see the sampleCount + * documentation for more details. + */ + numberOfImages?: number; + /** + * The aspect ratio of the generated images. The default value is square 1:1. + * Supported aspect ratios depend on the Imagen model, see {@link ImagenAspectRatio} + * for more details. + */ + aspectRatio?: ImagenAspectRatio; + /** + * The image format of the generated images. The default is PNG. + * + * See {@link ImagenImageFormat} for more details. + */ + imageFormat?: ImagenImageFormat; + /** + * Whether to add an invisible watermark to generated images. + * + * If set to `true`, an invisible SynthID watermark is embedded in generated images to indicate + * that they are AI generated. If set to `false`, watermarking will be disabled. + * + * For Imagen 3 models, the default value is `true`; see the addWatermark + * documentation for more details. + */ + addWatermark?: boolean; +} + +/** + * A filter level controlling how aggressively to filter sensitive content. + * + * Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI + * are assessed against a list of safety filters, which include 'harmful categories' (for example, + * `violence`, `sexual`, `derogatory`, and `toxic`). This filter level controls how aggressively to + * filter out potentially harmful content from responses. See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation } + * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines} + * for more details. + * + * @beta + */ +export enum ImagenSafetyFilterLevel { + /** + * The most aggressive filtering level; most strict blocking. + */ + BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + /** + * Blocks some sensitive prompts and responses. + */ + BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + /** + * Blocks few sensitive prompts and responses. + */ + BLOCK_ONLY_HIGH = 'block_only_high', + /** + * The least aggressive filtering level; blocks very few sensitive prompts and responses. + * + * Access to this feature is restricted and may require your case to be reviewed and approved by + * Cloud support. + */ + BLOCK_NONE = 'block_none' +} + +/** + * A filter level controlling whether generation of images containing people or faces is allowed. + * + * See the personGeneration + * documentation for more details. + * + * @beta + */ +export enum ImagenPersonFilterLevel { + /** + * Disallow generation of images containing people or faces; images of people are filtered out. + */ + BLOCK_ALL = 'dont_allow', + /** + * Allow generation of images containing adults only; images of children are filtered out. + * + * Generation of images containing people or faces may require your use case to be + * reviewed and approved by Cloud support; see the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen | Responsible AI and usage guidelines} + * for more details. + */ + ALLOW_ADULT = 'allow_adult', + /** + * Allow generation of images containing adults only; images of children are filtered out. + * + * Generation of images containing people or faces may require your use case to be + * reviewed and approved by Cloud support; see the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen | Responsible AI and usage guidelines} + * for more details. + */ + ALLOW_ALL = 'allow_all' +} + +/** + * Settings for controlling the aggressiveness of filtering out sensitive content. + * + * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation } + * for more details. + * + * @beta + */ +export interface ImagenSafetySettings { + /** + * A filter level controlling how aggressive to filter out sensitive content from generated + * images. + */ + safetyFilterLevel?: ImagenSafetyFilterLevel; + /** + * A filter level controlling whether generation of images containing people or faces is allowed. + */ + personFilterLevel?: ImagenPersonFilterLevel; +} + +/** + * Aspect ratios for Imagen images. + * + * To specify an aspect ratio for generated images, set the `aspectRatio` property in your + * {@link ImagenGenerationConfig}. + * + * See the the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation } + * for more details and examples of the supported aspect ratios. + * + * @beta + */ +export enum ImagenAspectRatio { + /** + * Square (1:1) aspect ratio. + */ + SQUARE = '1:1', + /** + * Landscape (3:4) aspect ratio. + */ + LANDSCAPE_3x4 = '3:4', + /** + * Portrait (4:3) aspect ratio. + */ + PORTRAIT_4x3 = '4:3', + /** + * Landscape (16:9) aspect ratio. + */ + LANDSCAPE_16x9 = '16:9', + /** + * Portrait (9:16) aspect ratio. + */ + PORTRAIT_9x16 = '9:16' +} diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts new file mode 100644 index 00000000000..c5cf5dd9057 --- /dev/null +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An image generated by Imagen, represented as inline data. + * + * @beta + */ +export interface ImagenInlineImage { + /** + * The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + * + * To request a different format, set the `imageFormat` property in your {@link ImagenGenerationConfig}. + */ + mimeType: string; + /** + * The base64-encoded image data. + */ + bytesBase64Encoded: string; +} + +/** + * An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. + * + * This feature is not available yet. + */ +export interface ImagenGCSImage { + /** + * The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + * + * To request a different format, set the `imageFormat` property in your {@link ImagenGenerationConfig}. + */ + mimeType: string; + /** + * The URI of the file stored in a Cloud Storage for Firebase bucket. + * + * @example `"gs://bucket-name/path/sample_0.jpg"`. + */ + gcsURI: string; +} + +/** + * The response from a request to generate images with Imagen. + * + * @beta + */ +export interface ImagenGenerationResponse< + T extends ImagenInlineImage | ImagenGCSImage +> { + /** + * The images generated by Imagen. + * + * The number of images generated may be fewer than the number requested if one or more were + * filtered out; see `filteredReason`. + */ + images: T[]; + /** + * The reason that images were filtered out. This property will only be defined if one + * or more images were filtered. + * + * Images may be filtered out due to the {@link ImagenSafetyFilterLevel}, + * {@link ImagenPersonFilterLevel}, or filtering included in the model. + * The filter levels may be adjusted in your {@link ImagenSafetySettings}. + * + * See the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen | Responsible AI and usage guidelines for Imagen} + * for more details. + */ + filteredReason?: string; +} diff --git a/packages/vertexai/src/types/index.ts b/packages/vertexai/src/types/index.ts index 85133aa07c5..f575c5ba8e9 100644 --- a/packages/vertexai/src/types/index.ts +++ b/packages/vertexai/src/types/index.ts @@ -21,3 +21,4 @@ export * from './requests'; export * from './responses'; export * from './error'; export * from './schema'; +export * from './imagen'; diff --git a/packages/vertexai/src/types/internal.ts b/packages/vertexai/src/types/internal.ts index 8271175feff..87c28a02ab2 100644 --- a/packages/vertexai/src/types/internal.ts +++ b/packages/vertexai/src/types/internal.ts @@ -18,6 +18,8 @@ import { AppCheckTokenResult } from '@firebase/app-check-interop-types'; import { FirebaseAuthTokenData } from '@firebase/auth-interop-types'; +export * from './imagen/internal'; + export interface ApiSettings { apiKey: string; project: string;