From ec35231ff406752d6b73cea4316ef38666646c9a Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 9 Dec 2024 11:57:18 -0500 Subject: [PATCH 01/31] WIP --- common/api-review/vertexai.api.md | 184 +++++++++++++ docs-devsite/_toc.yaml | 24 ++ docs-devsite/vertexai.imagengcsimage.md | 36 +++ .../vertexai.imagengcsimageresponse.md | 40 +++ .../vertexai.imagengenerationconfig.md | 49 ++++ docs-devsite/vertexai.imagenimageformat.md | 40 +++ docs-devsite/vertexai.imagenimagereponse.md | 40 +++ docs-devsite/vertexai.imageninlineimage.md | 34 +++ .../vertexai.imageninlineimageresponse.md | 46 ++++ docs-devsite/vertexai.imagenmodel.md | 124 +++++++++ docs-devsite/vertexai.imagenmodelconfig.md | 49 ++++ docs-devsite/vertexai.imagenmodelparams.md | 40 +++ docs-devsite/vertexai.imagensafetysettings.md | 45 ++++ docs-devsite/vertexai.md | 135 ++++++++++ docs-devsite/vertexai.safetyattributes.md | 40 +++ .../test/unit/transportoptions.test.ts | 3 +- packages/vertexai/api-extractor.json | 18 +- packages/vertexai/src/api.test.ts | 54 +++- packages/vertexai/src/api.ts | 31 ++- .../vertexai/src/models/imagen-model.test.ts | 246 ++++++++++++++++++ packages/vertexai/src/models/imagen-model.ts | 199 ++++++++++++++ .../src/requests/request-helpers.test.ts | 79 +++++- .../vertexai/src/requests/request-helpers.ts | 53 +++- packages/vertexai/src/requests/request.ts | 3 +- .../src/requests/response-helpers.test.ts | 78 +++++- .../vertexai/src/requests/response-helpers.ts | 52 ++++ packages/vertexai/src/types/imagen/index.ts | 19 ++ .../vertexai/src/types/imagen/internal.ts | 54 ++++ .../vertexai/src/types/imagen/requests.ts | 142 ++++++++++ .../vertexai/src/types/imagen/responses.ts | 73 ++++++ packages/vertexai/src/types/index.ts | 1 + packages/vertexai/src/types/internal.ts | 2 + packages/vertexai/src/types/responses.ts | 5 + 33 files changed, 2014 insertions(+), 24 deletions(-) create mode 100644 docs-devsite/vertexai.imagengcsimage.md create mode 100644 docs-devsite/vertexai.imagengcsimageresponse.md create mode 100644 docs-devsite/vertexai.imagengenerationconfig.md create mode 100644 docs-devsite/vertexai.imagenimageformat.md create mode 100644 docs-devsite/vertexai.imagenimagereponse.md create mode 100644 docs-devsite/vertexai.imageninlineimage.md create mode 100644 docs-devsite/vertexai.imageninlineimageresponse.md create mode 100644 docs-devsite/vertexai.imagenmodel.md create mode 100644 docs-devsite/vertexai.imagenmodelconfig.md create mode 100644 docs-devsite/vertexai.imagenmodelparams.md create mode 100644 docs-devsite/vertexai.imagensafetysettings.md create mode 100644 docs-devsite/vertexai.safetyattributes.md create mode 100644 packages/vertexai/src/models/imagen-model.test.ts create mode 100644 packages/vertexai/src/models/imagen-model.ts create mode 100644 packages/vertexai/src/types/imagen/index.ts create mode 100644 packages/vertexai/src/types/imagen/internal.ts create mode 100644 packages/vertexai/src/types/imagen/requests.ts create mode 100644 packages/vertexai/src/types/imagen/responses.ts diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 041bc62451f..aa35a3f024f 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -348,6 +348,9 @@ export class GenerativeModel { // @public export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +// @public +export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; + // @public export function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI; @@ -429,6 +432,148 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } +// @public (undocumented) +export enum ImagenAspectRatio { + // (undocumented) + CLASSIC_LANDSCAPE = "4:3", + // (undocumented) + CLASSIC_PORTRAIT = "3:4", + // (undocumented) + PORTRAIT = "9:16", + // (undocumented) + SQUARE = "1:1", + // (undocumented) + WIDESCREEN = "16:9" +} + +// Warning: (ae-incompatible-release-tags) The symbol "ImagenGCSImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal +// +// @public +export interface ImagenGCSImage extends ImagenImage { + gcsURI: string; +} + +// @public (undocumented) +export interface ImagenGCSImageResponse { + // (undocumented) + filteredReason?: string; + // (undocumented) + images: ImagenGCSImage[]; +} + +// @public (undocumented) +export interface ImagenGenerationConfig { + // (undocumented) + aspectRatio?: ImagenAspectRatio; + // (undocumented) + negativePrompt?: string; + // (undocumented) + numberOfImages?: number; +} + +// Warning: (ae-internal-missing-underscore) The name "ImagenImage" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface ImagenImage { + // (undocumented) + mimeType: string; +} + +// @public (undocumented) +export interface ImagenImageFormat { + // (undocumented) + compressionQuality?: number; + // (undocumented) + mimeType: string; +} + +// @public (undocumented) +export interface ImagenImageReponse { + // (undocumented) + filteredReason?: string; + // Warning: (ae-incompatible-release-tags) The symbol "images" is marked as @public, but its signature references "ImagenImage" which is marked as @internal + // + // (undocumented) + images: ImagenImage[]; +} + +// Warning: (ae-incompatible-release-tags) The symbol "ImagenInlineImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal +// +// @public +export interface ImagenInlineImage extends ImagenImage { + // (undocumented) + bytesBase64Encoded: string; +} + +// @public +export interface ImagenInlineImageResponse { + filteredReason?: string; + images: ImagenInlineImage[]; +} + +// @public +export class ImagenModel { + constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); + generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + // (undocumented) + model: string; + } + +// @public (undocumented) +export interface ImagenModelConfig { + // (undocumented) + addWatermark?: boolean; + // (undocumented) + imageFormat?: ImagenImageFormat; + // (undocumented) + safetySettings?: ImagenSafetySettings; +} + +// @public +export interface ImagenModelParams extends ImagenModelConfig { + // (undocumented) + model: string; +} + +// @public (undocumented) +export enum ImagenPersonFilterLevel { + // (undocumented) + ALLOW_ADULT = "allow_adult", + // (undocumented) + ALLOW_ALL = "allow_all", + // (undocumented) + BLOCK_ALL = "dont_allow" +} + +// Warning: (ae-internal-missing-underscore) The name "ImagenRequestConfig" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface ImagenRequestConfig extends ImagenModelConfig, ImagenGenerationConfig { + // (undocumented) + gcsURI?: string; + // (undocumented) + prompt: string; +} + +// @public (undocumented) +export enum ImagenSafetyFilterLevel { + // (undocumented) + BLOCK_LOW_AND_ABOVE = "block_low_and_above", + // (undocumented) + BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", + // (undocumented) + BLOCK_NONE = "block_none", + // (undocumented) + BLOCK_ONLY_HIGH = "block_only_high" +} + +// @public (undocumented) +export interface ImagenSafetySettings { + personFilterLevel?: ImagenPersonFilterLevel; + safetyFilterLevel?: ImagenSafetyFilterLevel; +} + // @public export interface InlineDataPart { // (undocumented) @@ -447,6 +592,9 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export function jpeg(compressionQuality: number): ImagenImageFormat; + // @public export interface ModelParams extends BaseParams { // (undocumented) @@ -490,9 +638,37 @@ export interface ObjectSchemaInterface extends SchemaInterface { // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +// @public +export function png(): ImagenImageFormat; + // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; +// Warning: (ae-internal-missing-underscore) The name "PredictRequestBody" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface PredictRequestBody { + // (undocumented) + instances: [ + { + prompt: string; + } + ]; + // (undocumented) + parameters: { + sampleCount: number; + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; + includeRaiReason: boolean; + }; +} + // @public export interface PromptFeedback { // (undocumented) @@ -520,6 +696,14 @@ export interface RetrievedContextAttribution { // @public export type Role = (typeof POSSIBLE_ROLES)[number]; +// @public (undocumented) +export interface SafetyAttributes { + // (undocumented) + categories: string[]; + // (undocumented) + scores: number[]; +} + // @public export interface SafetyRating { // (undocumented) diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 9d60c12906c..6156c51c3f9 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -528,6 +528,28 @@ toc: path: /docs/reference/js/vertexai.groundingattribution.md - title: GroundingMetadata path: /docs/reference/js/vertexai.groundingmetadata.md + - title: ImagenGCSImage + path: /docs/reference/js/vertexai.imagengcsimage.md + - title: ImagenGCSImageResponse + path: /docs/reference/js/vertexai.imagengcsimageresponse.md + - title: ImagenGenerationConfig + path: /docs/reference/js/vertexai.imagengenerationconfig.md + - title: ImagenImageFormat + path: /docs/reference/js/vertexai.imagenimageformat.md + - title: ImagenImageReponse + path: /docs/reference/js/vertexai.imagenimagereponse.md + - title: ImagenInlineImage + path: /docs/reference/js/vertexai.imageninlineimage.md + - title: ImagenInlineImageResponse + path: /docs/reference/js/vertexai.imageninlineimageresponse.md + - title: ImagenModel + path: /docs/reference/js/vertexai.imagenmodel.md + - title: ImagenModelConfig + path: /docs/reference/js/vertexai.imagenmodelconfig.md + - title: ImagenModelParams + path: /docs/reference/js/vertexai.imagenmodelparams.md + - title: ImagenSafetySettings + path: /docs/reference/js/vertexai.imagensafetysettings.md - title: InlineDataPart path: /docs/reference/js/vertexai.inlinedatapart.md - title: IntegerSchema @@ -546,6 +568,8 @@ toc: path: /docs/reference/js/vertexai.requestoptions.md - title: RetrievedContextAttribution path: /docs/reference/js/vertexai.retrievedcontextattribution.md + - title: SafetyAttributes + path: /docs/reference/js/vertexai.safetyattributes.md - title: SafetyRating path: /docs/reference/js/vertexai.safetyrating.md - title: SafetySetting diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md new file mode 100644 index 00000000000..eb2e9b76556 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -0,0 +1,36 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImage interface +Image generated by Imagen, stored in Google Cloud Storage (GCS). + +Signature: + +```typescript +export interface ImagenGCSImage extends ImagenImage +``` +Extends: ImagenImage + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI at which the generated image is stored. | + +## ImagenGCSImage.gcsURI + +The Google Cloud Storage (GCS) URI at which the generated image is stored. + +Signature: + +```typescript +gcsURI: string; +``` diff --git a/docs-devsite/vertexai.imagengcsimageresponse.md b/docs-devsite/vertexai.imagengcsimageresponse.md new file mode 100644 index 00000000000..0c20cb1e752 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimageresponse.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImageResponse interface +Signature: + +```typescript +export interface ImagenGCSImageResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | | +| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | | + +## ImagenGCSImageResponse.filteredReason + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenGCSImageResponse.images + +Signature: + +```typescript +images: ImagenGCSImage[]; +``` diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md new file mode 100644 index 00000000000..bf28bc291ae --- /dev/null +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGenerationConfig interface +Signature: + +```typescript +export interface ImagenGenerationConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | + +## ImagenGenerationConfig.aspectRatio + +Signature: + +```typescript +aspectRatio?: ImagenAspectRatio; +``` + +## ImagenGenerationConfig.negativePrompt + +Signature: + +```typescript +negativePrompt?: string; +``` + +## ImagenGenerationConfig.numberOfImages + +Signature: + +```typescript +numberOfImages?: number; +``` diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md new file mode 100644 index 00000000000..7ab7b9eb277 --- /dev/null +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageFormat interface +Signature: + +```typescript +export interface ImagenImageFormat +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | number | | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | string | | + +## ImagenImageFormat.compressionQuality + +Signature: + +```typescript +compressionQuality?: number; +``` + +## ImagenImageFormat.mimeType + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagenimagereponse.md b/docs-devsite/vertexai.imagenimagereponse.md new file mode 100644 index 00000000000..14e34651a41 --- /dev/null +++ b/docs-devsite/vertexai.imagenimagereponse.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageReponse interface +Signature: + +```typescript +export interface ImagenImageReponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagenimagereponse.md#imagenimagereponsefilteredreason) | string | | +| [images](./vertexai.imagenimagereponse.md#imagenimagereponseimages) | ImagenImage\[\] | | + +## ImagenImageReponse.filteredReason + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenImageReponse.images + +Signature: + +```typescript +images: ImagenImage[]; +``` diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md new file mode 100644 index 00000000000..fdfac65c15a --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -0,0 +1,34 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImage interface +Image generated by Imagen to inline bytes. + +Signature: + +```typescript +export interface ImagenInlineImage extends ImagenImage +``` +Extends: ImagenImage + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | + +## ImagenInlineImage.bytesBase64Encoded + +Signature: + +```typescript +bytesBase64Encoded: string; +``` diff --git a/docs-devsite/vertexai.imageninlineimageresponse.md b/docs-devsite/vertexai.imageninlineimageresponse.md new file mode 100644 index 00000000000..06cf3a2857f --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimageresponse.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImageResponse interface +Imagen image response. + +Signature: + +```typescript +export interface ImagenInlineImageResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered, this will be empty. | + +## ImagenInlineImageResponse.filteredReason + +The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenInlineImageResponse.images + +The images generated by Imagen. If all images were filtered, this will be empty. + +Signature: + +```typescript +images: ImagenInlineImage[]; +``` diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md new file mode 100644 index 00000000000..f1bff9689fe --- /dev/null +++ b/docs-devsite/vertexai.imagenmodel.md @@ -0,0 +1,124 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModel class +Class for Imagen model APIs. + +Signature: + +```typescript +export declare class ImagenModel +``` + +## Constructors + +| Constructor | Modifiers | Description | +| --- | --- | --- | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [model](./vertexai.imagenmodel.md#imagenmodelmodel) | | string | | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [generateImages(prompt, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | +| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images using the Imagen model and returns them as base64-encoded strings. | + +## ImagenModel.(constructor) + +Constructs a new instance of the `ImagenModel` class + +Signature: + +```typescript +constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | + +## ImagenModel.model + +Signature: + +```typescript +model: string; +``` + +## ImagenModel.generateImages() + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. + +Signature: + +```typescript +generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | The text prompt used to generate the images. | +| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +Returns: + +Promise<[ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface)> + +A promise that resolves to an [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) object containing the generated images. + +#### Exceptions + +If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). + +## ImagenModel.generateImagesGCS() + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. + +Signature: + +```typescript +generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | The text prompt used to generate the images. | +| gcsURI | string | The GCS URI where the images should be stored. | +| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +Returns: + +Promise<[ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface)> + +A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the generated images. + +#### Exceptions + +If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). + diff --git a/docs-devsite/vertexai.imagenmodelconfig.md b/docs-devsite/vertexai.imagenmodelconfig.md new file mode 100644 index 00000000000..9923d0de504 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelconfig.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelConfig interface +Signature: + +```typescript +export interface ImagenModelConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | | +| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | +| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | + +## ImagenModelConfig.addWatermark + +Signature: + +```typescript +addWatermark?: boolean; +``` + +## ImagenModelConfig.imageFormat + +Signature: + +```typescript +imageFormat?: ImagenImageFormat; +``` + +## ImagenModelConfig.safetySettings + +Signature: + +```typescript +safetySettings?: ImagenSafetySettings; +``` diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md new file mode 100644 index 00000000000..2d1070a3585 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelParams interface + Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +Signature: + +```typescript +export interface ImagenModelParams extends ImagenModelConfig +``` +Extends: [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | + +## ImagenModelParams.model + +Signature: + +```typescript +model: string; +``` diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md new file mode 100644 index 00000000000..0d4a45a6d52 --- /dev/null +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -0,0 +1,45 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenSafetySettings interface + +Signature: + +```typescript +export interface ImagenSafetySettings +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Generate people. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter level | + +## ImagenSafetySettings.personFilterLevel + +Generate people. + +Signature: + +```typescript +personFilterLevel?: ImagenPersonFilterLevel; +``` + +## ImagenSafetySettings.safetyFilterLevel + +Safety filter level + +Signature: + +```typescript +safetyFilterLevel?: ImagenSafetyFilterLevel; +``` diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index d9e26eabc5d..827babbcafe 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -20,6 +20,11 @@ The Vertex AI in Firebase Web SDK. | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. | +| function() | +| [png()](./vertexai.md#png) | Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| function(compressionQuality, ...) | +| [jpeg(compressionQuality)](./vertexai.md#jpeg_8e6711f) | Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | ## Classes @@ -29,6 +34,7 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs. | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | @@ -48,6 +54,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -83,12 +92,23 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | Image generated by Imagen, stored in Google Cloud Storage (GCS). | +| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | +| [ImagenImageReponse](./vertexai.imagenimagereponse.md#imagenimagereponse_interface) | | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Image generated by Imagen to inline bytes. | +| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | Imagen image response. | +| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Copyright 2024 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | +| [SafetyAttributes](./vertexai.safetyattributes.md#safetyattributes_interface) | | | [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | | [SchemaInterface](./vertexai.schemainterface.md#schemainterface_interface) | Interface for [Schema](./vertexai.schema.md#schema_class) class. | @@ -167,6 +187,68 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) +### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} + +Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. + +Signature: + +```typescript +export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | + +Returns: + +[ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) + +## function() + +### png() {:#png} + +Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare function png(): ImagenImageFormat; +``` +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + + +## function(compressionQuality, ...) + +### jpeg(compressionQuality) {:#jpeg_8e6711f} + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare function jpeg(compressionQuality: number): ImagenImageFormat; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| compressionQuality | number | The level of compression. | + +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + ## POSSIBLE\_ROLES Possible roles. @@ -363,6 +445,59 @@ export declare enum HarmSeverity | HARM\_SEVERITY\_MEDIUM | "HARM_SEVERITY_MEDIUM" | | | HARM\_SEVERITY\_NEGLIGIBLE | "HARM_SEVERITY_NEGLIGIBLE" | | +## ImagenAspectRatio + +Signature: + +```typescript +export declare enum ImagenAspectRatio +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| CLASSIC\_LANDSCAPE | "4:3" | | +| CLASSIC\_PORTRAIT | "3:4" | | +| PORTRAIT | "9:16" | | +| SQUARE | "1:1" | | +| WIDESCREEN | "16:9" | | + +## ImagenPersonFilterLevel + + +Signature: + +```typescript +export declare enum ImagenPersonFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ALLOW\_ADULT | "allow_adult" | | +| ALLOW\_ALL | "allow_all" | | +| BLOCK\_ALL | "dont_allow" | | + +## ImagenSafetyFilterLevel + + +Signature: + +```typescript +export declare enum ImagenSafetyFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | +| BLOCK\_NONE | "block_none" | | +| BLOCK\_ONLY\_HIGH | "block_only_high" | | + ## SchemaType Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) diff --git a/docs-devsite/vertexai.safetyattributes.md b/docs-devsite/vertexai.safetyattributes.md new file mode 100644 index 00000000000..837850ab80f --- /dev/null +++ b/docs-devsite/vertexai.safetyattributes.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# SafetyAttributes interface +Signature: + +```typescript +export interface SafetyAttributes +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [categories](./vertexai.safetyattributes.md#safetyattributescategories) | string\[\] | | +| [scores](./vertexai.safetyattributes.md#safetyattributesscores) | number\[\] | | + +## SafetyAttributes.categories + +Signature: + +```typescript +categories: string[]; +``` + +## SafetyAttributes.scores + +Signature: + +```typescript +scores: number[]; +``` diff --git a/packages/data-connect/test/unit/transportoptions.test.ts b/packages/data-connect/test/unit/transportoptions.test.ts index 91e090e6a54..a7136b5c408 100644 --- a/packages/data-connect/test/unit/transportoptions.test.ts +++ b/packages/data-connect/test/unit/transportoptions.test.ts @@ -16,6 +16,8 @@ */ import { expect } from 'chai'; + +import { queryRef } from '../../src'; import { TransportOptions, areTransportOptionsEqual, @@ -23,7 +25,6 @@ import { getDataConnect } from '../../src/api/DataConnect'; import { app } from '../util'; -import { queryRef } from '../../src'; describe('Transport Options', () => { it('should return false if transport options are not equal', () => { const transportOptions1: TransportOptions = { diff --git a/packages/vertexai/api-extractor.json b/packages/vertexai/api-extractor.json index 8a3c6cb251e..799c1fcf397 100644 --- a/packages/vertexai/api-extractor.json +++ b/packages/vertexai/api-extractor.json @@ -1,10 +1,10 @@ { - "extends": "../../config/api-extractor.json", - // Point it to your entry point d.ts file. - "mainEntryPointFilePath": "/dist/src/index.d.ts", - "dtsRollup": { - "enabled": true, - "untrimmedFilePath": "/dist/.d.ts", - "publicTrimmedFilePath": "/dist/-public.d.ts" - } -} \ No newline at end of file + "extends": "../../config/api-extractor.json", + // Point it to your entry point d.ts file. + "mainEntryPointFilePath": "/dist/src/index.d.ts", + "dtsRollup": { + "enabled": true, + "untrimmedFilePath": "/dist/.d.ts", + "publicTrimmedFilePath": "/dist/-public.d.ts" + } +} diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index b6c96923856..c9432d2a7ea 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ModelParams, VertexAIErrorCode } from './types'; +import { ImagenModelParams, ModelParams, VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; -import { getGenerativeModel } from './api'; +import { ImagenModel, getGenerativeModel, getImagenModel } from './api'; import { expect } from 'chai'; import { VertexAI } from './public-types'; import { GenerativeModel } from './models/generative-model'; @@ -84,4 +84,54 @@ describe('Top level API', () => { expect(genModel).to.be.an.instanceOf(GenerativeModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); }); + it('getImagenModel throws if no model is provided', () => { + try { + getImagenModel(fakeVertexAI, {} as ImagenModelParams); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_MODEL); + expect((e as VertexAIError).message).includes( + `VertexAI: Must provide a model name. Example: ` + + `getImagenModel({ model: 'my-model-name' }) (vertexAI/${VertexAIErrorCode.NO_MODEL})` + ); + } + }); + it('getImagenModel throws if no apiKey is provided', () => { + const fakeVertexNoApiKey = { + ...fakeVertexAI, + app: { options: { projectId: 'my-project' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoApiKey, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_API_KEY); + expect((e as VertexAIError).message).equals( + `VertexAI: The "apiKey" field is empty in the local ` + + `Firebase config. Firebase VertexAI requires this field to` + + ` contain a valid API key. (vertexAI/${VertexAIErrorCode.NO_API_KEY})` + ); + } + }); + it('getImagenModel throws if no projectId is provided', () => { + const fakeVertexNoProject = { + ...fakeVertexAI, + app: { options: { apiKey: 'my-key' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoProject, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes( + VertexAIErrorCode.NO_PROJECT_ID + ); + expect((e as VertexAIError).message).equals( + `VertexAI: The "projectId" field is empty in the local` + + ` Firebase config. Firebase VertexAI requires this field ` + + `to contain a valid project ID. (vertexAI/${VertexAIErrorCode.NO_PROJECT_ID})` + ); + } + }); + it('getGenerativeModel gets an ImagenModel', () => { + const genModel = getImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(genModel).to.be.an.instanceOf(ImagenModel); + expect(genModel.model).to.equal('publishers/google/models/my-model'); + }); }); diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 42f33ed2a85..07154356435 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -21,14 +21,22 @@ import { getModularInstance } from '@firebase/util'; import { DEFAULT_LOCATION, VERTEX_TYPE } from './constants'; import { VertexAIService } from './service'; import { VertexAI, VertexAIOptions } from './public-types'; -import { ModelParams, RequestOptions, VertexAIErrorCode } from './types'; +import { + ImagenModelParams, + ModelParams, + RequestOptions, + VertexAIErrorCode +} from './types'; import { VertexAIError } from './errors'; import { GenerativeModel } from './models/generative-model'; +import { ImagenModel, jpeg, png } from './models/imagen-model'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; -export { GenerativeModel }; +export { jpeg, png }; + +export { GenerativeModel, ImagenModel }; export { VertexAIError }; @@ -77,3 +85,22 @@ export function getGenerativeModel( } return new GenerativeModel(vertexAI, modelParams, requestOptions); } + +/** + * Returns a {@link ImagenModel} class with methods for using Imagen. + * + * @public + */ +export function getImagenModel( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + requestOptions?: RequestOptions +): ImagenModel { + if (!modelParams.model) { + throw new VertexAIError( + VertexAIErrorCode.NO_MODEL, + `Must provide a model name. Example: getImagenModel({ model: 'my-model-name' })` + ); + } + return new ImagenModel(vertexAI, modelParams, requestOptions); +} diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts new file mode 100644 index 00000000000..909bff3dea2 --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -0,0 +1,246 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { ImagenModel } from './imagen-model'; +import { + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel, + VertexAI, + VertexAIErrorCode +} from '../public-types'; +import * as request from '../requests/request'; +import sinonChai from 'sinon-chai'; +import { VertexAIError } from '../errors'; +import { getMockResponse } from '../../test-utils/mock-response'; +import { match, restore, stub } from 'sinon'; + +use(sinonChai); + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('ImagenModel', () => { + it('handles plain model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles models/ prefixed model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'models/my-model' + }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles full model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'publishers/google/models/my-model' + }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles prefixed tuned model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'tunedModels/my-model' + }); + expect(imagenModel.model).to.equal('tunedModels/my-model'); + }); + it('throws if not passed an api key', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + projectId: 'my-project' + } + }, + location: 'us-central1' + }; + try { + new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); + } + }); + it('throws if not passed a project ID', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key' + } + }, + location: 'us-central1' + }; + try { + new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + } catch (e) { + expect((e as VertexAIError).code).to.equal( + VertexAIErrorCode.NO_PROJECT_ID + ); + } + }); + it('generateImages makes a request to predict with default parameters', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('sampleCount') && + value.includes('includeRaiReason') && + value.includes('aspectRatio') && + value.includes('mimeType') + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with model-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model', + addWatermark: true, + imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + safetySettings: { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + } + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('safetyFilterLevel') && + value.includes('block_only_high') && + value.includes('personFilterLevel') && + value.includes('allow_adult') && + value.includes('mimeType') && + value.includes('image/jpeg') && + value.includes('addWatermark') && + value.includes('true') + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with request-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { + numberOfImages: 4, + aspectRatio: ImagenAspectRatio.WIDESCREEN, + negativePrompt: 'do not hallucinate' + }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('sampleCount') && + value.includes('4') && + value.includes('aspectRatio') && + value.includes('16:9') && + value.includes('negativePrompt') && + value.includes('do not hallucinate') + ); + }), + undefined + ); + + restore(); + }); + it('throws if prompt blocked', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-prompt-blocked.json' + ); + + stub(globalThis, 'fetch').resolves({ + ok: false, + status: 400, // TODO (dlarocque): Get this from the mock response + statusText: 'Bad Request', + json: mockResponse.json + } as Response); + + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + try { + await imagenModel.generateImages( + 'A photorealistic image of a toy boat at sea.', + { numberOfImages: 1 } + ); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); + expect((e as VertexAIError).message).to.include('400'); + expect((e as VertexAIError).message).to.include( + "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 42876398" + ); + } finally { + restore(); + } + }); +}); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts new file mode 100644 index 00000000000..17d1e1cfb2c --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.ts @@ -0,0 +1,199 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAIError } from '../errors'; +import { VertexAI } from '../public-types'; +import { Task, makeRequest } from '../requests/request'; +import { createPredictRequestBody } from '../requests/request-helpers'; +import { handlePredictResponse } from '../requests/response-helpers'; +import { VertexAIService } from '../service'; +import { + ImagenGCSImage, + ImagenGCSImageResponse, + ImagenImageFormat, + ImagenGenerationConfig, + ImagenInlineImage, + RequestOptions, + VertexAIErrorCode, + ImagenModelParams, + ImagenInlineImageResponse, + ImagenModelConfig +} from '../types'; +import { ApiSettings } from '../types/internal'; + +/** + * Class for Imagen model APIs. + * @public + */ +export class ImagenModel { + model: string; + private _apiSettings: ApiSettings; + private modelConfig: ImagenModelConfig; + + /** + * + * @param vertexAI + * @param modelParams + * @param requestOptions + */ + constructor( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + private requestOptions?: RequestOptions + ) { + const { model, ...modelConfig } = modelParams; + this.modelConfig = modelConfig; + if (model.includes('/')) { + if (model.startsWith('models/')) { + // Add "publishers/google" if the user is only passing in 'models/model-name'. + this.model = `publishers/google/${model}`; + } else { + // Any other custom format (e.g. tuned models) must be passed in correctly. + this.model = model; + } + } else { + // If path is not included, assume it's a non-tuned model. + this.model = `publishers/google/models/${model}`; + } + + if (!vertexAI.app?.options?.apiKey) { + throw new VertexAIError( + VertexAIErrorCode.NO_API_KEY, + `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` + ); + } else if (!vertexAI.app?.options?.projectId) { + throw new VertexAIError( + VertexAIErrorCode.NO_PROJECT_ID, + `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` + ); + } else { + this._apiSettings = { + apiKey: vertexAI.app.options.apiKey, + project: vertexAI.app.options.projectId, + location: vertexAI.location + }; + if ((vertexAI as VertexAIService).appCheck) { + this._apiSettings.getAppCheckToken = () => + (vertexAI as VertexAIService).appCheck!.getToken(); + } + + if ((vertexAI as VertexAIService).auth) { + this._apiSettings.getAuthToken = () => + (vertexAI as VertexAIService).auth!.getToken(); + } + } + } + + /** + * Generates images using the Imagen model and returns them as base64-encoded strings. + * + * @param prompt The text prompt used to generate the images. + * @param imagenRequestOptions Configuration options for the Imagen generation request. + * See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenInlineImageResponse} object containing the generated images. + * + * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * + * @remarks + * If one or more images are filtered, the returned object will have a defined `filteredReason` property. + * If all images are filtered, the `images` array will be empty, and no error will be thrown. + */ + async generateImages( + prompt: string, + imagenRequestOptions?: ImagenGenerationConfig + ): Promise { + const body = createPredictRequestBody({ + prompt, + ...imagenRequestOptions, + ...this.modelConfig + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } + + /** + * Generates images using the Imagen model and returns them as base64-encoded strings. + * + * @param prompt The text prompt used to generate the images. + * @param gcsURI The GCS URI where the images should be stored. + * @param imagenRequestOptions Configuration options for the Imagen generation request. + * See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenGCSImageResponse} object containing the generated images. + * + * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * + * @remarks + * If one or more images are filtered, the returned object will have a defined `filteredReason` property. + * If all images are filtered, the `images` array will be empty, and no error will be thrown. + */ + async generateImagesGCS( + prompt: string, + gcsURI: string, + imagenRequestOptions?: ImagenGenerationConfig + ): Promise { + const body = createPredictRequestBody({ + prompt, + gcsURI, + ...imagenRequestOptions, + ...this.modelConfig + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } +} + +/** + * Creates an {@link ImagenImageFormat} for a JPEG image, to be included in an {@link ImagenModelParams}. + * + * @param compressionQuality The level of compression. + * @returns {@link ImagenImageFormat} + * + * @public + */ +export function jpeg(compressionQuality: number): ImagenImageFormat { + return { + mimeType: 'image/jpeg', + compressionQuality + }; +} + +/** + * Creates an {@link ImageImageFormat} for a PNG image, to be included in a {@link ImagenModelParams}. + * + * @returns {@link ImageImageFormat} + * + * @public + */ +export function png(): ImagenImageFormat { + return { + mimeType: 'image/png' + }; +} diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 76b2f0ca1bf..9582f384a2c 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -17,8 +17,16 @@ import { expect, use } from 'chai'; import sinonChai from 'sinon-chai'; -import { Content } from '../types'; -import { formatGenerateContentInput } from './request-helpers'; +import { + Content, + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel +} from '../types'; +import { + createPredictRequestBody, + formatGenerateContentInput +} from './request-helpers'; use(sinonChai); @@ -199,4 +207,71 @@ describe('request formatting methods', () => { }); }); }); + describe('createPredictRequestBody', () => { + it('creates body with default request parameters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const body = createPredictRequestBody({ + prompt + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.sampleCount).to.equal(1); + expect(body.parameters.mimeType).to.equal('image/png'); + expect(body.parameters.includeRaiReason).to.be.true; + expect(body.parameters.aspectRatio).to.equal('1:1'); + + // Parameters without default values should be undefined + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.compressionQuality).to.be.undefined; + expect(body.parameters.negativePrompt).to.be.undefined; + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.addWatermark).to.be.undefined; + expect(body.parameters.safetyFilterLevel).to.be.undefined; + expect(body.parameters.personGeneration).to.be.undefined; + }); + }); + it('creates body with non-default request paramaters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const imageFormat = { mimeType: 'image/jpeg', compressionQuality: 75 }; + const safetySettings = { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_LOW_AND_ABOVE, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + }; + const addWatermark = true; + const numberOfImages = 4; + const negativePrompt = 'do not hallucinate'; + const aspectRatio = ImagenAspectRatio.WIDESCREEN; + const body = createPredictRequestBody({ + prompt, + numberOfImages, + imageFormat, + safetySettings, + addWatermark, + negativePrompt, + aspectRatio + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters).deep.equal({ + sampleCount: numberOfImages, + mimeType: imageFormat.mimeType, + compressionQuality: imageFormat.compressionQuality, + addWatermark, + negativePrompt, + safetyFilterLevel: safetySettings.safetyFilterLevel, + personFilterLevel: safetySettings.personFilterLevel, + aspectRatio, + includeRaiReason: true, + storageUri: undefined + }); + }); + it('creates body with GCS URI', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const gcsURI = 'gcs-uri'; + const body = createPredictRequestBody({ + prompt, + gcsURI + }); + + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.storageUri).to.equal(gcsURI); + }); }); diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 9e525b2a875..fa4bdf402d7 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -18,8 +18,11 @@ import { Content, GenerateContentRequest, + PredictRequestBody, Part, - VertexAIErrorCode + VertexAIErrorCode, + ImagenAspectRatio, + ImagenRequestConfig } from '../types'; import { VertexAIError } from '../errors'; @@ -49,11 +52,13 @@ export function formatNewContent( if (typeof request === 'string') { newParts = [{ text: request }]; } else { - for (const partOrString of request) { - if (typeof partOrString === 'string') { - newParts.push({ text: partOrString }); + for (const elem of request) { + // This throws an error if request is not iterable + if (typeof elem === 'string') { + newParts.push({ text: elem }); } else { - newParts.push(partOrString); + // We assume this is a Part, but it could be anything. + newParts.push(elem); // This could be } } } @@ -114,6 +119,7 @@ export function formatGenerateContentInput( formattedRequest = params as GenerateContentRequest; } else { // Array or string + // ... or something else const content = formatNewContent(params as string | Array); formattedRequest = { contents: [content] }; } @@ -124,3 +130,40 @@ export function formatGenerateContentInput( } return formattedRequest; } + +/** + * Convert the user-defined parameters in {@link ImagenRequestConfig} to the format + * that is expected from the REST API. + * + * @internal + */ +export function createPredictRequestBody({ + prompt, + gcsURI, + imageFormat = { mimeType: 'image/png' }, + addWatermark, + safetySettings, + numberOfImages = 1, + negativePrompt, + aspectRatio = ImagenAspectRatio.SQUARE +}: ImagenRequestConfig): PredictRequestBody { + // Properties that are undefined will be omitted from the JSON string. + const body: PredictRequestBody = { + instances: [ + { + prompt + } + ], + parameters: { + storageUri: gcsURI, + ...imageFormat, + addWatermark, + ...safetySettings, + sampleCount: numberOfImages, + includeRaiReason: true, + negativePrompt, + aspectRatio + } + }; + return body; +} diff --git a/packages/vertexai/src/requests/request.ts b/packages/vertexai/src/requests/request.ts index f81b40635e3..9b9465db776 100644 --- a/packages/vertexai/src/requests/request.ts +++ b/packages/vertexai/src/requests/request.ts @@ -30,7 +30,8 @@ import { logger } from '../logger'; export enum Task { GENERATE_CONTENT = 'generateContent', STREAM_GENERATE_CONTENT = 'streamGenerateContent', - COUNT_TOKENS = 'countTokens' + COUNT_TOKENS = 'countTokens', + PREDICT = 'predict' } export class RequestUrl { diff --git a/packages/vertexai/src/requests/response-helpers.test.ts b/packages/vertexai/src/requests/response-helpers.test.ts index 91a60d2cfce..4cab8cde047 100644 --- a/packages/vertexai/src/requests/response-helpers.test.ts +++ b/packages/vertexai/src/requests/response-helpers.test.ts @@ -15,7 +15,11 @@ * limitations under the License. */ -import { addHelpers, formatBlockErrorMessage } from './response-helpers'; +import { + addHelpers, + formatBlockErrorMessage, + handlePredictResponse +} from './response-helpers'; import { expect, use } from 'chai'; import { restore } from 'sinon'; import sinonChai from 'sinon-chai'; @@ -23,8 +27,11 @@ import { BlockReason, Content, FinishReason, - GenerateContentResponse + GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage } from '../types'; +import { getMockResponse } from '../../test-utils/mock-response'; use(sinonChai); @@ -246,4 +253,71 @@ describe('response-helpers methods', () => { ); }); }); + + describe('handlePredictResponse', () => { + it('returns base64 images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded.length).to.be.greaterThan(0); + }); + }); + }); + it('returns GCS images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-gcs.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach((image, i) => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.equal( + `gs://test-project-id-1234.firebasestorage.app/images/1234567890123/sample_${i}.jpg` + ); + }); + }); + it('has filtered reason and no images if all images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-all-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 39322892, 29310472" + ); + expect(res.images.length).to.equal(0); + }); + it('has filtered reason and no images if all base64 images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-base64-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded).to.have.length.greaterThan(0); + }); + }); + it('has filtered reason and no images if all GCS images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-gcs-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.have.length.greaterThan(0); + }); + }); }); diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 27347d10f0d..6b82463dfa1 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -21,10 +21,13 @@ import { FunctionCall, GenerateContentCandidate, GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage, VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; import { logger } from '../logger'; +import { ImagenResponseInternal } from '../types/internal'; /** * Creates an EnhancedGenerateContentResponse object that has helper functions and @@ -196,3 +199,52 @@ export function formatBlockErrorMessage( } return message; } + +/** + * Convert a generic successful fetch {@link Response} body to an Imagen response object + * that can be returned to the user. This converts the REST APIs response format to our + * representation of a response. + * + * @internal + */ + +export async function handlePredictResponse< + T extends ImagenInlineImage | ImagenGCSImage +>(response: Response): Promise<{ images: T[]; filteredReason?: string }> { + const responseJson: ImagenResponseInternal = await response.json(); + + const images: T[] = []; + let filteredReason: string | undefined = undefined; + + if (!responseJson.predictions || responseJson.predictions?.length === 0) { + throw new VertexAIError( + VertexAIErrorCode.ERROR, + "Predictions array is undefined or empty in response. Was 'includeRaiReason' enabled in the request?" + ); + } + + for (const prediction of responseJson.predictions) { + if (prediction.raiFilteredReason) { + filteredReason = prediction.raiFilteredReason; + } else if (prediction.mimeType && prediction.bytesBase64Encoded) { + images.push({ + mimeType: prediction.mimeType, + bytesBase64Encoded: prediction.bytesBase64Encoded + } as T); + } else if (prediction.mimeType && prediction.gcsUri) { + images.push({ + mimeType: prediction.mimeType, + gcsURI: prediction.gcsUri + } as T); + } else { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + `Predictions array in response has missing properties. Response: ${JSON.stringify( + responseJson + )}` + ); + } + } + + return { images, filteredReason }; +} diff --git a/packages/vertexai/src/types/imagen/index.ts b/packages/vertexai/src/types/imagen/index.ts new file mode 100644 index 00000000000..0037b185e50 --- /dev/null +++ b/packages/vertexai/src/types/imagen/index.ts @@ -0,0 +1,19 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './requests'; +export * from './responses'; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts new file mode 100644 index 00000000000..1171df81278 --- /dev/null +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -0,0 +1,54 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Internal Imagen types + +/** + * A response from the REST API is expected to look like this in the success case: + * { + * "predictions": [ + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "iVBORw0KG..." + * }, + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "i4BOtw0KG..." + * } + * ] + * } + * + * And like this in the failure case: + * { + * "predictions": [ + * { + * "raiFilteredReason": "..." + * } + * ] + * } + */ +export interface ImagenResponseInternal { + predictions?: Array<{ + // Defined if the prediction was not filtered + mimeType?: string; + bytesBase64Encoded?: string; + gcsUri?: string; + + // Defined if the prediction was filtered, and there is no image + raiFilteredReason?: string; + }>; +} diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts new file mode 100644 index 00000000000..d87723a31c7 --- /dev/null +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -0,0 +1,142 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface ImagenModelParams extends ImagenModelConfig { + model: string; +} + +export interface ImagenModelConfig { + imageFormat?: ImagenImageFormat; + addWatermark?: boolean; + safetySettings?: ImagenSafetySettings; +} + +export interface ImagenGenerationConfig { + numberOfImages?: number; // Default to 1. Possible values are [1...4] + negativePrompt?: string; // Default to null + aspectRatio?: ImagenAspectRatio; // Default to "1:1" +} + +/** + * Contains all possible REST API paramaters. + * This is the intersection of the model-level (`ImagenModelParams`), + * request-level (`ImagenGenerationConfig`) configurations, along with + * the other required parameters prompt and gcsURI (for GCS generation only). + * + * @internal + */ +export interface ImagenRequestConfig + extends ImagenModelConfig, + ImagenGenerationConfig { + prompt: string; + gcsURI?: string; +} + +export interface ImagenImageFormat { + mimeType: string; // image/png, or image/jpeg, default image/png + compressionQuality?: number; // 0-100, default 75. Only for image/jpeg +} + +/** + * @public + */ +export enum ImagenSafetyFilterLevel { + BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + BLOCK_ONLY_HIGH = 'block_only_high', + BLOCK_NONE = 'block_none' +} + +/** + * @public + */ +export enum ImagenPersonFilterLevel { + BLOCK_ALL = 'dont_allow', + ALLOW_ADULT = 'allow_adult', + ALLOW_ALL = 'allow_all' +} + +/** + * @public + */ +export interface ImagenSafetySettings { + /** + * Safety filter level + */ + safetyFilterLevel?: ImagenSafetyFilterLevel; + /** + * Generate people. + */ + personFilterLevel?: ImagenPersonFilterLevel; +} + +export enum ImagenAspectRatio { + SQUARE = '1:1', + CLASSIC_PORTRAIT = '3:4', + CLASSIC_LANDSCAPE = '4:3', + WIDESCREEN = '16:9', + PORTRAIT = '9:16' +} + +/** + * The parameters to be sent in the request body of the HTTP call + * to the Vertex AI backend. + * + * We need a seperate internal-only interface for this because the REST + * API expects different parameter names than what we show to our users. + * + * This interface should be populated from the {@link ImagenGenerationConfig} that + * the user defines. + * + * Sample request body JSON: + * { + * "instances": [ + * { + * "prompt": "Portrait of a golden retriever on a beach." + * } + * ], + * "parameters": { + * "mimeType": "image/png", + * "safetyFilterLevel": "block_low_and_above", + * "personGeneration": "allow_all", + * "sampleCount": 2, + * "includeRaiReason": true, + * "aspectRatio": "9:16" + * } + * } + * + * @internal + */ +export interface PredictRequestBody { + instances: [ + { + prompt: string; + } + ]; + parameters: { + sampleCount: number; // maps to numberOfImages + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; + includeRaiReason: boolean; + }; +} diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts new file mode 100644 index 00000000000..620f49be4b9 --- /dev/null +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Base class for types of images that the Imagen Model can return. + * + * @internal + */ +export interface ImagenImage { + mimeType: string; +} + +/** + * Image generated by Imagen to inline bytes. + * + * @public + */ +export interface ImagenInlineImage extends ImagenImage { + bytesBase64Encoded: string; +} + +/** + * Image generated by Imagen, stored in Google Cloud Storage (GCS). + * + * @public + */ +export interface ImagenGCSImage extends ImagenImage { + /** + * The Google Cloud Storage (GCS) URI at which the generated image is stored. + */ + gcsURI: string; +} + +/** + * Imagen image response. + * + * @public + */ +export interface ImagenInlineImageResponse { + /** + * The images generated by Imagen. If all images were filtered, this will be empty. + */ + images: ImagenInlineImage[]; + /** + * The reason the missing images were filtered. + * For the mappings of error codes to reasons, see {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + */ + filteredReason?: string; +} + +export interface ImagenGCSImageResponse { + images: ImagenGCSImage[]; + filteredReason?: string; +} + +export interface ImagenImageReponse { + images: ImagenImage[]; + filteredReason?: string; +} diff --git a/packages/vertexai/src/types/index.ts b/packages/vertexai/src/types/index.ts index 85133aa07c5..f575c5ba8e9 100644 --- a/packages/vertexai/src/types/index.ts +++ b/packages/vertexai/src/types/index.ts @@ -21,3 +21,4 @@ export * from './requests'; export * from './responses'; export * from './error'; export * from './schema'; +export * from './imagen'; diff --git a/packages/vertexai/src/types/internal.ts b/packages/vertexai/src/types/internal.ts index 8271175feff..87c28a02ab2 100644 --- a/packages/vertexai/src/types/internal.ts +++ b/packages/vertexai/src/types/internal.ts @@ -18,6 +18,8 @@ import { AppCheckTokenResult } from '@firebase/app-check-interop-types'; import { FirebaseAuthTokenData } from '@firebase/auth-interop-types'; +export * from './imagen/internal'; + export interface ApiSettings { apiKey: string; project: string; diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 83cd4366f12..e2a442821da 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -46,6 +46,11 @@ export interface GenerateContentStreamResult { response: Promise; } +export interface SafetyAttributes { + categories: string[]; + scores: number[]; +} + /** * Response object wrapped with helper methods. * From 180f091d6bd8704ce1f1fa4553e417bf0b19c3cb Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:38:21 -0600 Subject: [PATCH 02/31] Introduce VertexAIModel base class, add documentation, and respond to other comments --- common/api-review/vertexai.api.md | 145 ++++------------- packages/vertexai/src/api.test.ts | 2 +- packages/vertexai/src/api.ts | 10 +- .../src/models/generative-model.test.ts | 22 --- .../vertexai/src/models/generative-model.ts | 50 +----- .../vertexai/src/models/imagen-model.test.ts | 66 +------- packages/vertexai/src/models/imagen-model.ts | 151 +++++++---------- packages/vertexai/src/models/index.ts | 3 + .../src/models/vertexai-model.test.ts | 103 ++++++++++++ .../vertexai/src/models/vertexai-model.ts | 94 +++++++++++ .../src/requests/imagen-image-format.ts | 48 ++++++ .../src/requests/request-helpers.test.ts | 2 +- .../vertexai/src/requests/request-helpers.ts | 12 +- .../vertexai/src/requests/response-helpers.ts | 4 +- .../vertexai/src/types/imagen/internal.ts | 85 +++++++++- .../vertexai/src/types/imagen/requests.ts | 152 +++++++++--------- .../vertexai/src/types/imagen/responses.ts | 61 ++++--- packages/vertexai/src/types/responses.ts | 5 - 18 files changed, 552 insertions(+), 463 deletions(-) create mode 100644 packages/vertexai/src/models/index.ts create mode 100644 packages/vertexai/src/models/vertexai-model.test.ts create mode 100644 packages/vertexai/src/models/vertexai-model.ts create mode 100644 packages/vertexai/src/requests/imagen-image-format.ts diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index aa35a3f024f..5b5c00b122e 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -323,7 +323,7 @@ export interface GenerativeContentBlob { } // @public -export class GenerativeModel { +export class GenerativeModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions); countTokens(request: CountTokensRequest | string | Array): Promise; generateContent(request: GenerateContentRequest | string | Array): Promise; @@ -331,8 +331,6 @@ export class GenerativeModel { // (undocumented) generationConfig: GenerationConfig; // (undocumented) - model: string; - // (undocumented) requestOptions?: RequestOptions; // (undocumented) safetySettings: SafetySetting[]; @@ -432,77 +430,53 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } -// @public (undocumented) +// @public export enum ImagenAspectRatio { // (undocumented) - CLASSIC_LANDSCAPE = "4:3", + LANDSCAPE_16x9 = "16:9", // (undocumented) - CLASSIC_PORTRAIT = "3:4", + LANDSCAPE_3x4 = "3:4", // (undocumented) - PORTRAIT = "9:16", + PORTRAIT_4x3 = "4:3", // (undocumented) - SQUARE = "1:1", + PORTRAIT_9x16 = "9:16", // (undocumented) - WIDESCREEN = "16:9" + SQUARE = "1:1" } -// Warning: (ae-incompatible-release-tags) The symbol "ImagenGCSImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal -// // @public -export interface ImagenGCSImage extends ImagenImage { +export interface ImagenGCSImage { gcsURI: string; + mimeType: string; } -// @public (undocumented) +// @public export interface ImagenGCSImageResponse { - // (undocumented) filteredReason?: string; - // (undocumented) images: ImagenGCSImage[]; } -// @public (undocumented) +// @public export interface ImagenGenerationConfig { - // (undocumented) aspectRatio?: ImagenAspectRatio; - // (undocumented) negativePrompt?: string; - // (undocumented) numberOfImages?: number; } -// Warning: (ae-internal-missing-underscore) The name "ImagenImage" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal -export interface ImagenImage { - // (undocumented) - mimeType: string; -} - -// @public (undocumented) -export interface ImagenImageFormat { +// @public +export class ImagenImageFormat { // (undocumented) compressionQuality?: number; + static jpeg(compressionQuality: number): ImagenImageFormat; // (undocumented) mimeType: string; + static png(): ImagenImageFormat; } -// @public (undocumented) -export interface ImagenImageReponse { - // (undocumented) - filteredReason?: string; - // Warning: (ae-incompatible-release-tags) The symbol "images" is marked as @public, but its signature references "ImagenImage" which is marked as @internal - // - // (undocumented) - images: ImagenImage[]; -} - -// Warning: (ae-incompatible-release-tags) The symbol "ImagenInlineImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal -// // @public -export interface ImagenInlineImage extends ImagenImage { - // (undocumented) +export interface ImagenInlineImage { bytesBase64Encoded: string; + mimeType: string; } // @public @@ -512,63 +486,43 @@ export interface ImagenInlineImageResponse { } // @public -export class ImagenModel { +export class ImagenModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + readonly modelConfig: ImagenModelConfig; // (undocumented) - model: string; - } + readonly requestOptions?: RequestOptions | undefined; +} -// @public (undocumented) +// @public export interface ImagenModelConfig { - // (undocumented) addWatermark?: boolean; - // (undocumented) imageFormat?: ImagenImageFormat; - // (undocumented) safetySettings?: ImagenSafetySettings; } // @public export interface ImagenModelParams extends ImagenModelConfig { - // (undocumented) model: string; } -// @public (undocumented) +// @public export enum ImagenPersonFilterLevel { - // (undocumented) ALLOW_ADULT = "allow_adult", - // (undocumented) ALLOW_ALL = "allow_all", - // (undocumented) BLOCK_ALL = "dont_allow" } -// Warning: (ae-internal-missing-underscore) The name "ImagenRequestConfig" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal -export interface ImagenRequestConfig extends ImagenModelConfig, ImagenGenerationConfig { - // (undocumented) - gcsURI?: string; - // (undocumented) - prompt: string; -} - -// @public (undocumented) +// @public export enum ImagenSafetyFilterLevel { - // (undocumented) BLOCK_LOW_AND_ABOVE = "block_low_and_above", - // (undocumented) BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", - // (undocumented) BLOCK_NONE = "block_none", - // (undocumented) BLOCK_ONLY_HIGH = "block_only_high" } -// @public (undocumented) +// @public export interface ImagenSafetySettings { personFilterLevel?: ImagenPersonFilterLevel; safetyFilterLevel?: ImagenSafetyFilterLevel; @@ -592,9 +546,6 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } -// @public -export function jpeg(compressionQuality: number): ImagenImageFormat; - // @public export interface ModelParams extends BaseParams { // (undocumented) @@ -638,37 +589,9 @@ export interface ObjectSchemaInterface extends SchemaInterface { // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; -// @public -export function png(): ImagenImageFormat; - // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; -// Warning: (ae-internal-missing-underscore) The name "PredictRequestBody" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal -export interface PredictRequestBody { - // (undocumented) - instances: [ - { - prompt: string; - } - ]; - // (undocumented) - parameters: { - sampleCount: number; - aspectRatio: string; - mimeType: string; - compressionQuality?: number; - negativePrompt?: string; - storageUri?: string; - addWatermark?: boolean; - safetyFilterLevel?: string; - personGeneration?: string; - includeRaiReason: boolean; - }; -} - // @public export interface PromptFeedback { // (undocumented) @@ -696,14 +619,6 @@ export interface RetrievedContextAttribution { // @public export type Role = (typeof POSSIBLE_ROLES)[number]; -// @public (undocumented) -export interface SafetyAttributes { - // (undocumented) - categories: string[]; - // (undocumented) - scores: number[]; -} - // @public export interface SafetyRating { // (undocumented) @@ -902,6 +817,16 @@ export const enum VertexAIErrorCode { RESPONSE_ERROR = "response-error" } +// @public +export class VertexAIModel { + // @internal + protected constructor(vertexAI: VertexAI, modelName: string); + // (undocumented) + protected _apiSettings: ApiSettings; + readonly model: string; + static normalizeModelName(modelName: string): string; +} + // @public export interface VertexAIOptions { // (undocumented) diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index c9432d2a7ea..c1b2635ce70 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -129,7 +129,7 @@ describe('Top level API', () => { ); } }); - it('getGenerativeModel gets an ImagenModel', () => { + it('getImagenModel gets an ImagenModel', () => { const genModel = getImagenModel(fakeVertexAI, { model: 'my-model' }); expect(genModel).to.be.an.instanceOf(ImagenModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 07154356435..6ad61798c4f 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -28,16 +28,12 @@ import { VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; -import { GenerativeModel } from './models/generative-model'; -import { ImagenModel, jpeg, png } from './models/imagen-model'; +import { VertexAIModel, GenerativeModel, ImagenModel } from './models'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; - -export { jpeg, png }; - -export { GenerativeModel, ImagenModel }; - +export { ImagenImageFormat } from './requests/imagen-image-format'; +export { VertexAIModel, GenerativeModel, ImagenModel }; export { VertexAIError }; declare module '@firebase/component' { diff --git a/packages/vertexai/src/models/generative-model.test.ts b/packages/vertexai/src/models/generative-model.test.ts index e03f39e8a83..c2dbdfac75c 100644 --- a/packages/vertexai/src/models/generative-model.test.ts +++ b/packages/vertexai/src/models/generative-model.test.ts @@ -37,28 +37,6 @@ const fakeVertexAI: VertexAI = { }; describe('GenerativeModel', () => { - it('handles plain model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles models/ prefixed model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'models/my-model' - }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles full model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'publishers/google/models/my-model' - }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles prefixed tuned model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'tunedModels/my-model' - }); - expect(genModel.model).to.equal('tunedModels/my-model'); - }); it('passes params through to generateContent', async () => { const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model', diff --git a/packages/vertexai/src/models/generative-model.ts b/packages/vertexai/src/models/generative-model.ts index e719529967c..b4cf464f025 100644 --- a/packages/vertexai/src/models/generative-model.ts +++ b/packages/vertexai/src/models/generative-model.ts @@ -33,10 +33,8 @@ import { SafetySetting, StartChatParams, Tool, - ToolConfig, - VertexAIErrorCode + ToolConfig } from '../types'; -import { VertexAIError } from '../errors'; import { ChatSession } from '../methods/chat-session'; import { countTokens } from '../methods/count-tokens'; import { @@ -44,16 +42,13 @@ import { formatSystemInstruction } from '../requests/request-helpers'; import { VertexAI } from '../public-types'; -import { ApiSettings } from '../types/internal'; -import { VertexAIService } from '../service'; +import { VertexAIModel } from './vertexai-model'; /** * Class for generative model APIs. * @public */ -export class GenerativeModel { - private _apiSettings: ApiSettings; - model: string; +export class GenerativeModel extends VertexAIModel { generationConfig: GenerationConfig; safetySettings: SafetySetting[]; requestOptions?: RequestOptions; @@ -66,44 +61,7 @@ export class GenerativeModel { modelParams: ModelParams, requestOptions?: RequestOptions ) { - if (!vertexAI.app?.options?.apiKey) { - throw new VertexAIError( - VertexAIErrorCode.NO_API_KEY, - `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` - ); - } else if (!vertexAI.app?.options?.projectId) { - throw new VertexAIError( - VertexAIErrorCode.NO_PROJECT_ID, - `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` - ); - } else { - this._apiSettings = { - apiKey: vertexAI.app.options.apiKey, - project: vertexAI.app.options.projectId, - location: vertexAI.location - }; - if ((vertexAI as VertexAIService).appCheck) { - this._apiSettings.getAppCheckToken = () => - (vertexAI as VertexAIService).appCheck!.getToken(); - } - - if ((vertexAI as VertexAIService).auth) { - this._apiSettings.getAuthToken = () => - (vertexAI as VertexAIService).auth!.getToken(); - } - } - if (modelParams.model.includes('/')) { - if (modelParams.model.startsWith('models/')) { - // Add "publishers/google" if the user is only passing in 'models/model-name'. - this.model = `publishers/google/${modelParams.model}`; - } else { - // Any other custom format (e.g. tuned models) must be passed in correctly. - this.model = modelParams.model; - } - } else { - // If path is not included, assume it's a non-tuned model. - this.model = `publishers/google/models/${modelParams.model}`; - } + super(vertexAI, modelParams.model); this.generationConfig = modelParams.generationConfig || {}; this.safetySettings = modelParams.safetySettings || []; this.tools = modelParams.tools; diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index 909bff3dea2..a0bc857a53b 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -44,68 +44,6 @@ const fakeVertexAI: VertexAI = { }; describe('ImagenModel', () => { - it('handles plain model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' }); - expect(imagenModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles models/ prefixed model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'models/my-model' - }); - expect(imagenModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles full model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'publishers/google/models/my-model' - }); - expect(imagenModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles prefixed tuned model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'tunedModels/my-model' - }); - expect(imagenModel.model).to.equal('tunedModels/my-model'); - }); - it('throws if not passed an api key', () => { - const fakeVertexAI: VertexAI = { - app: { - name: 'DEFAULT', - automaticDataCollectionEnabled: true, - options: { - projectId: 'my-project' - } - }, - location: 'us-central1' - }; - try { - new ImagenModel(fakeVertexAI, { - model: 'my-model' - }); - } catch (e) { - expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); - } - }); - it('throws if not passed a project ID', () => { - const fakeVertexAI: VertexAI = { - app: { - name: 'DEFAULT', - automaticDataCollectionEnabled: true, - options: { - apiKey: 'key' - } - }, - location: 'us-central1' - }; - try { - new ImagenModel(fakeVertexAI, { - model: 'my-model' - }); - } catch (e) { - expect((e as VertexAIError).code).to.equal( - VertexAIErrorCode.NO_PROJECT_ID - ); - } - }); it('generateImages makes a request to predict with default parameters', async () => { const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' @@ -190,7 +128,7 @@ describe('ImagenModel', () => { const prompt = 'A photorealistic image of a toy boat at sea.'; await imagenModel.generateImages(prompt, { numberOfImages: 4, - aspectRatio: ImagenAspectRatio.WIDESCREEN, + aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, negativePrompt: 'do not hallucinate' }); expect(makeRequestStub).to.be.calledWith( @@ -237,7 +175,7 @@ describe('ImagenModel', () => { expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); expect((e as VertexAIError).message).to.include('400'); expect((e as VertexAIError).message).to.include( - "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 42876398" + "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback." ); } finally { restore(); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index 17d1e1cfb2c..3434e1dd99d 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -15,102 +15,85 @@ * limitations under the License. */ -import { VertexAIError } from '../errors'; import { VertexAI } from '../public-types'; import { Task, makeRequest } from '../requests/request'; import { createPredictRequestBody } from '../requests/request-helpers'; import { handlePredictResponse } from '../requests/response-helpers'; -import { VertexAIService } from '../service'; import { ImagenGCSImage, ImagenGCSImageResponse, - ImagenImageFormat, ImagenGenerationConfig, ImagenInlineImage, RequestOptions, - VertexAIErrorCode, ImagenModelParams, ImagenInlineImageResponse, ImagenModelConfig } from '../types'; -import { ApiSettings } from '../types/internal'; +import { VertexAIModel } from './vertexai-model'; /** * Class for Imagen model APIs. + * + * This class provides methods for generating images using the Imagen model. + * You can generate images inline as base64-encoded strings, or directly to + * Google Cloud Storage (GCS). + * + * @example + * ```javascript + * const imagen = new ImagenModel(vertexAI, { + * model: 'imagen-3.0-generate-001' + * }); + * + * const response = await imagen.generateImages('A photo of a cat'); + * console.log(response.images[0].bytesBase64Encoded); + * ``` + * * @public */ -export class ImagenModel { - model: string; - private _apiSettings: ApiSettings; - private modelConfig: ImagenModelConfig; +export class ImagenModel extends VertexAIModel { + /** + * Model-level configurations to use when using Imagen. + */ + readonly modelConfig: ImagenModelConfig; /** + * Constructs a new instance of the {@link ImagenModel} class. * - * @param vertexAI - * @param modelParams - * @param requestOptions + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelParams - Parameters to use when making Imagen requests. + * @param requestOptions - Additional options to use when making requests. + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. */ constructor( vertexAI: VertexAI, modelParams: ImagenModelParams, - private requestOptions?: RequestOptions + readonly requestOptions?: RequestOptions ) { const { model, ...modelConfig } = modelParams; + super(vertexAI, model); this.modelConfig = modelConfig; - if (model.includes('/')) { - if (model.startsWith('models/')) { - // Add "publishers/google" if the user is only passing in 'models/model-name'. - this.model = `publishers/google/${model}`; - } else { - // Any other custom format (e.g. tuned models) must be passed in correctly. - this.model = model; - } - } else { - // If path is not included, assume it's a non-tuned model. - this.model = `publishers/google/models/${model}`; - } - - if (!vertexAI.app?.options?.apiKey) { - throw new VertexAIError( - VertexAIErrorCode.NO_API_KEY, - `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` - ); - } else if (!vertexAI.app?.options?.projectId) { - throw new VertexAIError( - VertexAIErrorCode.NO_PROJECT_ID, - `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` - ); - } else { - this._apiSettings = { - apiKey: vertexAI.app.options.apiKey, - project: vertexAI.app.options.projectId, - location: vertexAI.location - }; - if ((vertexAI as VertexAIService).appCheck) { - this._apiSettings.getAppCheckToken = () => - (vertexAI as VertexAIService).appCheck!.getToken(); - } - - if ((vertexAI as VertexAIService).auth) { - this._apiSettings.getAuthToken = () => - (vertexAI as VertexAIService).auth!.getToken(); - } - } } /** - * Generates images using the Imagen model and returns them as base64-encoded strings. + * Generates images using the Imagen model and returns them as + * base64-encoded strings. * - * @param prompt The text prompt used to generate the images. - * @param imagenRequestOptions Configuration options for the Imagen generation request. + * @param prompt - The text prompt used to generate the images. + * @param imagenRequestOptions - Configuration options for the Imagen + * generation request. * See {@link ImagenGenerationConfig}. - * @returns A promise that resolves to an {@link ImagenInlineImageResponse} object containing the generated images. + * @returns A promise that resolves to an {@link ImagenInlineImageResponse} + * object containing the generated images. * - * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * @throws If the request to generate images fails. This happens if the + * prompt is blocked. * * @remarks - * If one or more images are filtered, the returned object will have a defined `filteredReason` property. - * If all images are filtered, the `images` array will be empty, and no error will be thrown. + * If one or more images are filtered, the returned object will have a + * defined `filteredReason` property. If all images are filtered, the + * `images` array will be empty, and no error will be thrown. */ async generateImages( prompt: string, @@ -133,19 +116,23 @@ export class ImagenModel { } /** - * Generates images using the Imagen model and returns them as base64-encoded strings. + * Generates images to Google Cloud Storage (GCS) using the Imagen model. * - * @param prompt The text prompt used to generate the images. - * @param gcsURI The GCS URI where the images should be stored. - * @param imagenRequestOptions Configuration options for the Imagen generation request. - * See {@link ImagenGenerationConfig}. - * @returns A promise that resolves to an {@link ImagenGCSImageResponse} object containing the generated images. + * @param prompt - The text prompt used to generate the images. + * @param gcsURI - The GCS URI where the images should be stored. + * This should be a directory. For example, `gs://my-bucket/my-directory/`. + * @param imagenRequestOptions - Configuration options for the Imagen + * generation request. See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenGCSImageResponse} + * object containing the URLs of the generated images. * - * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * @throws If the request fails to generate images fails. This happens if + * the prompt is blocked. * * @remarks - * If one or more images are filtered, the returned object will have a defined `filteredReason` property. - * If all images are filtered, the `images` array will be empty, and no error will be thrown. + * If one or more images are filtered due to safety reasons, the returned object + * will have a defined `filteredReason` property. If all images are filtered, + * the `images` array will be empty, and no error will be thrown. */ async generateImagesGCS( prompt: string, @@ -169,31 +156,3 @@ export class ImagenModel { return handlePredictResponse(response); } } - -/** - * Creates an {@link ImagenImageFormat} for a JPEG image, to be included in an {@link ImagenModelParams}. - * - * @param compressionQuality The level of compression. - * @returns {@link ImagenImageFormat} - * - * @public - */ -export function jpeg(compressionQuality: number): ImagenImageFormat { - return { - mimeType: 'image/jpeg', - compressionQuality - }; -} - -/** - * Creates an {@link ImageImageFormat} for a PNG image, to be included in a {@link ImagenModelParams}. - * - * @returns {@link ImageImageFormat} - * - * @public - */ -export function png(): ImagenImageFormat { - return { - mimeType: 'image/png' - }; -} diff --git a/packages/vertexai/src/models/index.ts b/packages/vertexai/src/models/index.ts new file mode 100644 index 00000000000..a6ada0d894c --- /dev/null +++ b/packages/vertexai/src/models/index.ts @@ -0,0 +1,3 @@ +export * from './vertexai-model'; +export * from './generative-model'; +export * from './imagen-model'; \ No newline at end of file diff --git a/packages/vertexai/src/models/vertexai-model.test.ts b/packages/vertexai/src/models/vertexai-model.test.ts new file mode 100644 index 00000000000..6b5cdd48e2a --- /dev/null +++ b/packages/vertexai/src/models/vertexai-model.test.ts @@ -0,0 +1,103 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { VertexAI, VertexAIErrorCode } from '../public-types'; +import sinonChai from 'sinon-chai'; +import { VertexAIModel } from './vertexai-model'; +import { VertexAIError } from '../errors'; + +use(sinonChai); + +/** + * A class that extends VertexAIModel that allows us to test the protected constructor. + */ +class TestModel extends VertexAIModel { + /* eslint-disable @typescript-eslint/no-useless-constructor */ + constructor( + vertexAI: VertexAI, + modelName: string + ) { + super(vertexAI, modelName); + } +} + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('VertexAIModel', () => { + it('handles plain model name', () => { + const testModel = new TestModel(fakeVertexAI, 'my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles models/ prefixed model name', () => { + const testModel = new TestModel(fakeVertexAI, 'models/my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles full model name', () => { + const testModel = new TestModel(fakeVertexAI, 'publishers/google/models/my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles prefixed tuned model name', () => { + const testModel = new TestModel(fakeVertexAI, 'tunedModels/my-model'); + expect(testModel.model).to.equal('tunedModels/my-model'); + }); + it('throws if not passed an api key', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + projectId: 'my-project' + } + }, + location: 'us-central1' + }; + try { + new TestModel(fakeVertexAI, 'my-model'); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); + } + }); + it('throws if not passed a project ID', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key' + } + }, + location: 'us-central1' + }; + try { + new TestModel(fakeVertexAI, 'my-model'); + } catch (e) { + expect((e as VertexAIError).code).to.equal( + VertexAIErrorCode.NO_PROJECT_ID + ); + } + }); +}); diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts new file mode 100644 index 00000000000..0a05eb6fea6 --- /dev/null +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -0,0 +1,94 @@ +import { VertexAIError } from "../errors"; +import { VertexAI, VertexAIErrorCode } from "../public-types"; +import { VertexAIService } from "../service"; +import { ApiSettings } from "../types/internal"; + +/** + * Base class for Vertex AI in Firebase model APIs. + * + * @public + */ +export class VertexAIModel { + /** + * The fully qualified model resource name to use for generating images + * (e.g. `publishers/google/models/imagen-3.0-generate-001`). + */ + readonly model: string; + + protected _apiSettings: ApiSettings; + + /** + * Constructs a new instance of the {@link VertexAIModel} class. + * + * This constructor should only be called from subclasses that provide + * a model API. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelName - The name of the model being used. It can be in one of the following formats: + * - `my-model` (short name, will resolve to `publishers/google/models/my-model`) + * - `models/my-model` (will resolve to `publishers/google/models/my-model`) + * - `publishers/my-publisher/models/my-model` (fully qualified model name) + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + * + * @internal + */ + protected constructor( + vertexAI: VertexAI, + modelName: string + ) { + this.model = VertexAIModel.normalizeModelName(modelName); + + if (!vertexAI.app?.options?.apiKey) { + throw new VertexAIError( + VertexAIErrorCode.NO_API_KEY, + `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` + ); + } else if (!vertexAI.app?.options?.projectId) { + throw new VertexAIError( + VertexAIErrorCode.NO_PROJECT_ID, + `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` + ); + } else { + this._apiSettings = { + apiKey: vertexAI.app.options.apiKey, + project: vertexAI.app.options.projectId, + location: vertexAI.location + }; + if ((vertexAI as VertexAIService).appCheck) { + this._apiSettings.getAppCheckToken = () => + (vertexAI as VertexAIService).appCheck!.getToken(); + } + + if ((vertexAI as VertexAIService).auth) { + this._apiSettings.getAuthToken = () => + (vertexAI as VertexAIService).auth!.getToken(); + } + } + } + + /** + * Normalizes the given model name to a fully qualified model resource name. + * + * @param modelName - The model name to normalize. + * @returns The fully qualified model resource name. + */ + static normalizeModelName(modelName: string): string { + let model: string; + if (modelName.includes('/')) { + if (modelName.startsWith('models/')) { + // Add 'publishers/google' if the user is only passing in 'models/model-name'. + model = `publishers/google/${modelName}`; + } else { + // Any other custom format (e.g. tuned models) must be passed in correctly. + model = modelName; + } + } else { + // If path is not included, assume it's a non-tuned model. + model = `publishers/google/models/${modelName}`; + } + + return model; + } +} \ No newline at end of file diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts new file mode 100644 index 00000000000..6b7c84d8f2b --- /dev/null +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -0,0 +1,48 @@ +/** + * Defines the image format for images output by Imagen. + * + * Use this class to specify the desired format (JPEG or PNG) and compression quality + * for images generated by Imagen. This is typically included as part of + * {@link ImagenModelParams}. + * + * @example + * ```javascript + * const imagenModelParams = { + * // ... other ImagenModelParams + * imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. + * } + * ``` + * + * @public + */ +export class ImagenImageFormat { + mimeType: string; + compressionQuality?: number; + + private constructor() { + this.mimeType = "image/png"; + } + + /** + * Creates an ImagenImageFormat for a JPEG image. + * + * @param compressionQuality - The level of compression (a number between 0 and 100). + * @returns ImagenImageFormat + * + * @public + */ + static jpeg(compressionQuality: number): ImagenImageFormat { + return { mimeType: "image/jpeg", compressionQuality }; + } + + /** + * Creates an ImageImageFormat for a PNG image. + * + * @returns ImageImageFormat + * + * @public + */ + static png(): ImagenImageFormat { + return { mimeType: "image/png" }; + } +} \ No newline at end of file diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 9582f384a2c..2317457aca4 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -239,7 +239,7 @@ describe('request formatting methods', () => { const addWatermark = true; const numberOfImages = 4; const negativePrompt = 'do not hallucinate'; - const aspectRatio = ImagenAspectRatio.WIDESCREEN; + const aspectRatio = ImagenAspectRatio.LANDSCAPE_16x9; const body = createPredictRequestBody({ prompt, numberOfImages, diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index fa4bdf402d7..831bb6a26a5 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -18,13 +18,13 @@ import { Content, GenerateContentRequest, - PredictRequestBody, Part, VertexAIErrorCode, ImagenAspectRatio, - ImagenRequestConfig } from '../types'; import { VertexAIError } from '../errors'; +import { ImagenImageFormat } from './imagen-image-format'; +import { ImagenRequestConfig, PredictRequestBody } from '../types/internal'; export function formatSystemInstruction( input?: string | Part | Content @@ -53,12 +53,10 @@ export function formatNewContent( newParts = [{ text: request }]; } else { for (const elem of request) { - // This throws an error if request is not iterable if (typeof elem === 'string') { newParts.push({ text: elem }); } else { - // We assume this is a Part, but it could be anything. - newParts.push(elem); // This could be + newParts.push(elem); } } } @@ -118,8 +116,6 @@ export function formatGenerateContentInput( if ((params as GenerateContentRequest).contents) { formattedRequest = params as GenerateContentRequest; } else { - // Array or string - // ... or something else const content = formatNewContent(params as string | Array); formattedRequest = { contents: [content] }; } @@ -140,7 +136,7 @@ export function formatGenerateContentInput( export function createPredictRequestBody({ prompt, gcsURI, - imageFormat = { mimeType: 'image/png' }, + imageFormat = ImagenImageFormat.png(), addWatermark, safetySettings, numberOfImages = 1, diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 6b82463dfa1..acdc6e741d1 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -201,9 +201,9 @@ export function formatBlockErrorMessage( } /** - * Convert a generic successful fetch {@link Response} body to an Imagen response object + * Convert a generic successful fetch response body to an Imagen response object * that can be returned to the user. This converts the REST APIs response format to our - * representation of a response. + * APIs representation of a response. * * @internal */ diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 1171df81278..ba7ee2c8f49 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -// Internal Imagen types +import { ImagenGenerationConfig, ImagenModelConfig } from "./requests"; /** * A response from the REST API is expected to look like this in the success case: @@ -43,12 +43,91 @@ */ export interface ImagenResponseInternal { predictions?: Array<{ - // Defined if the prediction was not filtered + /** + * The MIME type of the generated image. + */ mimeType?: string; + /** + * The image data encoded as a base64 string. + */ bytesBase64Encoded?: string; + gcsUri?: string; - // Defined if the prediction was filtered, and there is no image + /** + * The reason why the image was filtered. + */ raiFilteredReason?: string; }>; } + +/** + * The parameters to be sent in the request body of the HTTP call + * to the Vertex AI backend. + * + * We need a seperate internal-only interface for this because the REST + * API expects different parameter names than what we show to our users. + * + * This interface should be populated from the ImagenGenerationConfig that + * the user defines. + * + * Sample request body JSON: + * { + * "instances": [ + * { + * "prompt": "Portrait of a golden retriever on a beach." + * } + * ], + * "parameters": { + * "mimeType": "image/png", + * "safetyFilterLevel": "block_low_and_above", + * "personGeneration": "allow_all", + * "sampleCount": 2, + * "includeRaiReason": true, + * "aspectRatio": "9:16" + * } + * } + * + * @internal + */ +export interface PredictRequestBody { + instances: [ + { + prompt: string; + } + ]; + parameters: { + sampleCount: number; // Maps to numberOfImages + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; // Maps to gcsURI + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; // Maps to personFilterLevel + includeRaiReason: boolean; + }; +} + +/** + * Contains all possible REST API paramaters. + * This is the intersection of the model-level (`ImagenModelParams`), + * request-level (`ImagenGenerationConfig`) configurations, along with + * the other required parameters prompt and gcsURI (for GCS generation only). + * + * @internal + */ +export interface ImagenRequestConfig + extends ImagenModelConfig, + ImagenGenerationConfig { + /** + * The text prompt used to generate the images. + */ + prompt: string; + /** + * The Google Cloud Storage (GCS) URI where the images should be stored + * (for GCS requests only). + */ + gcsURI?: string; +} \ No newline at end of file diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index d87723a31c7..a9a9c664b45 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -15,128 +15,130 @@ * limitations under the License. */ +import { ImagenImageFormat } from '../../requests/imagen-image-format'; + +/** + * Parameters for configuring an {@link ImagenModel}. + * + * @public + */ export interface ImagenModelParams extends ImagenModelConfig { + /** + * The Imagen model to use for generating images. + * For example: `imagen-3.0-generate-001`. + */ model: string; } +/** + * Model-level configuration options for Imagen. + * + * @public + */ export interface ImagenModelConfig { + /** + * The image format of the generated images. + */ imageFormat?: ImagenImageFormat; + /** + * Whether to add a watermark to generated images. + */ addWatermark?: boolean; + /** + * Safety settings for filtering inapropriate content. + */ safetySettings?: ImagenSafetySettings; } -export interface ImagenGenerationConfig { - numberOfImages?: number; // Default to 1. Possible values are [1...4] - negativePrompt?: string; // Default to null - aspectRatio?: ImagenAspectRatio; // Default to "1:1" -} - /** - * Contains all possible REST API paramaters. - * This is the intersection of the model-level (`ImagenModelParams`), - * request-level (`ImagenGenerationConfig`) configurations, along with - * the other required parameters prompt and gcsURI (for GCS generation only). + * Request-level configuration options for generating images with Imagen. * - * @internal + * @public */ -export interface ImagenRequestConfig - extends ImagenModelConfig, - ImagenGenerationConfig { - prompt: string; - gcsURI?: string; -} - -export interface ImagenImageFormat { - mimeType: string; // image/png, or image/jpeg, default image/png - compressionQuality?: number; // 0-100, default 75. Only for image/jpeg +export interface ImagenGenerationConfig { + /** + * The number of images to generate. Must be between 1 and 4. Defaults to 1. + */ + numberOfImages?: number; + /** + * A text prompt describing what should not be included in the image. + */ + negativePrompt?: string; + /** + * The aspect ratio of the generated images. Defaults to `1:1`. + */ + aspectRatio?: ImagenAspectRatio; } /** + * Safety filter levels for Imagen. + * * @public */ export enum ImagenSafetyFilterLevel { + /** + * Block images with low or higher safety severity. + */ BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + /** + * Block images with medium or higher safety severity. + */ BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + /** + * Block images with high safety severity. + */ BLOCK_ONLY_HIGH = 'block_only_high', + /** + * Do not block any images based on safety. + */ BLOCK_NONE = 'block_none' } /** + * Person filter levels for Imagen. + * * @public */ export enum ImagenPersonFilterLevel { + /** + * Do not allow any person generation. + */ BLOCK_ALL = 'dont_allow', + /** + * Allow only adults in generated images. + */ ALLOW_ADULT = 'allow_adult', + /** + * Allow all person generation. + */ ALLOW_ALL = 'allow_all' } /** + * Safety settings for Imagen. + * * @public */ export interface ImagenSafetySettings { /** - * Safety filter level + * The safety filter level to use. */ safetyFilterLevel?: ImagenSafetyFilterLevel; /** - * Generate people. + * The person filter level to use. */ personFilterLevel?: ImagenPersonFilterLevel; } -export enum ImagenAspectRatio { - SQUARE = '1:1', - CLASSIC_PORTRAIT = '3:4', - CLASSIC_LANDSCAPE = '4:3', - WIDESCREEN = '16:9', - PORTRAIT = '9:16' -} - /** - * The parameters to be sent in the request body of the HTTP call - * to the Vertex AI backend. - * - * We need a seperate internal-only interface for this because the REST - * API expects different parameter names than what we show to our users. - * - * This interface should be populated from the {@link ImagenGenerationConfig} that - * the user defines. - * - * Sample request body JSON: - * { - * "instances": [ - * { - * "prompt": "Portrait of a golden retriever on a beach." - * } - * ], - * "parameters": { - * "mimeType": "image/png", - * "safetyFilterLevel": "block_low_and_above", - * "personGeneration": "allow_all", - * "sampleCount": 2, - * "includeRaiReason": true, - * "aspectRatio": "9:16" - * } - * } + * Aspect ratios for Imagen images. * - * @internal + * @public */ -export interface PredictRequestBody { - instances: [ - { - prompt: string; - } - ]; - parameters: { - sampleCount: number; // maps to numberOfImages - aspectRatio: string; - mimeType: string; - compressionQuality?: number; - negativePrompt?: string; - storageUri?: string; - addWatermark?: boolean; - safetyFilterLevel?: string; - personGeneration?: string; - includeRaiReason: boolean; - }; +export enum ImagenAspectRatio { + SQUARE = '1:1', + LANDSCAPE_3x4 = '3:4', + PORTRAIT_4x3 = '4:3', + LANDSCAPE_16x9 = '16:9', + PORTRAIT_9x16 = '9:16' } diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index 620f49be4b9..a05e3fda0f7 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -16,58 +16,73 @@ */ /** - * Base class for types of images that the Imagen Model can return. - * - * @internal - */ -export interface ImagenImage { - mimeType: string; -} - -/** - * Image generated by Imagen to inline bytes. + * An image generated by Imagen, represented as inline bytes. * * @public */ -export interface ImagenInlineImage extends ImagenImage { +export interface ImagenInlineImage { + /** + * The MIME type of the image. + */ + mimeType: string; + /** + * The image data encoded as a base64 string. + */ bytesBase64Encoded: string; } /** - * Image generated by Imagen, stored in Google Cloud Storage (GCS). + * An image generated by Imagen, stored in Google Cloud Storage (GCS). * * @public */ -export interface ImagenGCSImage extends ImagenImage { +export interface ImagenGCSImage { /** - * The Google Cloud Storage (GCS) URI at which the generated image is stored. + * The MIME type of the image. + */ + mimeType: string; + /** + * The Google Cloud Storage (GCS) URI where the image is stored. */ gcsURI: string; } /** - * Imagen image response. + * The response from a request to generate images to inline bytes. * * @public */ export interface ImagenInlineImageResponse { /** - * The images generated by Imagen. If all images were filtered, this will be empty. + * The images generated by Imagen. + * If all images were filtered out due to safety reasons, this array will be empty. */ images: ImagenInlineImage[]; /** - * The reason the missing images were filtered. - * For the mappings of error codes to reasons, see {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + * The reason why any images were filtered. This field is only present if one + * or more images were filtered. + * For the mappings of error codes to reasons, see + * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. */ filteredReason?: string; } +/** + * The response from a request to generate images to Google Cloud Storage (GCS). + * + * @public + */ export interface ImagenGCSImageResponse { + /** + * The images generated by Imagen. + * If all images were filtered due to safety reasons, this array will be empty. + */ images: ImagenGCSImage[]; - filteredReason?: string; -} - -export interface ImagenImageReponse { - images: ImagenImage[]; + /** + * The reason why any images were filtered. This field is only present if one + * or more images were filtered. + * For the mappings of error codes to reasons, see + * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + */ filteredReason?: string; } diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index e2a442821da..83cd4366f12 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -46,11 +46,6 @@ export interface GenerateContentStreamResult { response: Promise; } -export interface SafetyAttributes { - categories: string[]; - scores: number[]; -} - /** * Response object wrapped with helper methods. * From 497727f4eaffb7baaa7f2b108d6d420d186412be Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:47:36 -0600 Subject: [PATCH 03/31] Generate devsite docs --- common/api-review/vertexai.api.md | 9 +- docs-devsite/_toc.yaml | 6 +- docs-devsite/vertexai.generativemodel.md | 12 +-- docs-devsite/vertexai.imagengcsimage.md | 20 +++- .../vertexai.imagengcsimageresponse.md | 10 +- .../vertexai.imagengenerationconfig.md | 14 ++- docs-devsite/vertexai.imagenimageformat.md | 74 ++++++++++++- docs-devsite/vertexai.imagenimagereponse.md | 40 ------- docs-devsite/vertexai.imageninlineimage.md | 20 +++- .../vertexai.imageninlineimageresponse.md | 10 +- docs-devsite/vertexai.imagenmodel.md | 63 ++++++++--- docs-devsite/vertexai.imagenmodelconfig.md | 14 ++- docs-devsite/vertexai.imagenmodelparams.md | 12 +-- docs-devsite/vertexai.imagensafetysettings.md | 9 +- docs-devsite/vertexai.md | 101 ++++++------------ docs-devsite/vertexai.safetyattributes.md | 40 ------- docs-devsite/vertexai.vertexaimodel.md | 66 ++++++++++++ .../vertexai/src/models/vertexai-model.ts | 3 + .../src/requests/imagen-image-format.ts | 6 ++ .../vertexai/src/types/imagen/requests.ts | 15 +++ 20 files changed, 316 insertions(+), 228 deletions(-) delete mode 100644 docs-devsite/vertexai.imagenimagereponse.md delete mode 100644 docs-devsite/vertexai.safetyattributes.md create mode 100644 docs-devsite/vertexai.vertexaimodel.md diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 5b5c00b122e..8affc2a3922 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -432,15 +432,10 @@ export enum HarmSeverity { // @public export enum ImagenAspectRatio { - // (undocumented) LANDSCAPE_16x9 = "16:9", - // (undocumented) LANDSCAPE_3x4 = "3:4", - // (undocumented) PORTRAIT_4x3 = "4:3", - // (undocumented) PORTRAIT_9x16 = "9:16", - // (undocumented) SQUARE = "1:1" } @@ -465,10 +460,8 @@ export interface ImagenGenerationConfig { // @public export class ImagenImageFormat { - // (undocumented) compressionQuality?: number; static jpeg(compressionQuality: number): ImagenImageFormat; - // (undocumented) mimeType: string; static png(): ImagenImageFormat; } @@ -821,7 +814,7 @@ export const enum VertexAIErrorCode { export class VertexAIModel { // @internal protected constructor(vertexAI: VertexAI, modelName: string); - // (undocumented) + // @internal (undocumented) protected _apiSettings: ApiSettings; readonly model: string; static normalizeModelName(modelName: string): string; diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 6156c51c3f9..8bc943df066 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -536,8 +536,6 @@ toc: path: /docs/reference/js/vertexai.imagengenerationconfig.md - title: ImagenImageFormat path: /docs/reference/js/vertexai.imagenimageformat.md - - title: ImagenImageReponse - path: /docs/reference/js/vertexai.imagenimagereponse.md - title: ImagenInlineImage path: /docs/reference/js/vertexai.imageninlineimage.md - title: ImagenInlineImageResponse @@ -568,8 +566,6 @@ toc: path: /docs/reference/js/vertexai.requestoptions.md - title: RetrievedContextAttribution path: /docs/reference/js/vertexai.retrievedcontextattribution.md - - title: SafetyAttributes - path: /docs/reference/js/vertexai.safetyattributes.md - title: SafetyRating path: /docs/reference/js/vertexai.safetyrating.md - title: SafetySetting @@ -600,6 +596,8 @@ toc: path: /docs/reference/js/vertexai.vertexai.md - title: VertexAIError path: /docs/reference/js/vertexai.vertexaierror.md + - title: VertexAIModel + path: /docs/reference/js/vertexai.vertexaimodel.md - title: VertexAIOptions path: /docs/reference/js/vertexai.vertexaioptions.md - title: VideoMetadata diff --git a/docs-devsite/vertexai.generativemodel.md b/docs-devsite/vertexai.generativemodel.md index 7105f9c100b..b734e241e78 100644 --- a/docs-devsite/vertexai.generativemodel.md +++ b/docs-devsite/vertexai.generativemodel.md @@ -15,8 +15,9 @@ Class for generative model APIs. Signature: ```typescript -export declare class GenerativeModel +export declare class GenerativeModel extends VertexAIModel ``` +Extends: [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) ## Constructors @@ -29,7 +30,6 @@ export declare class GenerativeModel | Property | Modifiers | Type | Description | | --- | --- | --- | --- | | [generationConfig](./vertexai.generativemodel.md#generativemodelgenerationconfig) | | [GenerationConfig](./vertexai.generationconfig.md#generationconfig_interface) | | -| [model](./vertexai.generativemodel.md#generativemodelmodel) | | string | | | [requestOptions](./vertexai.generativemodel.md#generativemodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | | [safetySettings](./vertexai.generativemodel.md#generativemodelsafetysettings) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface)\[\] | | | [systemInstruction](./vertexai.generativemodel.md#generativemodelsysteminstruction) | | [Content](./vertexai.content.md#content_interface) | | @@ -71,14 +71,6 @@ constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: Reque generationConfig: GenerationConfig; ``` -## GenerativeModel.model - -Signature: - -```typescript -model: string; -``` - ## GenerativeModel.requestOptions Signature: diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md index eb2e9b76556..af9887ba9dd 100644 --- a/docs-devsite/vertexai.imagengcsimage.md +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -10,27 +10,37 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImage interface -Image generated by Imagen, stored in Google Cloud Storage (GCS). +An image generated by Imagen, stored in Google Cloud Storage (GCS). Signature: ```typescript -export interface ImagenGCSImage extends ImagenImage +export interface ImagenGCSImage ``` -Extends: ImagenImage ## Properties | Property | Type | Description | | --- | --- | --- | -| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI at which the generated image is stored. | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI where the image is stored. | +| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image. | ## ImagenGCSImage.gcsURI -The Google Cloud Storage (GCS) URI at which the generated image is stored. +The Google Cloud Storage (GCS) URI where the image is stored. Signature: ```typescript gcsURI: string; ``` + +## ImagenGCSImage.mimeType + +The MIME type of the image. + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagengcsimageresponse.md b/docs-devsite/vertexai.imagengcsimageresponse.md index 0c20cb1e752..cb1fa2cfe70 100644 --- a/docs-devsite/vertexai.imagengcsimageresponse.md +++ b/docs-devsite/vertexai.imagengcsimageresponse.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImageResponse interface +The response from a request to generate images to Google Cloud Storage (GCS). + Signature: ```typescript @@ -20,11 +22,13 @@ export interface ImagenGCSImageResponse | Property | Type | Description | | --- | --- | --- | -| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | | -| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | | +| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. | ## ImagenGCSImageResponse.filteredReason +The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). + Signature: ```typescript @@ -33,6 +37,8 @@ filteredReason?: string; ## ImagenGCSImageResponse.images +The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md index bf28bc291ae..9de48e45e81 100644 --- a/docs-devsite/vertexai.imagengenerationconfig.md +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationConfig interface +Request-level configuration options for generating images with Imagen. + Signature: ```typescript @@ -20,12 +22,14 @@ export interface ImagenGenerationConfig | Property | Type | Description | | --- | --- | --- | -| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | -| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | -| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | The aspect ratio of the generated images. Defaults to 1:1. | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | A text prompt describing what should not be included in the image. | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | The number of images to generate. Must be between 1 and 4. Defaults to 1. | ## ImagenGenerationConfig.aspectRatio +The aspect ratio of the generated images. Defaults to `1:1`. + Signature: ```typescript @@ -34,6 +38,8 @@ aspectRatio?: ImagenAspectRatio; ## ImagenGenerationConfig.negativePrompt +A text prompt describing what should not be included in the image. + Signature: ```typescript @@ -42,6 +48,8 @@ negativePrompt?: string; ## ImagenGenerationConfig.numberOfImages +The number of images to generate. Must be between 1 and 4. Defaults to 1. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md index 7ab7b9eb277..abdd561caa4 100644 --- a/docs-devsite/vertexai.imagenimageformat.md +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -9,22 +9,35 @@ overwritten. Changes should be made in the source code at https://github.com/firebase/firebase-js-sdk {% endcomment %} -# ImagenImageFormat interface +# ImagenImageFormat class +Defines the image format for images output by Imagen. + +Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + Signature: ```typescript -export interface ImagenImageFormat +export declare class ImagenImageFormat ``` ## Properties -| Property | Type | Description | +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | The level of compression (a number between 0 and 100). | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | The MIME type. | + +## Methods + +| Method | Modifiers | Description | | --- | --- | --- | -| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | number | | -| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | string | | +| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | Creates an ImagenImageFormat for a JPEG image. | +| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | Creates an ImageImageFormat for a PNG image. | ## ImagenImageFormat.compressionQuality +The level of compression (a number between 0 and 100). + Signature: ```typescript @@ -33,8 +46,59 @@ compressionQuality?: number; ## ImagenImageFormat.mimeType +The MIME type. + Signature: ```typescript mimeType: string; ``` + +## ImagenImageFormat.jpeg() + +Creates an ImagenImageFormat for a JPEG image. + +Signature: + +```typescript +static jpeg(compressionQuality: number): ImagenImageFormat; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| compressionQuality | number | The level of compression (a number between 0 and 100). | + +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) + +ImagenImageFormat + +## ImagenImageFormat.png() + +Creates an ImageImageFormat for a PNG image. + +Signature: + +```typescript +static png(): ImagenImageFormat; +``` +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) + +ImageImageFormat + +### Example + + +```javascript +const imagenModelParams = { + // ... other ImagenModelParams + imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. +} + +``` + diff --git a/docs-devsite/vertexai.imagenimagereponse.md b/docs-devsite/vertexai.imagenimagereponse.md deleted file mode 100644 index 14e34651a41..00000000000 --- a/docs-devsite/vertexai.imagenimagereponse.md +++ /dev/null @@ -1,40 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# ImagenImageReponse interface -Signature: - -```typescript -export interface ImagenImageReponse -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [filteredReason](./vertexai.imagenimagereponse.md#imagenimagereponsefilteredreason) | string | | -| [images](./vertexai.imagenimagereponse.md#imagenimagereponseimages) | ImagenImage\[\] | | - -## ImagenImageReponse.filteredReason - -Signature: - -```typescript -filteredReason?: string; -``` - -## ImagenImageReponse.images - -Signature: - -```typescript -images: ImagenImage[]; -``` diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md index fdfac65c15a..6d3a3972445 100644 --- a/docs-devsite/vertexai.imageninlineimage.md +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -10,25 +10,37 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenInlineImage interface -Image generated by Imagen to inline bytes. +An image generated by Imagen, represented as inline bytes. Signature: ```typescript -export interface ImagenInlineImage extends ImagenImage +export interface ImagenInlineImage ``` -Extends: ImagenImage ## Properties | Property | Type | Description | | --- | --- | --- | -| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | The image data encoded as a base64 string. | +| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | The MIME type of the image. | ## ImagenInlineImage.bytesBase64Encoded +The image data encoded as a base64 string. + Signature: ```typescript bytesBase64Encoded: string; ``` + +## ImagenInlineImage.mimeType + +The MIME type of the image. + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imageninlineimageresponse.md b/docs-devsite/vertexai.imageninlineimageresponse.md index 06cf3a2857f..c895dea6ea4 100644 --- a/docs-devsite/vertexai.imageninlineimageresponse.md +++ b/docs-devsite/vertexai.imageninlineimageresponse.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenInlineImageResponse interface -Imagen image response. +The response from a request to generate images to inline bytes. Signature: @@ -22,12 +22,12 @@ export interface ImagenInlineImageResponse | Property | Type | Description | | --- | --- | --- | -| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | -| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered, this will be empty. | +| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered out due to safety reasons, this array will be empty. | ## ImagenInlineImageResponse.filteredReason -The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). +The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). Signature: @@ -37,7 +37,7 @@ filteredReason?: string; ## ImagenInlineImageResponse.images -The images generated by Imagen. If all images were filtered, this will be empty. +The images generated by Imagen. If all images were filtered out due to safety reasons, this array will be empty. Signature: diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md index f1bff9689fe..190b9e7307a 100644 --- a/docs-devsite/vertexai.imagenmodel.md +++ b/docs-devsite/vertexai.imagenmodel.md @@ -12,34 +12,38 @@ https://github.com/firebase/firebase-js-sdk # ImagenModel class Class for Imagen model APIs. +This class provides methods for generating images using the Imagen model. You can generate images inline as base64-encoded strings, or directly to Google Cloud Storage (GCS). + Signature: ```typescript -export declare class ImagenModel +export declare class ImagenModel extends VertexAIModel ``` +Extends: [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) ## Constructors | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [model](./vertexai.imagenmodel.md#imagenmodelmodel) | | string | | +| [modelConfig](./vertexai.imagenmodel.md#imagenmodelmodelconfig) | | [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configurations to use when using Imagen. | +| [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | ## Methods | Method | Modifiers | Description | | --- | --- | --- | | [generateImages(prompt, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | -| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images using the Imagen model and returns them as base64-encoded strings. | +| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images to Google Cloud Storage (GCS) using the Imagen model. | ## ImagenModel.(constructor) -Constructs a new instance of the `ImagenModel` class +Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. Signature: @@ -51,16 +55,30 @@ constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: | Parameter | Type | Description | | --- | --- | --- | -| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | -| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | -| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making Imagen requests. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | Additional options to use when making requests. | + +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. + +## ImagenModel.modelConfig -## ImagenModel.model +Model-level configurations to use when using Imagen. Signature: ```typescript -model: string; +readonly modelConfig: ImagenModelConfig; +``` + +## ImagenModel.requestOptions + +Signature: + +```typescript +readonly requestOptions?: RequestOptions | undefined; ``` ## ImagenModel.generateImages() @@ -90,13 +108,13 @@ A promise that resolves to an [ImagenInlineImageResponse](./vertexai.imageninlin #### Exceptions -If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). +If the request to generate images fails. This happens if the prompt is blocked. ## ImagenModel.generateImagesGCS() -Generates images using the Imagen model and returns them as base64-encoded strings. +Generates images to Google Cloud Storage (GCS) using the Imagen model. -If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. +If one or more images are filtered due to safety reasons, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. Signature: @@ -109,16 +127,29 @@ generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenG | Parameter | Type | Description | | --- | --- | --- | | prompt | string | The text prompt used to generate the images. | -| gcsURI | string | The GCS URI where the images should be stored. | +| gcsURI | string | The GCS URI where the images should be stored. This should be a directory. For example, gs://my-bucket/my-directory/. | | imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | Returns: Promise<[ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface)> -A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the generated images. +A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the URLs of the generated images. #### Exceptions -If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). +If the request fails to generate images fails. This happens if the prompt is blocked. + +### Example + + +```javascript +const imagen = new ImagenModel(vertexAI, { + model: 'imagen-3.0-generate-001' +}); + +const response = await imagen.generateImages('A photo of a cat'); +console.log(response.images[0].bytesBase64Encoded); + +``` diff --git a/docs-devsite/vertexai.imagenmodelconfig.md b/docs-devsite/vertexai.imagenmodelconfig.md index 9923d0de504..e8948276eb6 100644 --- a/docs-devsite/vertexai.imagenmodelconfig.md +++ b/docs-devsite/vertexai.imagenmodelconfig.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModelConfig interface +Model-level configuration options for Imagen. + Signature: ```typescript @@ -20,12 +22,14 @@ export interface ImagenModelConfig | Property | Type | Description | | --- | --- | --- | -| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | | -| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | -| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | Whether to add a watermark to generated images. | +| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | The image format of the generated images. | +| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inapropriate content. | ## ImagenModelConfig.addWatermark +Whether to add a watermark to generated images. + Signature: ```typescript @@ -34,6 +38,8 @@ addWatermark?: boolean; ## ImagenModelConfig.imageFormat +The image format of the generated images. + Signature: ```typescript @@ -42,6 +48,8 @@ imageFormat?: ImagenImageFormat; ## ImagenModelConfig.safetySettings +Safety settings for filtering inapropriate content. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md index 2d1070a3585..256979e0498 100644 --- a/docs-devsite/vertexai.imagenmodelparams.md +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -10,13 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModelParams interface - Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). Signature: @@ -29,10 +23,12 @@ export interface ImagenModelParams extends ImagenModelConfig | Property | Type | Description | | --- | --- | --- | -| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | The Imagen model to use for generating images. For example: imagen-3.0-generate-001. | ## ImagenModelParams.model +The Imagen model to use for generating images. For example: `imagen-3.0-generate-001`. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md index 0d4a45a6d52..a544b9688bc 100644 --- a/docs-devsite/vertexai.imagensafetysettings.md +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenSafetySettings interface +Safety settings for Imagen. Signature: @@ -21,12 +22,12 @@ export interface ImagenSafetySettings | Property | Type | Description | | --- | --- | --- | -| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Generate people. | -| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter level | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | The person filter level to use. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | The safety filter level to use. | ## ImagenSafetySettings.personFilterLevel -Generate people. +The person filter level to use. Signature: @@ -36,7 +37,7 @@ personFilterLevel?: ImagenPersonFilterLevel; ## ImagenSafetySettings.safetyFilterLevel -Safety filter level +The safety filter level to use. Signature: diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index 827babbcafe..3814ccf42a4 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -21,10 +21,6 @@ The Vertex AI in Firebase Web SDK. | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | | [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. | -| function() | -| [png()](./vertexai.md#png) | Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | -| function(compressionQuality, ...) | -| [jpeg(compressionQuality)](./vertexai.md#jpeg_8e6711f) | Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | ## Classes @@ -34,13 +30,15 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | -| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs. | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | Defines the image format for images output by Imagen.Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs.This class provides methods for generating images using the Imagen model. You can generate images inline as base64-encoded strings, or directly to Google Cloud Storage (GCS). | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | | [Schema](./vertexai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with JSON.stringify() into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) | | [StringSchema](./vertexai.stringschema.md#stringschema_class) | Schema class for "string" types. Can be used with or without enum values. | | [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) | Error class for the Vertex AI in Firebase SDK. | +| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | Base class for Vertex AI in Firebase model APIs. | ## Enumerations @@ -54,9 +52,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | -| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | -| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | -| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | Aspect ratios for Imagen images. | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Person filter levels for Imagen. | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter levels for Imagen. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -92,23 +90,20 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | -| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | Image generated by Imagen, stored in Google Cloud Storage (GCS). | -| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | | -| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | -| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | -| [ImagenImageReponse](./vertexai.imagenimagereponse.md#imagenimagereponse_interface) | | -| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Image generated by Imagen to inline bytes. | -| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | Imagen image response. | -| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | | -| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Copyright 2024 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | -| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in Google Cloud Storage (GCS). | +| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | The response from a request to generate images to Google Cloud Storage (GCS). | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Request-level configuration options for generating images with Imagen. | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | An image generated by Imagen, represented as inline bytes. | +| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | The response from a request to generate images to inline bytes. | +| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configuration options for Imagen. | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for Imagen. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | -| [SafetyAttributes](./vertexai.safetyattributes.md#safetyattributes_interface) | | | [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | | [SchemaInterface](./vertexai.schemainterface.md#schemainterface_interface) | Interface for [Schema](./vertexai.schema.md#schema_class) class. | @@ -209,46 +204,6 @@ export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenMo [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) -## function() - -### png() {:#png} - -Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). - -Signature: - -```typescript -export declare function png(): ImagenImageFormat; -``` -Returns: - -[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) - - -## function(compressionQuality, ...) - -### jpeg(compressionQuality) {:#jpeg_8e6711f} - -Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). - -Signature: - -```typescript -export declare function jpeg(compressionQuality: number): ImagenImageFormat; -``` - -#### Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| compressionQuality | number | The level of compression. | - -Returns: - -[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) - -[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) - ## POSSIBLE\_ROLES Possible roles. @@ -447,6 +402,8 @@ export declare enum HarmSeverity ## ImagenAspectRatio +Aspect ratios for Imagen images. + Signature: ```typescript @@ -457,14 +414,15 @@ export declare enum ImagenAspectRatio | Member | Value | Description | | --- | --- | --- | -| CLASSIC\_LANDSCAPE | "4:3" | | -| CLASSIC\_PORTRAIT | "3:4" | | -| PORTRAIT | "9:16" | | -| SQUARE | "1:1" | | -| WIDESCREEN | "16:9" | | +| LANDSCAPE\_16x9 | "16:9" | Landscape 16:9 aspect ratio. | +| LANDSCAPE\_3x4 | "3:4" | Landscape 3:4 aspect ratio. | +| PORTRAIT\_4x3 | "4:3" | Portrait 4:3 aspect ratio. | +| PORTRAIT\_9x16 | "9:16" | Portrait 9:16 aspect ratio. | +| SQUARE | "1:1" | Square 1:1 aspect ratio. | ## ImagenPersonFilterLevel +Person filter levels for Imagen. Signature: @@ -476,12 +434,13 @@ export declare enum ImagenPersonFilterLevel | Member | Value | Description | | --- | --- | --- | -| ALLOW\_ADULT | "allow_adult" | | -| ALLOW\_ALL | "allow_all" | | -| BLOCK\_ALL | "dont_allow" | | +| ALLOW\_ADULT | "allow_adult" | Allow only adults in generated images. | +| ALLOW\_ALL | "allow_all" | Allow all person generation. | +| BLOCK\_ALL | "dont_allow" | Do not allow any person generation. | ## ImagenSafetyFilterLevel +Safety filter levels for Imagen. Signature: @@ -493,10 +452,10 @@ export declare enum ImagenSafetyFilterLevel | Member | Value | Description | | --- | --- | --- | -| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | -| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | -| BLOCK\_NONE | "block_none" | | -| BLOCK\_ONLY\_HIGH | "block_only_high" | | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | Block images with low or higher safety severity. | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | Block images with medium or higher safety severity. | +| BLOCK\_NONE | "block_none" | Do not block any images based on safety. | +| BLOCK\_ONLY\_HIGH | "block_only_high" | Block images with high safety severity. | ## SchemaType diff --git a/docs-devsite/vertexai.safetyattributes.md b/docs-devsite/vertexai.safetyattributes.md deleted file mode 100644 index 837850ab80f..00000000000 --- a/docs-devsite/vertexai.safetyattributes.md +++ /dev/null @@ -1,40 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# SafetyAttributes interface -Signature: - -```typescript -export interface SafetyAttributes -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [categories](./vertexai.safetyattributes.md#safetyattributescategories) | string\[\] | | -| [scores](./vertexai.safetyattributes.md#safetyattributesscores) | number\[\] | | - -## SafetyAttributes.categories - -Signature: - -```typescript -categories: string[]; -``` - -## SafetyAttributes.scores - -Signature: - -```typescript -scores: number[]; -``` diff --git a/docs-devsite/vertexai.vertexaimodel.md b/docs-devsite/vertexai.vertexaimodel.md new file mode 100644 index 00000000000..ad35e68dded --- /dev/null +++ b/docs-devsite/vertexai.vertexaimodel.md @@ -0,0 +1,66 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# VertexAIModel class +Base class for Vertex AI in Firebase model APIs. + +The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `VertexAIModel` class. + +Signature: + +```typescript +export declare class VertexAIModel +``` + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | The fully qualified model resource name to use for generating images (e.g. publishers/google/models/imagen-3.0-generate-001). | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [normalizeModelName(modelName)](./vertexai.vertexaimodel.md#vertexaimodelnormalizemodelname) | static | Normalizes the given model name to a fully qualified model resource name. | + +## VertexAIModel.model + +The fully qualified model resource name to use for generating images (e.g. `publishers/google/models/imagen-3.0-generate-001`). + +Signature: + +```typescript +readonly model: string; +``` + +## VertexAIModel.normalizeModelName() + +Normalizes the given model name to a fully qualified model resource name. + +Signature: + +```typescript +static normalizeModelName(modelName: string): string; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| modelName | string | The model name to normalize. | + +Returns: + +string + +The fully qualified model resource name. + diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index 0a05eb6fea6..e1ce26c6272 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -15,6 +15,9 @@ export class VertexAIModel { */ readonly model: string; + /** + * @internal + */ protected _apiSettings: ApiSettings; /** diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 6b7c84d8f2b..1b7b9eaf6ac 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -16,7 +16,13 @@ * @public */ export class ImagenImageFormat { + /** + * The MIME type. + */ mimeType: string; + /** + * The level of compression (a number between 0 and 100). + */ compressionQuality?: number; private constructor() { diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index a9a9c664b45..99b69ad09b8 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -136,9 +136,24 @@ export interface ImagenSafetySettings { * @public */ export enum ImagenAspectRatio { + /** + * Square 1:1 aspect ratio. + */ SQUARE = '1:1', + /** + * Landscape 3:4 aspect ratio. + */ LANDSCAPE_3x4 = '3:4', + /** + * Portrait 4:3 aspect ratio. + */ PORTRAIT_4x3 = '4:3', + /** + * Landscape 16:9 aspect ratio. + */ LANDSCAPE_16x9 = '16:9', + /** + * Portrait 9:16 aspect ratio. + */ PORTRAIT_9x16 = '9:16' } From 8471bbcb14ecb0422c05492a8c361d20ca3db77f Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:55:35 -0600 Subject: [PATCH 04/31] revert dataconnect change --- packages/data-connect/test/unit/transportoptions.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/data-connect/test/unit/transportoptions.test.ts b/packages/data-connect/test/unit/transportoptions.test.ts index a7136b5c408..91e090e6a54 100644 --- a/packages/data-connect/test/unit/transportoptions.test.ts +++ b/packages/data-connect/test/unit/transportoptions.test.ts @@ -16,8 +16,6 @@ */ import { expect } from 'chai'; - -import { queryRef } from '../../src'; import { TransportOptions, areTransportOptionsEqual, @@ -25,6 +23,7 @@ import { getDataConnect } from '../../src/api/DataConnect'; import { app } from '../util'; +import { queryRef } from '../../src'; describe('Transport Options', () => { it('should return false if transport options are not equal', () => { const transportOptions1: TransportOptions = { From 5c518b11dd21c12b45aa2ea9d6578c99493efa9f Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:56:16 -0600 Subject: [PATCH 05/31] formatting --- packages/vertexai/src/models/index.ts | 19 +++++++- .../src/models/vertexai-model.test.ts | 10 ++--- .../vertexai/src/models/vertexai-model.ts | 44 ++++++++++++------- .../src/requests/imagen-image-format.ts | 29 +++++++++--- .../vertexai/src/requests/request-helpers.ts | 2 +- .../vertexai/src/types/imagen/internal.ts | 4 +- 6 files changed, 78 insertions(+), 30 deletions(-) diff --git a/packages/vertexai/src/models/index.ts b/packages/vertexai/src/models/index.ts index a6ada0d894c..aec06be26fd 100644 --- a/packages/vertexai/src/models/index.ts +++ b/packages/vertexai/src/models/index.ts @@ -1,3 +1,20 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + export * from './vertexai-model'; export * from './generative-model'; -export * from './imagen-model'; \ No newline at end of file +export * from './imagen-model'; diff --git a/packages/vertexai/src/models/vertexai-model.test.ts b/packages/vertexai/src/models/vertexai-model.test.ts index 6b5cdd48e2a..d570230a64c 100644 --- a/packages/vertexai/src/models/vertexai-model.test.ts +++ b/packages/vertexai/src/models/vertexai-model.test.ts @@ -27,10 +27,7 @@ use(sinonChai); */ class TestModel extends VertexAIModel { /* eslint-disable @typescript-eslint/no-useless-constructor */ - constructor( - vertexAI: VertexAI, - modelName: string - ) { + constructor(vertexAI: VertexAI, modelName: string) { super(vertexAI, modelName); } } @@ -57,7 +54,10 @@ describe('VertexAIModel', () => { expect(testModel.model).to.equal('publishers/google/models/my-model'); }); it('handles full model name', () => { - const testModel = new TestModel(fakeVertexAI, 'publishers/google/models/my-model'); + const testModel = new TestModel( + fakeVertexAI, + 'publishers/google/models/my-model' + ); expect(testModel.model).to.equal('publishers/google/models/my-model'); }); it('handles prefixed tuned model name', () => { diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index e1ce26c6272..37099b032b8 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -1,7 +1,24 @@ -import { VertexAIError } from "../errors"; -import { VertexAI, VertexAIErrorCode } from "../public-types"; -import { VertexAIService } from "../service"; -import { ApiSettings } from "../types/internal"; +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAIError } from '../errors'; +import { VertexAI, VertexAIErrorCode } from '../public-types'; +import { VertexAIService } from '../service'; +import { ApiSettings } from '../types/internal'; /** * Base class for Vertex AI in Firebase model APIs. @@ -10,7 +27,7 @@ import { ApiSettings } from "../types/internal"; */ export class VertexAIModel { /** - * The fully qualified model resource name to use for generating images + * The fully qualified model resource name to use for generating images * (e.g. `publishers/google/models/imagen-3.0-generate-001`). */ readonly model: string; @@ -22,7 +39,7 @@ export class VertexAIModel { /** * Constructs a new instance of the {@link VertexAIModel} class. - * + * * This constructor should only be called from subclasses that provide * a model API. * @@ -30,17 +47,14 @@ export class VertexAIModel { * @param modelName - The name of the model being used. It can be in one of the following formats: * - `my-model` (short name, will resolve to `publishers/google/models/my-model`) * - `models/my-model` (will resolve to `publishers/google/models/my-model`) - * - `publishers/my-publisher/models/my-model` (fully qualified model name) - * + * - `publishers/my-publisher/models/my-model` (fully qualified model name) + * * @throws If the `apiKey` or `projectId` fields are missing in your * Firebase config. - * + * * @internal */ - protected constructor( - vertexAI: VertexAI, - modelName: string - ) { + protected constructor(vertexAI: VertexAI, modelName: string) { this.model = VertexAIModel.normalizeModelName(modelName); if (!vertexAI.app?.options?.apiKey) { @@ -73,7 +87,7 @@ export class VertexAIModel { /** * Normalizes the given model name to a fully qualified model resource name. - * + * * @param modelName - The model name to normalize. * @returns The fully qualified model resource name. */ @@ -94,4 +108,4 @@ export class VertexAIModel { return model; } -} \ No newline at end of file +} diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 1b7b9eaf6ac..8315947b732 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -1,8 +1,25 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** * Defines the image format for images output by Imagen. * - * Use this class to specify the desired format (JPEG or PNG) and compression quality - * for images generated by Imagen. This is typically included as part of + * Use this class to specify the desired format (JPEG or PNG) and compression quality + * for images generated by Imagen. This is typically included as part of * {@link ImagenModelParams}. * * @example @@ -26,7 +43,7 @@ export class ImagenImageFormat { compressionQuality?: number; private constructor() { - this.mimeType = "image/png"; + this.mimeType = 'image/png'; } /** @@ -38,7 +55,7 @@ export class ImagenImageFormat { * @public */ static jpeg(compressionQuality: number): ImagenImageFormat { - return { mimeType: "image/jpeg", compressionQuality }; + return { mimeType: 'image/jpeg', compressionQuality }; } /** @@ -49,6 +66,6 @@ export class ImagenImageFormat { * @public */ static png(): ImagenImageFormat { - return { mimeType: "image/png" }; + return { mimeType: 'image/png' }; } -} \ No newline at end of file +} diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 831bb6a26a5..49731f748b7 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -20,7 +20,7 @@ import { GenerateContentRequest, Part, VertexAIErrorCode, - ImagenAspectRatio, + ImagenAspectRatio } from '../types'; import { VertexAIError } from '../errors'; import { ImagenImageFormat } from './imagen-image-format'; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index ba7ee2c8f49..26a70201eea 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { ImagenGenerationConfig, ImagenModelConfig } from "./requests"; +import { ImagenGenerationConfig, ImagenModelConfig } from './requests'; /** * A response from the REST API is expected to look like this in the success case: @@ -130,4 +130,4 @@ export interface ImagenRequestConfig * (for GCS requests only). */ gcsURI?: string; -} \ No newline at end of file +} From b680a2a23e65b1c21e2b3638ae3c9157e9cd5465 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 9 Dec 2024 11:57:18 -0500 Subject: [PATCH 06/31] WIP --- common/api-review/vertexai.api.md | 184 +++++++++++++ docs-devsite/_toc.yaml | 22 ++ docs-devsite/vertexai.imagengcsimage.md | 36 +++ .../vertexai.imagengcsimageresponse.md | 40 +++ .../vertexai.imagengenerationconfig.md | 49 ++++ docs-devsite/vertexai.imagenimageformat.md | 40 +++ docs-devsite/vertexai.imagenimagereponse.md | 40 +++ docs-devsite/vertexai.imageninlineimage.md | 34 +++ .../vertexai.imageninlineimageresponse.md | 46 ++++ docs-devsite/vertexai.imagenmodel.md | 124 +++++++++ docs-devsite/vertexai.imagenmodelconfig.md | 49 ++++ docs-devsite/vertexai.imagenmodelparams.md | 40 +++ docs-devsite/vertexai.imagensafetysettings.md | 45 ++++ docs-devsite/vertexai.md | 135 ++++++++++ docs-devsite/vertexai.safetyattributes.md | 40 +++ .../test/unit/transportoptions.test.ts | 3 +- packages/vertexai/api-extractor.json | 18 +- packages/vertexai/src/api.test.ts | 54 +++- packages/vertexai/src/api.ts | 31 ++- .../vertexai/src/models/imagen-model.test.ts | 246 ++++++++++++++++++ packages/vertexai/src/models/imagen-model.ts | 199 ++++++++++++++ .../src/requests/request-helpers.test.ts | 79 +++++- .../vertexai/src/requests/request-helpers.ts | 53 +++- packages/vertexai/src/requests/request.ts | 3 +- .../src/requests/response-helpers.test.ts | 78 +++++- .../vertexai/src/requests/response-helpers.ts | 52 ++++ packages/vertexai/src/types/imagen/index.ts | 19 ++ .../vertexai/src/types/imagen/internal.ts | 54 ++++ .../vertexai/src/types/imagen/requests.ts | 142 ++++++++++ .../vertexai/src/types/imagen/responses.ts | 73 ++++++ packages/vertexai/src/types/index.ts | 1 + packages/vertexai/src/types/internal.ts | 2 + packages/vertexai/src/types/responses.ts | 5 + 33 files changed, 2012 insertions(+), 24 deletions(-) create mode 100644 docs-devsite/vertexai.imagengcsimage.md create mode 100644 docs-devsite/vertexai.imagengcsimageresponse.md create mode 100644 docs-devsite/vertexai.imagengenerationconfig.md create mode 100644 docs-devsite/vertexai.imagenimageformat.md create mode 100644 docs-devsite/vertexai.imagenimagereponse.md create mode 100644 docs-devsite/vertexai.imageninlineimage.md create mode 100644 docs-devsite/vertexai.imageninlineimageresponse.md create mode 100644 docs-devsite/vertexai.imagenmodel.md create mode 100644 docs-devsite/vertexai.imagenmodelconfig.md create mode 100644 docs-devsite/vertexai.imagenmodelparams.md create mode 100644 docs-devsite/vertexai.imagensafetysettings.md create mode 100644 docs-devsite/vertexai.safetyattributes.md create mode 100644 packages/vertexai/src/models/imagen-model.test.ts create mode 100644 packages/vertexai/src/models/imagen-model.ts create mode 100644 packages/vertexai/src/types/imagen/index.ts create mode 100644 packages/vertexai/src/types/imagen/internal.ts create mode 100644 packages/vertexai/src/types/imagen/requests.ts create mode 100644 packages/vertexai/src/types/imagen/responses.ts diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 041bc62451f..aa35a3f024f 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -348,6 +348,9 @@ export class GenerativeModel { // @public export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +// @public +export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; + // @public export function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI; @@ -429,6 +432,148 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } +// @public (undocumented) +export enum ImagenAspectRatio { + // (undocumented) + CLASSIC_LANDSCAPE = "4:3", + // (undocumented) + CLASSIC_PORTRAIT = "3:4", + // (undocumented) + PORTRAIT = "9:16", + // (undocumented) + SQUARE = "1:1", + // (undocumented) + WIDESCREEN = "16:9" +} + +// Warning: (ae-incompatible-release-tags) The symbol "ImagenGCSImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal +// +// @public +export interface ImagenGCSImage extends ImagenImage { + gcsURI: string; +} + +// @public (undocumented) +export interface ImagenGCSImageResponse { + // (undocumented) + filteredReason?: string; + // (undocumented) + images: ImagenGCSImage[]; +} + +// @public (undocumented) +export interface ImagenGenerationConfig { + // (undocumented) + aspectRatio?: ImagenAspectRatio; + // (undocumented) + negativePrompt?: string; + // (undocumented) + numberOfImages?: number; +} + +// Warning: (ae-internal-missing-underscore) The name "ImagenImage" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface ImagenImage { + // (undocumented) + mimeType: string; +} + +// @public (undocumented) +export interface ImagenImageFormat { + // (undocumented) + compressionQuality?: number; + // (undocumented) + mimeType: string; +} + +// @public (undocumented) +export interface ImagenImageReponse { + // (undocumented) + filteredReason?: string; + // Warning: (ae-incompatible-release-tags) The symbol "images" is marked as @public, but its signature references "ImagenImage" which is marked as @internal + // + // (undocumented) + images: ImagenImage[]; +} + +// Warning: (ae-incompatible-release-tags) The symbol "ImagenInlineImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal +// +// @public +export interface ImagenInlineImage extends ImagenImage { + // (undocumented) + bytesBase64Encoded: string; +} + +// @public +export interface ImagenInlineImageResponse { + filteredReason?: string; + images: ImagenInlineImage[]; +} + +// @public +export class ImagenModel { + constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); + generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + // (undocumented) + model: string; + } + +// @public (undocumented) +export interface ImagenModelConfig { + // (undocumented) + addWatermark?: boolean; + // (undocumented) + imageFormat?: ImagenImageFormat; + // (undocumented) + safetySettings?: ImagenSafetySettings; +} + +// @public +export interface ImagenModelParams extends ImagenModelConfig { + // (undocumented) + model: string; +} + +// @public (undocumented) +export enum ImagenPersonFilterLevel { + // (undocumented) + ALLOW_ADULT = "allow_adult", + // (undocumented) + ALLOW_ALL = "allow_all", + // (undocumented) + BLOCK_ALL = "dont_allow" +} + +// Warning: (ae-internal-missing-underscore) The name "ImagenRequestConfig" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface ImagenRequestConfig extends ImagenModelConfig, ImagenGenerationConfig { + // (undocumented) + gcsURI?: string; + // (undocumented) + prompt: string; +} + +// @public (undocumented) +export enum ImagenSafetyFilterLevel { + // (undocumented) + BLOCK_LOW_AND_ABOVE = "block_low_and_above", + // (undocumented) + BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", + // (undocumented) + BLOCK_NONE = "block_none", + // (undocumented) + BLOCK_ONLY_HIGH = "block_only_high" +} + +// @public (undocumented) +export interface ImagenSafetySettings { + personFilterLevel?: ImagenPersonFilterLevel; + safetyFilterLevel?: ImagenSafetyFilterLevel; +} + // @public export interface InlineDataPart { // (undocumented) @@ -447,6 +592,9 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export function jpeg(compressionQuality: number): ImagenImageFormat; + // @public export interface ModelParams extends BaseParams { // (undocumented) @@ -490,9 +638,37 @@ export interface ObjectSchemaInterface extends SchemaInterface { // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +// @public +export function png(): ImagenImageFormat; + // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; +// Warning: (ae-internal-missing-underscore) The name "PredictRequestBody" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface PredictRequestBody { + // (undocumented) + instances: [ + { + prompt: string; + } + ]; + // (undocumented) + parameters: { + sampleCount: number; + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; + includeRaiReason: boolean; + }; +} + // @public export interface PromptFeedback { // (undocumented) @@ -520,6 +696,14 @@ export interface RetrievedContextAttribution { // @public export type Role = (typeof POSSIBLE_ROLES)[number]; +// @public (undocumented) +export interface SafetyAttributes { + // (undocumented) + categories: string[]; + // (undocumented) + scores: number[]; +} + // @public export interface SafetyRating { // (undocumented) diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index ca06d4f9398..80444baaa1c 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -528,6 +528,26 @@ toc: path: /docs/reference/js/vertexai.groundingattribution.md - title: GroundingMetadata path: /docs/reference/js/vertexai.groundingmetadata.md + - title: ImagenGCSImage + path: /docs/reference/js/vertexai.imagengcsimage.md + - title: ImagenGCSImageResponse + path: /docs/reference/js/vertexai.imagengcsimageresponse.md + - title: ImagenGenerationConfig + path: /docs/reference/js/vertexai.imagengenerationconfig.md + - title: ImagenImageFormat + path: /docs/reference/js/vertexai.imagenimageformat.md + - title: ImagenInlineImage + path: /docs/reference/js/vertexai.imageninlineimage.md + - title: ImagenInlineImageResponse + path: /docs/reference/js/vertexai.imageninlineimageresponse.md + - title: ImagenModel + path: /docs/reference/js/vertexai.imagenmodel.md + - title: ImagenModelConfig + path: /docs/reference/js/vertexai.imagenmodelconfig.md + - title: ImagenModelParams + path: /docs/reference/js/vertexai.imagenmodelparams.md + - title: ImagenSafetySettings + path: /docs/reference/js/vertexai.imagensafetysettings.md - title: InlineDataPart path: /docs/reference/js/vertexai.inlinedatapart.md - title: IntegerSchema @@ -576,6 +596,8 @@ toc: path: /docs/reference/js/vertexai.vertexai.md - title: VertexAIError path: /docs/reference/js/vertexai.vertexaierror.md + - title: VertexAIModel + path: /docs/reference/js/vertexai.vertexaimodel.md - title: VertexAIOptions path: /docs/reference/js/vertexai.vertexaioptions.md - title: VideoMetadata diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md new file mode 100644 index 00000000000..eb2e9b76556 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -0,0 +1,36 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImage interface +Image generated by Imagen, stored in Google Cloud Storage (GCS). + +Signature: + +```typescript +export interface ImagenGCSImage extends ImagenImage +``` +Extends: ImagenImage + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI at which the generated image is stored. | + +## ImagenGCSImage.gcsURI + +The Google Cloud Storage (GCS) URI at which the generated image is stored. + +Signature: + +```typescript +gcsURI: string; +``` diff --git a/docs-devsite/vertexai.imagengcsimageresponse.md b/docs-devsite/vertexai.imagengcsimageresponse.md new file mode 100644 index 00000000000..0c20cb1e752 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimageresponse.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImageResponse interface +Signature: + +```typescript +export interface ImagenGCSImageResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | | +| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | | + +## ImagenGCSImageResponse.filteredReason + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenGCSImageResponse.images + +Signature: + +```typescript +images: ImagenGCSImage[]; +``` diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md new file mode 100644 index 00000000000..bf28bc291ae --- /dev/null +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGenerationConfig interface +Signature: + +```typescript +export interface ImagenGenerationConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | + +## ImagenGenerationConfig.aspectRatio + +Signature: + +```typescript +aspectRatio?: ImagenAspectRatio; +``` + +## ImagenGenerationConfig.negativePrompt + +Signature: + +```typescript +negativePrompt?: string; +``` + +## ImagenGenerationConfig.numberOfImages + +Signature: + +```typescript +numberOfImages?: number; +``` diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md new file mode 100644 index 00000000000..7ab7b9eb277 --- /dev/null +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageFormat interface +Signature: + +```typescript +export interface ImagenImageFormat +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | number | | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | string | | + +## ImagenImageFormat.compressionQuality + +Signature: + +```typescript +compressionQuality?: number; +``` + +## ImagenImageFormat.mimeType + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagenimagereponse.md b/docs-devsite/vertexai.imagenimagereponse.md new file mode 100644 index 00000000000..14e34651a41 --- /dev/null +++ b/docs-devsite/vertexai.imagenimagereponse.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageReponse interface +Signature: + +```typescript +export interface ImagenImageReponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagenimagereponse.md#imagenimagereponsefilteredreason) | string | | +| [images](./vertexai.imagenimagereponse.md#imagenimagereponseimages) | ImagenImage\[\] | | + +## ImagenImageReponse.filteredReason + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenImageReponse.images + +Signature: + +```typescript +images: ImagenImage[]; +``` diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md new file mode 100644 index 00000000000..fdfac65c15a --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -0,0 +1,34 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImage interface +Image generated by Imagen to inline bytes. + +Signature: + +```typescript +export interface ImagenInlineImage extends ImagenImage +``` +Extends: ImagenImage + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | + +## ImagenInlineImage.bytesBase64Encoded + +Signature: + +```typescript +bytesBase64Encoded: string; +``` diff --git a/docs-devsite/vertexai.imageninlineimageresponse.md b/docs-devsite/vertexai.imageninlineimageresponse.md new file mode 100644 index 00000000000..06cf3a2857f --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimageresponse.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImageResponse interface +Imagen image response. + +Signature: + +```typescript +export interface ImagenInlineImageResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered, this will be empty. | + +## ImagenInlineImageResponse.filteredReason + +The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenInlineImageResponse.images + +The images generated by Imagen. If all images were filtered, this will be empty. + +Signature: + +```typescript +images: ImagenInlineImage[]; +``` diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md new file mode 100644 index 00000000000..f1bff9689fe --- /dev/null +++ b/docs-devsite/vertexai.imagenmodel.md @@ -0,0 +1,124 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModel class +Class for Imagen model APIs. + +Signature: + +```typescript +export declare class ImagenModel +``` + +## Constructors + +| Constructor | Modifiers | Description | +| --- | --- | --- | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [model](./vertexai.imagenmodel.md#imagenmodelmodel) | | string | | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [generateImages(prompt, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | +| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images using the Imagen model and returns them as base64-encoded strings. | + +## ImagenModel.(constructor) + +Constructs a new instance of the `ImagenModel` class + +Signature: + +```typescript +constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | + +## ImagenModel.model + +Signature: + +```typescript +model: string; +``` + +## ImagenModel.generateImages() + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. + +Signature: + +```typescript +generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | The text prompt used to generate the images. | +| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +Returns: + +Promise<[ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface)> + +A promise that resolves to an [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) object containing the generated images. + +#### Exceptions + +If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). + +## ImagenModel.generateImagesGCS() + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. + +Signature: + +```typescript +generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | The text prompt used to generate the images. | +| gcsURI | string | The GCS URI where the images should be stored. | +| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +Returns: + +Promise<[ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface)> + +A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the generated images. + +#### Exceptions + +If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). + diff --git a/docs-devsite/vertexai.imagenmodelconfig.md b/docs-devsite/vertexai.imagenmodelconfig.md new file mode 100644 index 00000000000..9923d0de504 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelconfig.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelConfig interface +Signature: + +```typescript +export interface ImagenModelConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | | +| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | +| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | + +## ImagenModelConfig.addWatermark + +Signature: + +```typescript +addWatermark?: boolean; +``` + +## ImagenModelConfig.imageFormat + +Signature: + +```typescript +imageFormat?: ImagenImageFormat; +``` + +## ImagenModelConfig.safetySettings + +Signature: + +```typescript +safetySettings?: ImagenSafetySettings; +``` diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md new file mode 100644 index 00000000000..2d1070a3585 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelParams interface + Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +Signature: + +```typescript +export interface ImagenModelParams extends ImagenModelConfig +``` +Extends: [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | + +## ImagenModelParams.model + +Signature: + +```typescript +model: string; +``` diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md new file mode 100644 index 00000000000..0d4a45a6d52 --- /dev/null +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -0,0 +1,45 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenSafetySettings interface + +Signature: + +```typescript +export interface ImagenSafetySettings +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Generate people. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter level | + +## ImagenSafetySettings.personFilterLevel + +Generate people. + +Signature: + +```typescript +personFilterLevel?: ImagenPersonFilterLevel; +``` + +## ImagenSafetySettings.safetyFilterLevel + +Safety filter level + +Signature: + +```typescript +safetyFilterLevel?: ImagenSafetyFilterLevel; +``` diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index d9e26eabc5d..827babbcafe 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -20,6 +20,11 @@ The Vertex AI in Firebase Web SDK. | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. | +| function() | +| [png()](./vertexai.md#png) | Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| function(compressionQuality, ...) | +| [jpeg(compressionQuality)](./vertexai.md#jpeg_8e6711f) | Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | ## Classes @@ -29,6 +34,7 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs. | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | @@ -48,6 +54,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -83,12 +92,23 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | Image generated by Imagen, stored in Google Cloud Storage (GCS). | +| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | +| [ImagenImageReponse](./vertexai.imagenimagereponse.md#imagenimagereponse_interface) | | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Image generated by Imagen to inline bytes. | +| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | Imagen image response. | +| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Copyright 2024 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | +| [SafetyAttributes](./vertexai.safetyattributes.md#safetyattributes_interface) | | | [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | | [SchemaInterface](./vertexai.schemainterface.md#schemainterface_interface) | Interface for [Schema](./vertexai.schema.md#schema_class) class. | @@ -167,6 +187,68 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) +### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} + +Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. + +Signature: + +```typescript +export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | + +Returns: + +[ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) + +## function() + +### png() {:#png} + +Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare function png(): ImagenImageFormat; +``` +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + + +## function(compressionQuality, ...) + +### jpeg(compressionQuality) {:#jpeg_8e6711f} + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare function jpeg(compressionQuality: number): ImagenImageFormat; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| compressionQuality | number | The level of compression. | + +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + ## POSSIBLE\_ROLES Possible roles. @@ -363,6 +445,59 @@ export declare enum HarmSeverity | HARM\_SEVERITY\_MEDIUM | "HARM_SEVERITY_MEDIUM" | | | HARM\_SEVERITY\_NEGLIGIBLE | "HARM_SEVERITY_NEGLIGIBLE" | | +## ImagenAspectRatio + +Signature: + +```typescript +export declare enum ImagenAspectRatio +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| CLASSIC\_LANDSCAPE | "4:3" | | +| CLASSIC\_PORTRAIT | "3:4" | | +| PORTRAIT | "9:16" | | +| SQUARE | "1:1" | | +| WIDESCREEN | "16:9" | | + +## ImagenPersonFilterLevel + + +Signature: + +```typescript +export declare enum ImagenPersonFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ALLOW\_ADULT | "allow_adult" | | +| ALLOW\_ALL | "allow_all" | | +| BLOCK\_ALL | "dont_allow" | | + +## ImagenSafetyFilterLevel + + +Signature: + +```typescript +export declare enum ImagenSafetyFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | +| BLOCK\_NONE | "block_none" | | +| BLOCK\_ONLY\_HIGH | "block_only_high" | | + ## SchemaType Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) diff --git a/docs-devsite/vertexai.safetyattributes.md b/docs-devsite/vertexai.safetyattributes.md new file mode 100644 index 00000000000..837850ab80f --- /dev/null +++ b/docs-devsite/vertexai.safetyattributes.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# SafetyAttributes interface +Signature: + +```typescript +export interface SafetyAttributes +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [categories](./vertexai.safetyattributes.md#safetyattributescategories) | string\[\] | | +| [scores](./vertexai.safetyattributes.md#safetyattributesscores) | number\[\] | | + +## SafetyAttributes.categories + +Signature: + +```typescript +categories: string[]; +``` + +## SafetyAttributes.scores + +Signature: + +```typescript +scores: number[]; +``` diff --git a/packages/data-connect/test/unit/transportoptions.test.ts b/packages/data-connect/test/unit/transportoptions.test.ts index 91e090e6a54..a7136b5c408 100644 --- a/packages/data-connect/test/unit/transportoptions.test.ts +++ b/packages/data-connect/test/unit/transportoptions.test.ts @@ -16,6 +16,8 @@ */ import { expect } from 'chai'; + +import { queryRef } from '../../src'; import { TransportOptions, areTransportOptionsEqual, @@ -23,7 +25,6 @@ import { getDataConnect } from '../../src/api/DataConnect'; import { app } from '../util'; -import { queryRef } from '../../src'; describe('Transport Options', () => { it('should return false if transport options are not equal', () => { const transportOptions1: TransportOptions = { diff --git a/packages/vertexai/api-extractor.json b/packages/vertexai/api-extractor.json index 8a3c6cb251e..799c1fcf397 100644 --- a/packages/vertexai/api-extractor.json +++ b/packages/vertexai/api-extractor.json @@ -1,10 +1,10 @@ { - "extends": "../../config/api-extractor.json", - // Point it to your entry point d.ts file. - "mainEntryPointFilePath": "/dist/src/index.d.ts", - "dtsRollup": { - "enabled": true, - "untrimmedFilePath": "/dist/.d.ts", - "publicTrimmedFilePath": "/dist/-public.d.ts" - } -} \ No newline at end of file + "extends": "../../config/api-extractor.json", + // Point it to your entry point d.ts file. + "mainEntryPointFilePath": "/dist/src/index.d.ts", + "dtsRollup": { + "enabled": true, + "untrimmedFilePath": "/dist/.d.ts", + "publicTrimmedFilePath": "/dist/-public.d.ts" + } +} diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index b6c96923856..c9432d2a7ea 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ModelParams, VertexAIErrorCode } from './types'; +import { ImagenModelParams, ModelParams, VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; -import { getGenerativeModel } from './api'; +import { ImagenModel, getGenerativeModel, getImagenModel } from './api'; import { expect } from 'chai'; import { VertexAI } from './public-types'; import { GenerativeModel } from './models/generative-model'; @@ -84,4 +84,54 @@ describe('Top level API', () => { expect(genModel).to.be.an.instanceOf(GenerativeModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); }); + it('getImagenModel throws if no model is provided', () => { + try { + getImagenModel(fakeVertexAI, {} as ImagenModelParams); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_MODEL); + expect((e as VertexAIError).message).includes( + `VertexAI: Must provide a model name. Example: ` + + `getImagenModel({ model: 'my-model-name' }) (vertexAI/${VertexAIErrorCode.NO_MODEL})` + ); + } + }); + it('getImagenModel throws if no apiKey is provided', () => { + const fakeVertexNoApiKey = { + ...fakeVertexAI, + app: { options: { projectId: 'my-project' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoApiKey, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_API_KEY); + expect((e as VertexAIError).message).equals( + `VertexAI: The "apiKey" field is empty in the local ` + + `Firebase config. Firebase VertexAI requires this field to` + + ` contain a valid API key. (vertexAI/${VertexAIErrorCode.NO_API_KEY})` + ); + } + }); + it('getImagenModel throws if no projectId is provided', () => { + const fakeVertexNoProject = { + ...fakeVertexAI, + app: { options: { apiKey: 'my-key' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoProject, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes( + VertexAIErrorCode.NO_PROJECT_ID + ); + expect((e as VertexAIError).message).equals( + `VertexAI: The "projectId" field is empty in the local` + + ` Firebase config. Firebase VertexAI requires this field ` + + `to contain a valid project ID. (vertexAI/${VertexAIErrorCode.NO_PROJECT_ID})` + ); + } + }); + it('getGenerativeModel gets an ImagenModel', () => { + const genModel = getImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(genModel).to.be.an.instanceOf(ImagenModel); + expect(genModel.model).to.equal('publishers/google/models/my-model'); + }); }); diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 42f33ed2a85..07154356435 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -21,14 +21,22 @@ import { getModularInstance } from '@firebase/util'; import { DEFAULT_LOCATION, VERTEX_TYPE } from './constants'; import { VertexAIService } from './service'; import { VertexAI, VertexAIOptions } from './public-types'; -import { ModelParams, RequestOptions, VertexAIErrorCode } from './types'; +import { + ImagenModelParams, + ModelParams, + RequestOptions, + VertexAIErrorCode +} from './types'; import { VertexAIError } from './errors'; import { GenerativeModel } from './models/generative-model'; +import { ImagenModel, jpeg, png } from './models/imagen-model'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; -export { GenerativeModel }; +export { jpeg, png }; + +export { GenerativeModel, ImagenModel }; export { VertexAIError }; @@ -77,3 +85,22 @@ export function getGenerativeModel( } return new GenerativeModel(vertexAI, modelParams, requestOptions); } + +/** + * Returns a {@link ImagenModel} class with methods for using Imagen. + * + * @public + */ +export function getImagenModel( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + requestOptions?: RequestOptions +): ImagenModel { + if (!modelParams.model) { + throw new VertexAIError( + VertexAIErrorCode.NO_MODEL, + `Must provide a model name. Example: getImagenModel({ model: 'my-model-name' })` + ); + } + return new ImagenModel(vertexAI, modelParams, requestOptions); +} diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts new file mode 100644 index 00000000000..909bff3dea2 --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -0,0 +1,246 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { ImagenModel } from './imagen-model'; +import { + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel, + VertexAI, + VertexAIErrorCode +} from '../public-types'; +import * as request from '../requests/request'; +import sinonChai from 'sinon-chai'; +import { VertexAIError } from '../errors'; +import { getMockResponse } from '../../test-utils/mock-response'; +import { match, restore, stub } from 'sinon'; + +use(sinonChai); + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('ImagenModel', () => { + it('handles plain model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles models/ prefixed model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'models/my-model' + }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles full model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'publishers/google/models/my-model' + }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles prefixed tuned model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'tunedModels/my-model' + }); + expect(imagenModel.model).to.equal('tunedModels/my-model'); + }); + it('throws if not passed an api key', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + projectId: 'my-project' + } + }, + location: 'us-central1' + }; + try { + new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); + } + }); + it('throws if not passed a project ID', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key' + } + }, + location: 'us-central1' + }; + try { + new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + } catch (e) { + expect((e as VertexAIError).code).to.equal( + VertexAIErrorCode.NO_PROJECT_ID + ); + } + }); + it('generateImages makes a request to predict with default parameters', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('sampleCount') && + value.includes('includeRaiReason') && + value.includes('aspectRatio') && + value.includes('mimeType') + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with model-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model', + addWatermark: true, + imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + safetySettings: { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + } + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('safetyFilterLevel') && + value.includes('block_only_high') && + value.includes('personFilterLevel') && + value.includes('allow_adult') && + value.includes('mimeType') && + value.includes('image/jpeg') && + value.includes('addWatermark') && + value.includes('true') + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with request-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { + numberOfImages: 4, + aspectRatio: ImagenAspectRatio.WIDESCREEN, + negativePrompt: 'do not hallucinate' + }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('sampleCount') && + value.includes('4') && + value.includes('aspectRatio') && + value.includes('16:9') && + value.includes('negativePrompt') && + value.includes('do not hallucinate') + ); + }), + undefined + ); + + restore(); + }); + it('throws if prompt blocked', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-prompt-blocked.json' + ); + + stub(globalThis, 'fetch').resolves({ + ok: false, + status: 400, // TODO (dlarocque): Get this from the mock response + statusText: 'Bad Request', + json: mockResponse.json + } as Response); + + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + try { + await imagenModel.generateImages( + 'A photorealistic image of a toy boat at sea.', + { numberOfImages: 1 } + ); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); + expect((e as VertexAIError).message).to.include('400'); + expect((e as VertexAIError).message).to.include( + "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 42876398" + ); + } finally { + restore(); + } + }); +}); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts new file mode 100644 index 00000000000..17d1e1cfb2c --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.ts @@ -0,0 +1,199 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAIError } from '../errors'; +import { VertexAI } from '../public-types'; +import { Task, makeRequest } from '../requests/request'; +import { createPredictRequestBody } from '../requests/request-helpers'; +import { handlePredictResponse } from '../requests/response-helpers'; +import { VertexAIService } from '../service'; +import { + ImagenGCSImage, + ImagenGCSImageResponse, + ImagenImageFormat, + ImagenGenerationConfig, + ImagenInlineImage, + RequestOptions, + VertexAIErrorCode, + ImagenModelParams, + ImagenInlineImageResponse, + ImagenModelConfig +} from '../types'; +import { ApiSettings } from '../types/internal'; + +/** + * Class for Imagen model APIs. + * @public + */ +export class ImagenModel { + model: string; + private _apiSettings: ApiSettings; + private modelConfig: ImagenModelConfig; + + /** + * + * @param vertexAI + * @param modelParams + * @param requestOptions + */ + constructor( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + private requestOptions?: RequestOptions + ) { + const { model, ...modelConfig } = modelParams; + this.modelConfig = modelConfig; + if (model.includes('/')) { + if (model.startsWith('models/')) { + // Add "publishers/google" if the user is only passing in 'models/model-name'. + this.model = `publishers/google/${model}`; + } else { + // Any other custom format (e.g. tuned models) must be passed in correctly. + this.model = model; + } + } else { + // If path is not included, assume it's a non-tuned model. + this.model = `publishers/google/models/${model}`; + } + + if (!vertexAI.app?.options?.apiKey) { + throw new VertexAIError( + VertexAIErrorCode.NO_API_KEY, + `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` + ); + } else if (!vertexAI.app?.options?.projectId) { + throw new VertexAIError( + VertexAIErrorCode.NO_PROJECT_ID, + `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` + ); + } else { + this._apiSettings = { + apiKey: vertexAI.app.options.apiKey, + project: vertexAI.app.options.projectId, + location: vertexAI.location + }; + if ((vertexAI as VertexAIService).appCheck) { + this._apiSettings.getAppCheckToken = () => + (vertexAI as VertexAIService).appCheck!.getToken(); + } + + if ((vertexAI as VertexAIService).auth) { + this._apiSettings.getAuthToken = () => + (vertexAI as VertexAIService).auth!.getToken(); + } + } + } + + /** + * Generates images using the Imagen model and returns them as base64-encoded strings. + * + * @param prompt The text prompt used to generate the images. + * @param imagenRequestOptions Configuration options for the Imagen generation request. + * See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenInlineImageResponse} object containing the generated images. + * + * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * + * @remarks + * If one or more images are filtered, the returned object will have a defined `filteredReason` property. + * If all images are filtered, the `images` array will be empty, and no error will be thrown. + */ + async generateImages( + prompt: string, + imagenRequestOptions?: ImagenGenerationConfig + ): Promise { + const body = createPredictRequestBody({ + prompt, + ...imagenRequestOptions, + ...this.modelConfig + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } + + /** + * Generates images using the Imagen model and returns them as base64-encoded strings. + * + * @param prompt The text prompt used to generate the images. + * @param gcsURI The GCS URI where the images should be stored. + * @param imagenRequestOptions Configuration options for the Imagen generation request. + * See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenGCSImageResponse} object containing the generated images. + * + * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * + * @remarks + * If one or more images are filtered, the returned object will have a defined `filteredReason` property. + * If all images are filtered, the `images` array will be empty, and no error will be thrown. + */ + async generateImagesGCS( + prompt: string, + gcsURI: string, + imagenRequestOptions?: ImagenGenerationConfig + ): Promise { + const body = createPredictRequestBody({ + prompt, + gcsURI, + ...imagenRequestOptions, + ...this.modelConfig + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } +} + +/** + * Creates an {@link ImagenImageFormat} for a JPEG image, to be included in an {@link ImagenModelParams}. + * + * @param compressionQuality The level of compression. + * @returns {@link ImagenImageFormat} + * + * @public + */ +export function jpeg(compressionQuality: number): ImagenImageFormat { + return { + mimeType: 'image/jpeg', + compressionQuality + }; +} + +/** + * Creates an {@link ImageImageFormat} for a PNG image, to be included in a {@link ImagenModelParams}. + * + * @returns {@link ImageImageFormat} + * + * @public + */ +export function png(): ImagenImageFormat { + return { + mimeType: 'image/png' + }; +} diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 76b2f0ca1bf..9582f384a2c 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -17,8 +17,16 @@ import { expect, use } from 'chai'; import sinonChai from 'sinon-chai'; -import { Content } from '../types'; -import { formatGenerateContentInput } from './request-helpers'; +import { + Content, + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel +} from '../types'; +import { + createPredictRequestBody, + formatGenerateContentInput +} from './request-helpers'; use(sinonChai); @@ -199,4 +207,71 @@ describe('request formatting methods', () => { }); }); }); + describe('createPredictRequestBody', () => { + it('creates body with default request parameters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const body = createPredictRequestBody({ + prompt + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.sampleCount).to.equal(1); + expect(body.parameters.mimeType).to.equal('image/png'); + expect(body.parameters.includeRaiReason).to.be.true; + expect(body.parameters.aspectRatio).to.equal('1:1'); + + // Parameters without default values should be undefined + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.compressionQuality).to.be.undefined; + expect(body.parameters.negativePrompt).to.be.undefined; + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.addWatermark).to.be.undefined; + expect(body.parameters.safetyFilterLevel).to.be.undefined; + expect(body.parameters.personGeneration).to.be.undefined; + }); + }); + it('creates body with non-default request paramaters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const imageFormat = { mimeType: 'image/jpeg', compressionQuality: 75 }; + const safetySettings = { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_LOW_AND_ABOVE, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + }; + const addWatermark = true; + const numberOfImages = 4; + const negativePrompt = 'do not hallucinate'; + const aspectRatio = ImagenAspectRatio.WIDESCREEN; + const body = createPredictRequestBody({ + prompt, + numberOfImages, + imageFormat, + safetySettings, + addWatermark, + negativePrompt, + aspectRatio + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters).deep.equal({ + sampleCount: numberOfImages, + mimeType: imageFormat.mimeType, + compressionQuality: imageFormat.compressionQuality, + addWatermark, + negativePrompt, + safetyFilterLevel: safetySettings.safetyFilterLevel, + personFilterLevel: safetySettings.personFilterLevel, + aspectRatio, + includeRaiReason: true, + storageUri: undefined + }); + }); + it('creates body with GCS URI', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const gcsURI = 'gcs-uri'; + const body = createPredictRequestBody({ + prompt, + gcsURI + }); + + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.storageUri).to.equal(gcsURI); + }); }); diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 9e525b2a875..fa4bdf402d7 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -18,8 +18,11 @@ import { Content, GenerateContentRequest, + PredictRequestBody, Part, - VertexAIErrorCode + VertexAIErrorCode, + ImagenAspectRatio, + ImagenRequestConfig } from '../types'; import { VertexAIError } from '../errors'; @@ -49,11 +52,13 @@ export function formatNewContent( if (typeof request === 'string') { newParts = [{ text: request }]; } else { - for (const partOrString of request) { - if (typeof partOrString === 'string') { - newParts.push({ text: partOrString }); + for (const elem of request) { + // This throws an error if request is not iterable + if (typeof elem === 'string') { + newParts.push({ text: elem }); } else { - newParts.push(partOrString); + // We assume this is a Part, but it could be anything. + newParts.push(elem); // This could be } } } @@ -114,6 +119,7 @@ export function formatGenerateContentInput( formattedRequest = params as GenerateContentRequest; } else { // Array or string + // ... or something else const content = formatNewContent(params as string | Array); formattedRequest = { contents: [content] }; } @@ -124,3 +130,40 @@ export function formatGenerateContentInput( } return formattedRequest; } + +/** + * Convert the user-defined parameters in {@link ImagenRequestConfig} to the format + * that is expected from the REST API. + * + * @internal + */ +export function createPredictRequestBody({ + prompt, + gcsURI, + imageFormat = { mimeType: 'image/png' }, + addWatermark, + safetySettings, + numberOfImages = 1, + negativePrompt, + aspectRatio = ImagenAspectRatio.SQUARE +}: ImagenRequestConfig): PredictRequestBody { + // Properties that are undefined will be omitted from the JSON string. + const body: PredictRequestBody = { + instances: [ + { + prompt + } + ], + parameters: { + storageUri: gcsURI, + ...imageFormat, + addWatermark, + ...safetySettings, + sampleCount: numberOfImages, + includeRaiReason: true, + negativePrompt, + aspectRatio + } + }; + return body; +} diff --git a/packages/vertexai/src/requests/request.ts b/packages/vertexai/src/requests/request.ts index f81b40635e3..9b9465db776 100644 --- a/packages/vertexai/src/requests/request.ts +++ b/packages/vertexai/src/requests/request.ts @@ -30,7 +30,8 @@ import { logger } from '../logger'; export enum Task { GENERATE_CONTENT = 'generateContent', STREAM_GENERATE_CONTENT = 'streamGenerateContent', - COUNT_TOKENS = 'countTokens' + COUNT_TOKENS = 'countTokens', + PREDICT = 'predict' } export class RequestUrl { diff --git a/packages/vertexai/src/requests/response-helpers.test.ts b/packages/vertexai/src/requests/response-helpers.test.ts index 91a60d2cfce..4cab8cde047 100644 --- a/packages/vertexai/src/requests/response-helpers.test.ts +++ b/packages/vertexai/src/requests/response-helpers.test.ts @@ -15,7 +15,11 @@ * limitations under the License. */ -import { addHelpers, formatBlockErrorMessage } from './response-helpers'; +import { + addHelpers, + formatBlockErrorMessage, + handlePredictResponse +} from './response-helpers'; import { expect, use } from 'chai'; import { restore } from 'sinon'; import sinonChai from 'sinon-chai'; @@ -23,8 +27,11 @@ import { BlockReason, Content, FinishReason, - GenerateContentResponse + GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage } from '../types'; +import { getMockResponse } from '../../test-utils/mock-response'; use(sinonChai); @@ -246,4 +253,71 @@ describe('response-helpers methods', () => { ); }); }); + + describe('handlePredictResponse', () => { + it('returns base64 images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded.length).to.be.greaterThan(0); + }); + }); + }); + it('returns GCS images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-gcs.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach((image, i) => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.equal( + `gs://test-project-id-1234.firebasestorage.app/images/1234567890123/sample_${i}.jpg` + ); + }); + }); + it('has filtered reason and no images if all images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-all-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 39322892, 29310472" + ); + expect(res.images.length).to.equal(0); + }); + it('has filtered reason and no images if all base64 images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-base64-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded).to.have.length.greaterThan(0); + }); + }); + it('has filtered reason and no images if all GCS images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-gcs-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.have.length.greaterThan(0); + }); + }); }); diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 27347d10f0d..6b82463dfa1 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -21,10 +21,13 @@ import { FunctionCall, GenerateContentCandidate, GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage, VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; import { logger } from '../logger'; +import { ImagenResponseInternal } from '../types/internal'; /** * Creates an EnhancedGenerateContentResponse object that has helper functions and @@ -196,3 +199,52 @@ export function formatBlockErrorMessage( } return message; } + +/** + * Convert a generic successful fetch {@link Response} body to an Imagen response object + * that can be returned to the user. This converts the REST APIs response format to our + * representation of a response. + * + * @internal + */ + +export async function handlePredictResponse< + T extends ImagenInlineImage | ImagenGCSImage +>(response: Response): Promise<{ images: T[]; filteredReason?: string }> { + const responseJson: ImagenResponseInternal = await response.json(); + + const images: T[] = []; + let filteredReason: string | undefined = undefined; + + if (!responseJson.predictions || responseJson.predictions?.length === 0) { + throw new VertexAIError( + VertexAIErrorCode.ERROR, + "Predictions array is undefined or empty in response. Was 'includeRaiReason' enabled in the request?" + ); + } + + for (const prediction of responseJson.predictions) { + if (prediction.raiFilteredReason) { + filteredReason = prediction.raiFilteredReason; + } else if (prediction.mimeType && prediction.bytesBase64Encoded) { + images.push({ + mimeType: prediction.mimeType, + bytesBase64Encoded: prediction.bytesBase64Encoded + } as T); + } else if (prediction.mimeType && prediction.gcsUri) { + images.push({ + mimeType: prediction.mimeType, + gcsURI: prediction.gcsUri + } as T); + } else { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + `Predictions array in response has missing properties. Response: ${JSON.stringify( + responseJson + )}` + ); + } + } + + return { images, filteredReason }; +} diff --git a/packages/vertexai/src/types/imagen/index.ts b/packages/vertexai/src/types/imagen/index.ts new file mode 100644 index 00000000000..0037b185e50 --- /dev/null +++ b/packages/vertexai/src/types/imagen/index.ts @@ -0,0 +1,19 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './requests'; +export * from './responses'; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts new file mode 100644 index 00000000000..1171df81278 --- /dev/null +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -0,0 +1,54 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Internal Imagen types + +/** + * A response from the REST API is expected to look like this in the success case: + * { + * "predictions": [ + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "iVBORw0KG..." + * }, + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "i4BOtw0KG..." + * } + * ] + * } + * + * And like this in the failure case: + * { + * "predictions": [ + * { + * "raiFilteredReason": "..." + * } + * ] + * } + */ +export interface ImagenResponseInternal { + predictions?: Array<{ + // Defined if the prediction was not filtered + mimeType?: string; + bytesBase64Encoded?: string; + gcsUri?: string; + + // Defined if the prediction was filtered, and there is no image + raiFilteredReason?: string; + }>; +} diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts new file mode 100644 index 00000000000..d87723a31c7 --- /dev/null +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -0,0 +1,142 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface ImagenModelParams extends ImagenModelConfig { + model: string; +} + +export interface ImagenModelConfig { + imageFormat?: ImagenImageFormat; + addWatermark?: boolean; + safetySettings?: ImagenSafetySettings; +} + +export interface ImagenGenerationConfig { + numberOfImages?: number; // Default to 1. Possible values are [1...4] + negativePrompt?: string; // Default to null + aspectRatio?: ImagenAspectRatio; // Default to "1:1" +} + +/** + * Contains all possible REST API paramaters. + * This is the intersection of the model-level (`ImagenModelParams`), + * request-level (`ImagenGenerationConfig`) configurations, along with + * the other required parameters prompt and gcsURI (for GCS generation only). + * + * @internal + */ +export interface ImagenRequestConfig + extends ImagenModelConfig, + ImagenGenerationConfig { + prompt: string; + gcsURI?: string; +} + +export interface ImagenImageFormat { + mimeType: string; // image/png, or image/jpeg, default image/png + compressionQuality?: number; // 0-100, default 75. Only for image/jpeg +} + +/** + * @public + */ +export enum ImagenSafetyFilterLevel { + BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + BLOCK_ONLY_HIGH = 'block_only_high', + BLOCK_NONE = 'block_none' +} + +/** + * @public + */ +export enum ImagenPersonFilterLevel { + BLOCK_ALL = 'dont_allow', + ALLOW_ADULT = 'allow_adult', + ALLOW_ALL = 'allow_all' +} + +/** + * @public + */ +export interface ImagenSafetySettings { + /** + * Safety filter level + */ + safetyFilterLevel?: ImagenSafetyFilterLevel; + /** + * Generate people. + */ + personFilterLevel?: ImagenPersonFilterLevel; +} + +export enum ImagenAspectRatio { + SQUARE = '1:1', + CLASSIC_PORTRAIT = '3:4', + CLASSIC_LANDSCAPE = '4:3', + WIDESCREEN = '16:9', + PORTRAIT = '9:16' +} + +/** + * The parameters to be sent in the request body of the HTTP call + * to the Vertex AI backend. + * + * We need a seperate internal-only interface for this because the REST + * API expects different parameter names than what we show to our users. + * + * This interface should be populated from the {@link ImagenGenerationConfig} that + * the user defines. + * + * Sample request body JSON: + * { + * "instances": [ + * { + * "prompt": "Portrait of a golden retriever on a beach." + * } + * ], + * "parameters": { + * "mimeType": "image/png", + * "safetyFilterLevel": "block_low_and_above", + * "personGeneration": "allow_all", + * "sampleCount": 2, + * "includeRaiReason": true, + * "aspectRatio": "9:16" + * } + * } + * + * @internal + */ +export interface PredictRequestBody { + instances: [ + { + prompt: string; + } + ]; + parameters: { + sampleCount: number; // maps to numberOfImages + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; + includeRaiReason: boolean; + }; +} diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts new file mode 100644 index 00000000000..620f49be4b9 --- /dev/null +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Base class for types of images that the Imagen Model can return. + * + * @internal + */ +export interface ImagenImage { + mimeType: string; +} + +/** + * Image generated by Imagen to inline bytes. + * + * @public + */ +export interface ImagenInlineImage extends ImagenImage { + bytesBase64Encoded: string; +} + +/** + * Image generated by Imagen, stored in Google Cloud Storage (GCS). + * + * @public + */ +export interface ImagenGCSImage extends ImagenImage { + /** + * The Google Cloud Storage (GCS) URI at which the generated image is stored. + */ + gcsURI: string; +} + +/** + * Imagen image response. + * + * @public + */ +export interface ImagenInlineImageResponse { + /** + * The images generated by Imagen. If all images were filtered, this will be empty. + */ + images: ImagenInlineImage[]; + /** + * The reason the missing images were filtered. + * For the mappings of error codes to reasons, see {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + */ + filteredReason?: string; +} + +export interface ImagenGCSImageResponse { + images: ImagenGCSImage[]; + filteredReason?: string; +} + +export interface ImagenImageReponse { + images: ImagenImage[]; + filteredReason?: string; +} diff --git a/packages/vertexai/src/types/index.ts b/packages/vertexai/src/types/index.ts index 85133aa07c5..f575c5ba8e9 100644 --- a/packages/vertexai/src/types/index.ts +++ b/packages/vertexai/src/types/index.ts @@ -21,3 +21,4 @@ export * from './requests'; export * from './responses'; export * from './error'; export * from './schema'; +export * from './imagen'; diff --git a/packages/vertexai/src/types/internal.ts b/packages/vertexai/src/types/internal.ts index 8271175feff..87c28a02ab2 100644 --- a/packages/vertexai/src/types/internal.ts +++ b/packages/vertexai/src/types/internal.ts @@ -18,6 +18,8 @@ import { AppCheckTokenResult } from '@firebase/app-check-interop-types'; import { FirebaseAuthTokenData } from '@firebase/auth-interop-types'; +export * from './imagen/internal'; + export interface ApiSettings { apiKey: string; project: string; diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 83cd4366f12..e2a442821da 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -46,6 +46,11 @@ export interface GenerateContentStreamResult { response: Promise; } +export interface SafetyAttributes { + categories: string[]; + scores: number[]; +} + /** * Response object wrapped with helper methods. * From d00cccda169c17be72baf2e265c31669e5dfc462 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:38:21 -0600 Subject: [PATCH 07/31] Introduce VertexAIModel base class, add documentation, and respond to other comments --- common/api-review/vertexai.api.md | 145 ++++------------- packages/vertexai/src/api.test.ts | 2 +- packages/vertexai/src/api.ts | 10 +- .../src/models/generative-model.test.ts | 22 --- .../vertexai/src/models/generative-model.ts | 50 +----- .../vertexai/src/models/imagen-model.test.ts | 66 +------- packages/vertexai/src/models/imagen-model.ts | 151 +++++++---------- packages/vertexai/src/models/index.ts | 3 + .../src/models/vertexai-model.test.ts | 103 ++++++++++++ .../vertexai/src/models/vertexai-model.ts | 94 +++++++++++ .../src/requests/imagen-image-format.ts | 48 ++++++ .../src/requests/request-helpers.test.ts | 2 +- .../vertexai/src/requests/request-helpers.ts | 12 +- .../vertexai/src/requests/response-helpers.ts | 4 +- .../vertexai/src/types/imagen/internal.ts | 85 +++++++++- .../vertexai/src/types/imagen/requests.ts | 152 +++++++++--------- .../vertexai/src/types/imagen/responses.ts | 61 ++++--- packages/vertexai/src/types/responses.ts | 5 - 18 files changed, 552 insertions(+), 463 deletions(-) create mode 100644 packages/vertexai/src/models/index.ts create mode 100644 packages/vertexai/src/models/vertexai-model.test.ts create mode 100644 packages/vertexai/src/models/vertexai-model.ts create mode 100644 packages/vertexai/src/requests/imagen-image-format.ts diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index aa35a3f024f..5b5c00b122e 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -323,7 +323,7 @@ export interface GenerativeContentBlob { } // @public -export class GenerativeModel { +export class GenerativeModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions); countTokens(request: CountTokensRequest | string | Array): Promise; generateContent(request: GenerateContentRequest | string | Array): Promise; @@ -331,8 +331,6 @@ export class GenerativeModel { // (undocumented) generationConfig: GenerationConfig; // (undocumented) - model: string; - // (undocumented) requestOptions?: RequestOptions; // (undocumented) safetySettings: SafetySetting[]; @@ -432,77 +430,53 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } -// @public (undocumented) +// @public export enum ImagenAspectRatio { // (undocumented) - CLASSIC_LANDSCAPE = "4:3", + LANDSCAPE_16x9 = "16:9", // (undocumented) - CLASSIC_PORTRAIT = "3:4", + LANDSCAPE_3x4 = "3:4", // (undocumented) - PORTRAIT = "9:16", + PORTRAIT_4x3 = "4:3", // (undocumented) - SQUARE = "1:1", + PORTRAIT_9x16 = "9:16", // (undocumented) - WIDESCREEN = "16:9" + SQUARE = "1:1" } -// Warning: (ae-incompatible-release-tags) The symbol "ImagenGCSImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal -// // @public -export interface ImagenGCSImage extends ImagenImage { +export interface ImagenGCSImage { gcsURI: string; + mimeType: string; } -// @public (undocumented) +// @public export interface ImagenGCSImageResponse { - // (undocumented) filteredReason?: string; - // (undocumented) images: ImagenGCSImage[]; } -// @public (undocumented) +// @public export interface ImagenGenerationConfig { - // (undocumented) aspectRatio?: ImagenAspectRatio; - // (undocumented) negativePrompt?: string; - // (undocumented) numberOfImages?: number; } -// Warning: (ae-internal-missing-underscore) The name "ImagenImage" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal -export interface ImagenImage { - // (undocumented) - mimeType: string; -} - -// @public (undocumented) -export interface ImagenImageFormat { +// @public +export class ImagenImageFormat { // (undocumented) compressionQuality?: number; + static jpeg(compressionQuality: number): ImagenImageFormat; // (undocumented) mimeType: string; + static png(): ImagenImageFormat; } -// @public (undocumented) -export interface ImagenImageReponse { - // (undocumented) - filteredReason?: string; - // Warning: (ae-incompatible-release-tags) The symbol "images" is marked as @public, but its signature references "ImagenImage" which is marked as @internal - // - // (undocumented) - images: ImagenImage[]; -} - -// Warning: (ae-incompatible-release-tags) The symbol "ImagenInlineImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal -// // @public -export interface ImagenInlineImage extends ImagenImage { - // (undocumented) +export interface ImagenInlineImage { bytesBase64Encoded: string; + mimeType: string; } // @public @@ -512,63 +486,43 @@ export interface ImagenInlineImageResponse { } // @public -export class ImagenModel { +export class ImagenModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + readonly modelConfig: ImagenModelConfig; // (undocumented) - model: string; - } + readonly requestOptions?: RequestOptions | undefined; +} -// @public (undocumented) +// @public export interface ImagenModelConfig { - // (undocumented) addWatermark?: boolean; - // (undocumented) imageFormat?: ImagenImageFormat; - // (undocumented) safetySettings?: ImagenSafetySettings; } // @public export interface ImagenModelParams extends ImagenModelConfig { - // (undocumented) model: string; } -// @public (undocumented) +// @public export enum ImagenPersonFilterLevel { - // (undocumented) ALLOW_ADULT = "allow_adult", - // (undocumented) ALLOW_ALL = "allow_all", - // (undocumented) BLOCK_ALL = "dont_allow" } -// Warning: (ae-internal-missing-underscore) The name "ImagenRequestConfig" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal -export interface ImagenRequestConfig extends ImagenModelConfig, ImagenGenerationConfig { - // (undocumented) - gcsURI?: string; - // (undocumented) - prompt: string; -} - -// @public (undocumented) +// @public export enum ImagenSafetyFilterLevel { - // (undocumented) BLOCK_LOW_AND_ABOVE = "block_low_and_above", - // (undocumented) BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", - // (undocumented) BLOCK_NONE = "block_none", - // (undocumented) BLOCK_ONLY_HIGH = "block_only_high" } -// @public (undocumented) +// @public export interface ImagenSafetySettings { personFilterLevel?: ImagenPersonFilterLevel; safetyFilterLevel?: ImagenSafetyFilterLevel; @@ -592,9 +546,6 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } -// @public -export function jpeg(compressionQuality: number): ImagenImageFormat; - // @public export interface ModelParams extends BaseParams { // (undocumented) @@ -638,37 +589,9 @@ export interface ObjectSchemaInterface extends SchemaInterface { // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; -// @public -export function png(): ImagenImageFormat; - // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; -// Warning: (ae-internal-missing-underscore) The name "PredictRequestBody" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal -export interface PredictRequestBody { - // (undocumented) - instances: [ - { - prompt: string; - } - ]; - // (undocumented) - parameters: { - sampleCount: number; - aspectRatio: string; - mimeType: string; - compressionQuality?: number; - negativePrompt?: string; - storageUri?: string; - addWatermark?: boolean; - safetyFilterLevel?: string; - personGeneration?: string; - includeRaiReason: boolean; - }; -} - // @public export interface PromptFeedback { // (undocumented) @@ -696,14 +619,6 @@ export interface RetrievedContextAttribution { // @public export type Role = (typeof POSSIBLE_ROLES)[number]; -// @public (undocumented) -export interface SafetyAttributes { - // (undocumented) - categories: string[]; - // (undocumented) - scores: number[]; -} - // @public export interface SafetyRating { // (undocumented) @@ -902,6 +817,16 @@ export const enum VertexAIErrorCode { RESPONSE_ERROR = "response-error" } +// @public +export class VertexAIModel { + // @internal + protected constructor(vertexAI: VertexAI, modelName: string); + // (undocumented) + protected _apiSettings: ApiSettings; + readonly model: string; + static normalizeModelName(modelName: string): string; +} + // @public export interface VertexAIOptions { // (undocumented) diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index c9432d2a7ea..c1b2635ce70 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -129,7 +129,7 @@ describe('Top level API', () => { ); } }); - it('getGenerativeModel gets an ImagenModel', () => { + it('getImagenModel gets an ImagenModel', () => { const genModel = getImagenModel(fakeVertexAI, { model: 'my-model' }); expect(genModel).to.be.an.instanceOf(ImagenModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 07154356435..6ad61798c4f 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -28,16 +28,12 @@ import { VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; -import { GenerativeModel } from './models/generative-model'; -import { ImagenModel, jpeg, png } from './models/imagen-model'; +import { VertexAIModel, GenerativeModel, ImagenModel } from './models'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; - -export { jpeg, png }; - -export { GenerativeModel, ImagenModel }; - +export { ImagenImageFormat } from './requests/imagen-image-format'; +export { VertexAIModel, GenerativeModel, ImagenModel }; export { VertexAIError }; declare module '@firebase/component' { diff --git a/packages/vertexai/src/models/generative-model.test.ts b/packages/vertexai/src/models/generative-model.test.ts index e03f39e8a83..c2dbdfac75c 100644 --- a/packages/vertexai/src/models/generative-model.test.ts +++ b/packages/vertexai/src/models/generative-model.test.ts @@ -37,28 +37,6 @@ const fakeVertexAI: VertexAI = { }; describe('GenerativeModel', () => { - it('handles plain model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles models/ prefixed model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'models/my-model' - }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles full model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'publishers/google/models/my-model' - }); - expect(genModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles prefixed tuned model name', () => { - const genModel = new GenerativeModel(fakeVertexAI, { - model: 'tunedModels/my-model' - }); - expect(genModel.model).to.equal('tunedModels/my-model'); - }); it('passes params through to generateContent', async () => { const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model', diff --git a/packages/vertexai/src/models/generative-model.ts b/packages/vertexai/src/models/generative-model.ts index e719529967c..b4cf464f025 100644 --- a/packages/vertexai/src/models/generative-model.ts +++ b/packages/vertexai/src/models/generative-model.ts @@ -33,10 +33,8 @@ import { SafetySetting, StartChatParams, Tool, - ToolConfig, - VertexAIErrorCode + ToolConfig } from '../types'; -import { VertexAIError } from '../errors'; import { ChatSession } from '../methods/chat-session'; import { countTokens } from '../methods/count-tokens'; import { @@ -44,16 +42,13 @@ import { formatSystemInstruction } from '../requests/request-helpers'; import { VertexAI } from '../public-types'; -import { ApiSettings } from '../types/internal'; -import { VertexAIService } from '../service'; +import { VertexAIModel } from './vertexai-model'; /** * Class for generative model APIs. * @public */ -export class GenerativeModel { - private _apiSettings: ApiSettings; - model: string; +export class GenerativeModel extends VertexAIModel { generationConfig: GenerationConfig; safetySettings: SafetySetting[]; requestOptions?: RequestOptions; @@ -66,44 +61,7 @@ export class GenerativeModel { modelParams: ModelParams, requestOptions?: RequestOptions ) { - if (!vertexAI.app?.options?.apiKey) { - throw new VertexAIError( - VertexAIErrorCode.NO_API_KEY, - `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` - ); - } else if (!vertexAI.app?.options?.projectId) { - throw new VertexAIError( - VertexAIErrorCode.NO_PROJECT_ID, - `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` - ); - } else { - this._apiSettings = { - apiKey: vertexAI.app.options.apiKey, - project: vertexAI.app.options.projectId, - location: vertexAI.location - }; - if ((vertexAI as VertexAIService).appCheck) { - this._apiSettings.getAppCheckToken = () => - (vertexAI as VertexAIService).appCheck!.getToken(); - } - - if ((vertexAI as VertexAIService).auth) { - this._apiSettings.getAuthToken = () => - (vertexAI as VertexAIService).auth!.getToken(); - } - } - if (modelParams.model.includes('/')) { - if (modelParams.model.startsWith('models/')) { - // Add "publishers/google" if the user is only passing in 'models/model-name'. - this.model = `publishers/google/${modelParams.model}`; - } else { - // Any other custom format (e.g. tuned models) must be passed in correctly. - this.model = modelParams.model; - } - } else { - // If path is not included, assume it's a non-tuned model. - this.model = `publishers/google/models/${modelParams.model}`; - } + super(vertexAI, modelParams.model); this.generationConfig = modelParams.generationConfig || {}; this.safetySettings = modelParams.safetySettings || []; this.tools = modelParams.tools; diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index 909bff3dea2..a0bc857a53b 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -44,68 +44,6 @@ const fakeVertexAI: VertexAI = { }; describe('ImagenModel', () => { - it('handles plain model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' }); - expect(imagenModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles models/ prefixed model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'models/my-model' - }); - expect(imagenModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles full model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'publishers/google/models/my-model' - }); - expect(imagenModel.model).to.equal('publishers/google/models/my-model'); - }); - it('handles prefixed tuned model name', () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'tunedModels/my-model' - }); - expect(imagenModel.model).to.equal('tunedModels/my-model'); - }); - it('throws if not passed an api key', () => { - const fakeVertexAI: VertexAI = { - app: { - name: 'DEFAULT', - automaticDataCollectionEnabled: true, - options: { - projectId: 'my-project' - } - }, - location: 'us-central1' - }; - try { - new ImagenModel(fakeVertexAI, { - model: 'my-model' - }); - } catch (e) { - expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); - } - }); - it('throws if not passed a project ID', () => { - const fakeVertexAI: VertexAI = { - app: { - name: 'DEFAULT', - automaticDataCollectionEnabled: true, - options: { - apiKey: 'key' - } - }, - location: 'us-central1' - }; - try { - new ImagenModel(fakeVertexAI, { - model: 'my-model' - }); - } catch (e) { - expect((e as VertexAIError).code).to.equal( - VertexAIErrorCode.NO_PROJECT_ID - ); - } - }); it('generateImages makes a request to predict with default parameters', async () => { const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' @@ -190,7 +128,7 @@ describe('ImagenModel', () => { const prompt = 'A photorealistic image of a toy boat at sea.'; await imagenModel.generateImages(prompt, { numberOfImages: 4, - aspectRatio: ImagenAspectRatio.WIDESCREEN, + aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, negativePrompt: 'do not hallucinate' }); expect(makeRequestStub).to.be.calledWith( @@ -237,7 +175,7 @@ describe('ImagenModel', () => { expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); expect((e as VertexAIError).message).to.include('400'); expect((e as VertexAIError).message).to.include( - "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 42876398" + "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback." ); } finally { restore(); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index 17d1e1cfb2c..3434e1dd99d 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -15,102 +15,85 @@ * limitations under the License. */ -import { VertexAIError } from '../errors'; import { VertexAI } from '../public-types'; import { Task, makeRequest } from '../requests/request'; import { createPredictRequestBody } from '../requests/request-helpers'; import { handlePredictResponse } from '../requests/response-helpers'; -import { VertexAIService } from '../service'; import { ImagenGCSImage, ImagenGCSImageResponse, - ImagenImageFormat, ImagenGenerationConfig, ImagenInlineImage, RequestOptions, - VertexAIErrorCode, ImagenModelParams, ImagenInlineImageResponse, ImagenModelConfig } from '../types'; -import { ApiSettings } from '../types/internal'; +import { VertexAIModel } from './vertexai-model'; /** * Class for Imagen model APIs. + * + * This class provides methods for generating images using the Imagen model. + * You can generate images inline as base64-encoded strings, or directly to + * Google Cloud Storage (GCS). + * + * @example + * ```javascript + * const imagen = new ImagenModel(vertexAI, { + * model: 'imagen-3.0-generate-001' + * }); + * + * const response = await imagen.generateImages('A photo of a cat'); + * console.log(response.images[0].bytesBase64Encoded); + * ``` + * * @public */ -export class ImagenModel { - model: string; - private _apiSettings: ApiSettings; - private modelConfig: ImagenModelConfig; +export class ImagenModel extends VertexAIModel { + /** + * Model-level configurations to use when using Imagen. + */ + readonly modelConfig: ImagenModelConfig; /** + * Constructs a new instance of the {@link ImagenModel} class. * - * @param vertexAI - * @param modelParams - * @param requestOptions + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelParams - Parameters to use when making Imagen requests. + * @param requestOptions - Additional options to use when making requests. + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. */ constructor( vertexAI: VertexAI, modelParams: ImagenModelParams, - private requestOptions?: RequestOptions + readonly requestOptions?: RequestOptions ) { const { model, ...modelConfig } = modelParams; + super(vertexAI, model); this.modelConfig = modelConfig; - if (model.includes('/')) { - if (model.startsWith('models/')) { - // Add "publishers/google" if the user is only passing in 'models/model-name'. - this.model = `publishers/google/${model}`; - } else { - // Any other custom format (e.g. tuned models) must be passed in correctly. - this.model = model; - } - } else { - // If path is not included, assume it's a non-tuned model. - this.model = `publishers/google/models/${model}`; - } - - if (!vertexAI.app?.options?.apiKey) { - throw new VertexAIError( - VertexAIErrorCode.NO_API_KEY, - `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` - ); - } else if (!vertexAI.app?.options?.projectId) { - throw new VertexAIError( - VertexAIErrorCode.NO_PROJECT_ID, - `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` - ); - } else { - this._apiSettings = { - apiKey: vertexAI.app.options.apiKey, - project: vertexAI.app.options.projectId, - location: vertexAI.location - }; - if ((vertexAI as VertexAIService).appCheck) { - this._apiSettings.getAppCheckToken = () => - (vertexAI as VertexAIService).appCheck!.getToken(); - } - - if ((vertexAI as VertexAIService).auth) { - this._apiSettings.getAuthToken = () => - (vertexAI as VertexAIService).auth!.getToken(); - } - } } /** - * Generates images using the Imagen model and returns them as base64-encoded strings. + * Generates images using the Imagen model and returns them as + * base64-encoded strings. * - * @param prompt The text prompt used to generate the images. - * @param imagenRequestOptions Configuration options for the Imagen generation request. + * @param prompt - The text prompt used to generate the images. + * @param imagenRequestOptions - Configuration options for the Imagen + * generation request. * See {@link ImagenGenerationConfig}. - * @returns A promise that resolves to an {@link ImagenInlineImageResponse} object containing the generated images. + * @returns A promise that resolves to an {@link ImagenInlineImageResponse} + * object containing the generated images. * - * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * @throws If the request to generate images fails. This happens if the + * prompt is blocked. * * @remarks - * If one or more images are filtered, the returned object will have a defined `filteredReason` property. - * If all images are filtered, the `images` array will be empty, and no error will be thrown. + * If one or more images are filtered, the returned object will have a + * defined `filteredReason` property. If all images are filtered, the + * `images` array will be empty, and no error will be thrown. */ async generateImages( prompt: string, @@ -133,19 +116,23 @@ export class ImagenModel { } /** - * Generates images using the Imagen model and returns them as base64-encoded strings. + * Generates images to Google Cloud Storage (GCS) using the Imagen model. * - * @param prompt The text prompt used to generate the images. - * @param gcsURI The GCS URI where the images should be stored. - * @param imagenRequestOptions Configuration options for the Imagen generation request. - * See {@link ImagenGenerationConfig}. - * @returns A promise that resolves to an {@link ImagenGCSImageResponse} object containing the generated images. + * @param prompt - The text prompt used to generate the images. + * @param gcsURI - The GCS URI where the images should be stored. + * This should be a directory. For example, `gs://my-bucket/my-directory/`. + * @param imagenRequestOptions - Configuration options for the Imagen + * generation request. See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenGCSImageResponse} + * object containing the URLs of the generated images. * - * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * @throws If the request fails to generate images fails. This happens if + * the prompt is blocked. * * @remarks - * If one or more images are filtered, the returned object will have a defined `filteredReason` property. - * If all images are filtered, the `images` array will be empty, and no error will be thrown. + * If one or more images are filtered due to safety reasons, the returned object + * will have a defined `filteredReason` property. If all images are filtered, + * the `images` array will be empty, and no error will be thrown. */ async generateImagesGCS( prompt: string, @@ -169,31 +156,3 @@ export class ImagenModel { return handlePredictResponse(response); } } - -/** - * Creates an {@link ImagenImageFormat} for a JPEG image, to be included in an {@link ImagenModelParams}. - * - * @param compressionQuality The level of compression. - * @returns {@link ImagenImageFormat} - * - * @public - */ -export function jpeg(compressionQuality: number): ImagenImageFormat { - return { - mimeType: 'image/jpeg', - compressionQuality - }; -} - -/** - * Creates an {@link ImageImageFormat} for a PNG image, to be included in a {@link ImagenModelParams}. - * - * @returns {@link ImageImageFormat} - * - * @public - */ -export function png(): ImagenImageFormat { - return { - mimeType: 'image/png' - }; -} diff --git a/packages/vertexai/src/models/index.ts b/packages/vertexai/src/models/index.ts new file mode 100644 index 00000000000..a6ada0d894c --- /dev/null +++ b/packages/vertexai/src/models/index.ts @@ -0,0 +1,3 @@ +export * from './vertexai-model'; +export * from './generative-model'; +export * from './imagen-model'; \ No newline at end of file diff --git a/packages/vertexai/src/models/vertexai-model.test.ts b/packages/vertexai/src/models/vertexai-model.test.ts new file mode 100644 index 00000000000..6b5cdd48e2a --- /dev/null +++ b/packages/vertexai/src/models/vertexai-model.test.ts @@ -0,0 +1,103 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { VertexAI, VertexAIErrorCode } from '../public-types'; +import sinonChai from 'sinon-chai'; +import { VertexAIModel } from './vertexai-model'; +import { VertexAIError } from '../errors'; + +use(sinonChai); + +/** + * A class that extends VertexAIModel that allows us to test the protected constructor. + */ +class TestModel extends VertexAIModel { + /* eslint-disable @typescript-eslint/no-useless-constructor */ + constructor( + vertexAI: VertexAI, + modelName: string + ) { + super(vertexAI, modelName); + } +} + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('VertexAIModel', () => { + it('handles plain model name', () => { + const testModel = new TestModel(fakeVertexAI, 'my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles models/ prefixed model name', () => { + const testModel = new TestModel(fakeVertexAI, 'models/my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles full model name', () => { + const testModel = new TestModel(fakeVertexAI, 'publishers/google/models/my-model'); + expect(testModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles prefixed tuned model name', () => { + const testModel = new TestModel(fakeVertexAI, 'tunedModels/my-model'); + expect(testModel.model).to.equal('tunedModels/my-model'); + }); + it('throws if not passed an api key', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + projectId: 'my-project' + } + }, + location: 'us-central1' + }; + try { + new TestModel(fakeVertexAI, 'my-model'); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); + } + }); + it('throws if not passed a project ID', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key' + } + }, + location: 'us-central1' + }; + try { + new TestModel(fakeVertexAI, 'my-model'); + } catch (e) { + expect((e as VertexAIError).code).to.equal( + VertexAIErrorCode.NO_PROJECT_ID + ); + } + }); +}); diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts new file mode 100644 index 00000000000..0a05eb6fea6 --- /dev/null +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -0,0 +1,94 @@ +import { VertexAIError } from "../errors"; +import { VertexAI, VertexAIErrorCode } from "../public-types"; +import { VertexAIService } from "../service"; +import { ApiSettings } from "../types/internal"; + +/** + * Base class for Vertex AI in Firebase model APIs. + * + * @public + */ +export class VertexAIModel { + /** + * The fully qualified model resource name to use for generating images + * (e.g. `publishers/google/models/imagen-3.0-generate-001`). + */ + readonly model: string; + + protected _apiSettings: ApiSettings; + + /** + * Constructs a new instance of the {@link VertexAIModel} class. + * + * This constructor should only be called from subclasses that provide + * a model API. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelName - The name of the model being used. It can be in one of the following formats: + * - `my-model` (short name, will resolve to `publishers/google/models/my-model`) + * - `models/my-model` (will resolve to `publishers/google/models/my-model`) + * - `publishers/my-publisher/models/my-model` (fully qualified model name) + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + * + * @internal + */ + protected constructor( + vertexAI: VertexAI, + modelName: string + ) { + this.model = VertexAIModel.normalizeModelName(modelName); + + if (!vertexAI.app?.options?.apiKey) { + throw new VertexAIError( + VertexAIErrorCode.NO_API_KEY, + `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` + ); + } else if (!vertexAI.app?.options?.projectId) { + throw new VertexAIError( + VertexAIErrorCode.NO_PROJECT_ID, + `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` + ); + } else { + this._apiSettings = { + apiKey: vertexAI.app.options.apiKey, + project: vertexAI.app.options.projectId, + location: vertexAI.location + }; + if ((vertexAI as VertexAIService).appCheck) { + this._apiSettings.getAppCheckToken = () => + (vertexAI as VertexAIService).appCheck!.getToken(); + } + + if ((vertexAI as VertexAIService).auth) { + this._apiSettings.getAuthToken = () => + (vertexAI as VertexAIService).auth!.getToken(); + } + } + } + + /** + * Normalizes the given model name to a fully qualified model resource name. + * + * @param modelName - The model name to normalize. + * @returns The fully qualified model resource name. + */ + static normalizeModelName(modelName: string): string { + let model: string; + if (modelName.includes('/')) { + if (modelName.startsWith('models/')) { + // Add 'publishers/google' if the user is only passing in 'models/model-name'. + model = `publishers/google/${modelName}`; + } else { + // Any other custom format (e.g. tuned models) must be passed in correctly. + model = modelName; + } + } else { + // If path is not included, assume it's a non-tuned model. + model = `publishers/google/models/${modelName}`; + } + + return model; + } +} \ No newline at end of file diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts new file mode 100644 index 00000000000..6b7c84d8f2b --- /dev/null +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -0,0 +1,48 @@ +/** + * Defines the image format for images output by Imagen. + * + * Use this class to specify the desired format (JPEG or PNG) and compression quality + * for images generated by Imagen. This is typically included as part of + * {@link ImagenModelParams}. + * + * @example + * ```javascript + * const imagenModelParams = { + * // ... other ImagenModelParams + * imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. + * } + * ``` + * + * @public + */ +export class ImagenImageFormat { + mimeType: string; + compressionQuality?: number; + + private constructor() { + this.mimeType = "image/png"; + } + + /** + * Creates an ImagenImageFormat for a JPEG image. + * + * @param compressionQuality - The level of compression (a number between 0 and 100). + * @returns ImagenImageFormat + * + * @public + */ + static jpeg(compressionQuality: number): ImagenImageFormat { + return { mimeType: "image/jpeg", compressionQuality }; + } + + /** + * Creates an ImageImageFormat for a PNG image. + * + * @returns ImageImageFormat + * + * @public + */ + static png(): ImagenImageFormat { + return { mimeType: "image/png" }; + } +} \ No newline at end of file diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 9582f384a2c..2317457aca4 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -239,7 +239,7 @@ describe('request formatting methods', () => { const addWatermark = true; const numberOfImages = 4; const negativePrompt = 'do not hallucinate'; - const aspectRatio = ImagenAspectRatio.WIDESCREEN; + const aspectRatio = ImagenAspectRatio.LANDSCAPE_16x9; const body = createPredictRequestBody({ prompt, numberOfImages, diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index fa4bdf402d7..831bb6a26a5 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -18,13 +18,13 @@ import { Content, GenerateContentRequest, - PredictRequestBody, Part, VertexAIErrorCode, ImagenAspectRatio, - ImagenRequestConfig } from '../types'; import { VertexAIError } from '../errors'; +import { ImagenImageFormat } from './imagen-image-format'; +import { ImagenRequestConfig, PredictRequestBody } from '../types/internal'; export function formatSystemInstruction( input?: string | Part | Content @@ -53,12 +53,10 @@ export function formatNewContent( newParts = [{ text: request }]; } else { for (const elem of request) { - // This throws an error if request is not iterable if (typeof elem === 'string') { newParts.push({ text: elem }); } else { - // We assume this is a Part, but it could be anything. - newParts.push(elem); // This could be + newParts.push(elem); } } } @@ -118,8 +116,6 @@ export function formatGenerateContentInput( if ((params as GenerateContentRequest).contents) { formattedRequest = params as GenerateContentRequest; } else { - // Array or string - // ... or something else const content = formatNewContent(params as string | Array); formattedRequest = { contents: [content] }; } @@ -140,7 +136,7 @@ export function formatGenerateContentInput( export function createPredictRequestBody({ prompt, gcsURI, - imageFormat = { mimeType: 'image/png' }, + imageFormat = ImagenImageFormat.png(), addWatermark, safetySettings, numberOfImages = 1, diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 6b82463dfa1..acdc6e741d1 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -201,9 +201,9 @@ export function formatBlockErrorMessage( } /** - * Convert a generic successful fetch {@link Response} body to an Imagen response object + * Convert a generic successful fetch response body to an Imagen response object * that can be returned to the user. This converts the REST APIs response format to our - * representation of a response. + * APIs representation of a response. * * @internal */ diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 1171df81278..ba7ee2c8f49 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -// Internal Imagen types +import { ImagenGenerationConfig, ImagenModelConfig } from "./requests"; /** * A response from the REST API is expected to look like this in the success case: @@ -43,12 +43,91 @@ */ export interface ImagenResponseInternal { predictions?: Array<{ - // Defined if the prediction was not filtered + /** + * The MIME type of the generated image. + */ mimeType?: string; + /** + * The image data encoded as a base64 string. + */ bytesBase64Encoded?: string; + gcsUri?: string; - // Defined if the prediction was filtered, and there is no image + /** + * The reason why the image was filtered. + */ raiFilteredReason?: string; }>; } + +/** + * The parameters to be sent in the request body of the HTTP call + * to the Vertex AI backend. + * + * We need a seperate internal-only interface for this because the REST + * API expects different parameter names than what we show to our users. + * + * This interface should be populated from the ImagenGenerationConfig that + * the user defines. + * + * Sample request body JSON: + * { + * "instances": [ + * { + * "prompt": "Portrait of a golden retriever on a beach." + * } + * ], + * "parameters": { + * "mimeType": "image/png", + * "safetyFilterLevel": "block_low_and_above", + * "personGeneration": "allow_all", + * "sampleCount": 2, + * "includeRaiReason": true, + * "aspectRatio": "9:16" + * } + * } + * + * @internal + */ +export interface PredictRequestBody { + instances: [ + { + prompt: string; + } + ]; + parameters: { + sampleCount: number; // Maps to numberOfImages + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; // Maps to gcsURI + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; // Maps to personFilterLevel + includeRaiReason: boolean; + }; +} + +/** + * Contains all possible REST API paramaters. + * This is the intersection of the model-level (`ImagenModelParams`), + * request-level (`ImagenGenerationConfig`) configurations, along with + * the other required parameters prompt and gcsURI (for GCS generation only). + * + * @internal + */ +export interface ImagenRequestConfig + extends ImagenModelConfig, + ImagenGenerationConfig { + /** + * The text prompt used to generate the images. + */ + prompt: string; + /** + * The Google Cloud Storage (GCS) URI where the images should be stored + * (for GCS requests only). + */ + gcsURI?: string; +} \ No newline at end of file diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index d87723a31c7..a9a9c664b45 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -15,128 +15,130 @@ * limitations under the License. */ +import { ImagenImageFormat } from '../../requests/imagen-image-format'; + +/** + * Parameters for configuring an {@link ImagenModel}. + * + * @public + */ export interface ImagenModelParams extends ImagenModelConfig { + /** + * The Imagen model to use for generating images. + * For example: `imagen-3.0-generate-001`. + */ model: string; } +/** + * Model-level configuration options for Imagen. + * + * @public + */ export interface ImagenModelConfig { + /** + * The image format of the generated images. + */ imageFormat?: ImagenImageFormat; + /** + * Whether to add a watermark to generated images. + */ addWatermark?: boolean; + /** + * Safety settings for filtering inapropriate content. + */ safetySettings?: ImagenSafetySettings; } -export interface ImagenGenerationConfig { - numberOfImages?: number; // Default to 1. Possible values are [1...4] - negativePrompt?: string; // Default to null - aspectRatio?: ImagenAspectRatio; // Default to "1:1" -} - /** - * Contains all possible REST API paramaters. - * This is the intersection of the model-level (`ImagenModelParams`), - * request-level (`ImagenGenerationConfig`) configurations, along with - * the other required parameters prompt and gcsURI (for GCS generation only). + * Request-level configuration options for generating images with Imagen. * - * @internal + * @public */ -export interface ImagenRequestConfig - extends ImagenModelConfig, - ImagenGenerationConfig { - prompt: string; - gcsURI?: string; -} - -export interface ImagenImageFormat { - mimeType: string; // image/png, or image/jpeg, default image/png - compressionQuality?: number; // 0-100, default 75. Only for image/jpeg +export interface ImagenGenerationConfig { + /** + * The number of images to generate. Must be between 1 and 4. Defaults to 1. + */ + numberOfImages?: number; + /** + * A text prompt describing what should not be included in the image. + */ + negativePrompt?: string; + /** + * The aspect ratio of the generated images. Defaults to `1:1`. + */ + aspectRatio?: ImagenAspectRatio; } /** + * Safety filter levels for Imagen. + * * @public */ export enum ImagenSafetyFilterLevel { + /** + * Block images with low or higher safety severity. + */ BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + /** + * Block images with medium or higher safety severity. + */ BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + /** + * Block images with high safety severity. + */ BLOCK_ONLY_HIGH = 'block_only_high', + /** + * Do not block any images based on safety. + */ BLOCK_NONE = 'block_none' } /** + * Person filter levels for Imagen. + * * @public */ export enum ImagenPersonFilterLevel { + /** + * Do not allow any person generation. + */ BLOCK_ALL = 'dont_allow', + /** + * Allow only adults in generated images. + */ ALLOW_ADULT = 'allow_adult', + /** + * Allow all person generation. + */ ALLOW_ALL = 'allow_all' } /** + * Safety settings for Imagen. + * * @public */ export interface ImagenSafetySettings { /** - * Safety filter level + * The safety filter level to use. */ safetyFilterLevel?: ImagenSafetyFilterLevel; /** - * Generate people. + * The person filter level to use. */ personFilterLevel?: ImagenPersonFilterLevel; } -export enum ImagenAspectRatio { - SQUARE = '1:1', - CLASSIC_PORTRAIT = '3:4', - CLASSIC_LANDSCAPE = '4:3', - WIDESCREEN = '16:9', - PORTRAIT = '9:16' -} - /** - * The parameters to be sent in the request body of the HTTP call - * to the Vertex AI backend. - * - * We need a seperate internal-only interface for this because the REST - * API expects different parameter names than what we show to our users. - * - * This interface should be populated from the {@link ImagenGenerationConfig} that - * the user defines. - * - * Sample request body JSON: - * { - * "instances": [ - * { - * "prompt": "Portrait of a golden retriever on a beach." - * } - * ], - * "parameters": { - * "mimeType": "image/png", - * "safetyFilterLevel": "block_low_and_above", - * "personGeneration": "allow_all", - * "sampleCount": 2, - * "includeRaiReason": true, - * "aspectRatio": "9:16" - * } - * } + * Aspect ratios for Imagen images. * - * @internal + * @public */ -export interface PredictRequestBody { - instances: [ - { - prompt: string; - } - ]; - parameters: { - sampleCount: number; // maps to numberOfImages - aspectRatio: string; - mimeType: string; - compressionQuality?: number; - negativePrompt?: string; - storageUri?: string; - addWatermark?: boolean; - safetyFilterLevel?: string; - personGeneration?: string; - includeRaiReason: boolean; - }; +export enum ImagenAspectRatio { + SQUARE = '1:1', + LANDSCAPE_3x4 = '3:4', + PORTRAIT_4x3 = '4:3', + LANDSCAPE_16x9 = '16:9', + PORTRAIT_9x16 = '9:16' } diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index 620f49be4b9..a05e3fda0f7 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -16,58 +16,73 @@ */ /** - * Base class for types of images that the Imagen Model can return. - * - * @internal - */ -export interface ImagenImage { - mimeType: string; -} - -/** - * Image generated by Imagen to inline bytes. + * An image generated by Imagen, represented as inline bytes. * * @public */ -export interface ImagenInlineImage extends ImagenImage { +export interface ImagenInlineImage { + /** + * The MIME type of the image. + */ + mimeType: string; + /** + * The image data encoded as a base64 string. + */ bytesBase64Encoded: string; } /** - * Image generated by Imagen, stored in Google Cloud Storage (GCS). + * An image generated by Imagen, stored in Google Cloud Storage (GCS). * * @public */ -export interface ImagenGCSImage extends ImagenImage { +export interface ImagenGCSImage { /** - * The Google Cloud Storage (GCS) URI at which the generated image is stored. + * The MIME type of the image. + */ + mimeType: string; + /** + * The Google Cloud Storage (GCS) URI where the image is stored. */ gcsURI: string; } /** - * Imagen image response. + * The response from a request to generate images to inline bytes. * * @public */ export interface ImagenInlineImageResponse { /** - * The images generated by Imagen. If all images were filtered, this will be empty. + * The images generated by Imagen. + * If all images were filtered out due to safety reasons, this array will be empty. */ images: ImagenInlineImage[]; /** - * The reason the missing images were filtered. - * For the mappings of error codes to reasons, see {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + * The reason why any images were filtered. This field is only present if one + * or more images were filtered. + * For the mappings of error codes to reasons, see + * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. */ filteredReason?: string; } +/** + * The response from a request to generate images to Google Cloud Storage (GCS). + * + * @public + */ export interface ImagenGCSImageResponse { + /** + * The images generated by Imagen. + * If all images were filtered due to safety reasons, this array will be empty. + */ images: ImagenGCSImage[]; - filteredReason?: string; -} - -export interface ImagenImageReponse { - images: ImagenImage[]; + /** + * The reason why any images were filtered. This field is only present if one + * or more images were filtered. + * For the mappings of error codes to reasons, see + * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + */ filteredReason?: string; } diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index e2a442821da..83cd4366f12 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -46,11 +46,6 @@ export interface GenerateContentStreamResult { response: Promise; } -export interface SafetyAttributes { - categories: string[]; - scores: number[]; -} - /** * Response object wrapped with helper methods. * From 4b6c42fd75651e57200e09644cbe43ad90054517 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:47:36 -0600 Subject: [PATCH 08/31] Generate devsite docs --- common/api-review/vertexai.api.md | 9 +- docs-devsite/vertexai.generativemodel.md | 12 +-- docs-devsite/vertexai.imagengcsimage.md | 20 +++- .../vertexai.imagengcsimageresponse.md | 10 +- .../vertexai.imagengenerationconfig.md | 14 ++- docs-devsite/vertexai.imagenimageformat.md | 74 ++++++++++++- docs-devsite/vertexai.imagenimagereponse.md | 40 ------- docs-devsite/vertexai.imageninlineimage.md | 20 +++- .../vertexai.imageninlineimageresponse.md | 10 +- docs-devsite/vertexai.imagenmodel.md | 63 ++++++++--- docs-devsite/vertexai.imagenmodelconfig.md | 14 ++- docs-devsite/vertexai.imagenmodelparams.md | 12 +-- docs-devsite/vertexai.imagensafetysettings.md | 9 +- docs-devsite/vertexai.md | 101 ++++++------------ docs-devsite/vertexai.safetyattributes.md | 40 ------- docs-devsite/vertexai.vertexaimodel.md | 66 ++++++++++++ .../vertexai/src/models/vertexai-model.ts | 3 + .../src/requests/imagen-image-format.ts | 6 ++ .../vertexai/src/types/imagen/requests.ts | 15 +++ 19 files changed, 314 insertions(+), 224 deletions(-) delete mode 100644 docs-devsite/vertexai.imagenimagereponse.md delete mode 100644 docs-devsite/vertexai.safetyattributes.md create mode 100644 docs-devsite/vertexai.vertexaimodel.md diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 5b5c00b122e..8affc2a3922 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -432,15 +432,10 @@ export enum HarmSeverity { // @public export enum ImagenAspectRatio { - // (undocumented) LANDSCAPE_16x9 = "16:9", - // (undocumented) LANDSCAPE_3x4 = "3:4", - // (undocumented) PORTRAIT_4x3 = "4:3", - // (undocumented) PORTRAIT_9x16 = "9:16", - // (undocumented) SQUARE = "1:1" } @@ -465,10 +460,8 @@ export interface ImagenGenerationConfig { // @public export class ImagenImageFormat { - // (undocumented) compressionQuality?: number; static jpeg(compressionQuality: number): ImagenImageFormat; - // (undocumented) mimeType: string; static png(): ImagenImageFormat; } @@ -821,7 +814,7 @@ export const enum VertexAIErrorCode { export class VertexAIModel { // @internal protected constructor(vertexAI: VertexAI, modelName: string); - // (undocumented) + // @internal (undocumented) protected _apiSettings: ApiSettings; readonly model: string; static normalizeModelName(modelName: string): string; diff --git a/docs-devsite/vertexai.generativemodel.md b/docs-devsite/vertexai.generativemodel.md index 7105f9c100b..b734e241e78 100644 --- a/docs-devsite/vertexai.generativemodel.md +++ b/docs-devsite/vertexai.generativemodel.md @@ -15,8 +15,9 @@ Class for generative model APIs. Signature: ```typescript -export declare class GenerativeModel +export declare class GenerativeModel extends VertexAIModel ``` +Extends: [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) ## Constructors @@ -29,7 +30,6 @@ export declare class GenerativeModel | Property | Modifiers | Type | Description | | --- | --- | --- | --- | | [generationConfig](./vertexai.generativemodel.md#generativemodelgenerationconfig) | | [GenerationConfig](./vertexai.generationconfig.md#generationconfig_interface) | | -| [model](./vertexai.generativemodel.md#generativemodelmodel) | | string | | | [requestOptions](./vertexai.generativemodel.md#generativemodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | | [safetySettings](./vertexai.generativemodel.md#generativemodelsafetysettings) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface)\[\] | | | [systemInstruction](./vertexai.generativemodel.md#generativemodelsysteminstruction) | | [Content](./vertexai.content.md#content_interface) | | @@ -71,14 +71,6 @@ constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: Reque generationConfig: GenerationConfig; ``` -## GenerativeModel.model - -Signature: - -```typescript -model: string; -``` - ## GenerativeModel.requestOptions Signature: diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md index eb2e9b76556..af9887ba9dd 100644 --- a/docs-devsite/vertexai.imagengcsimage.md +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -10,27 +10,37 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImage interface -Image generated by Imagen, stored in Google Cloud Storage (GCS). +An image generated by Imagen, stored in Google Cloud Storage (GCS). Signature: ```typescript -export interface ImagenGCSImage extends ImagenImage +export interface ImagenGCSImage ``` -Extends: ImagenImage ## Properties | Property | Type | Description | | --- | --- | --- | -| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI at which the generated image is stored. | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI where the image is stored. | +| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image. | ## ImagenGCSImage.gcsURI -The Google Cloud Storage (GCS) URI at which the generated image is stored. +The Google Cloud Storage (GCS) URI where the image is stored. Signature: ```typescript gcsURI: string; ``` + +## ImagenGCSImage.mimeType + +The MIME type of the image. + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagengcsimageresponse.md b/docs-devsite/vertexai.imagengcsimageresponse.md index 0c20cb1e752..cb1fa2cfe70 100644 --- a/docs-devsite/vertexai.imagengcsimageresponse.md +++ b/docs-devsite/vertexai.imagengcsimageresponse.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImageResponse interface +The response from a request to generate images to Google Cloud Storage (GCS). + Signature: ```typescript @@ -20,11 +22,13 @@ export interface ImagenGCSImageResponse | Property | Type | Description | | --- | --- | --- | -| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | | -| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | | +| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. | ## ImagenGCSImageResponse.filteredReason +The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). + Signature: ```typescript @@ -33,6 +37,8 @@ filteredReason?: string; ## ImagenGCSImageResponse.images +The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md index bf28bc291ae..9de48e45e81 100644 --- a/docs-devsite/vertexai.imagengenerationconfig.md +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationConfig interface +Request-level configuration options for generating images with Imagen. + Signature: ```typescript @@ -20,12 +22,14 @@ export interface ImagenGenerationConfig | Property | Type | Description | | --- | --- | --- | -| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | -| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | -| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | The aspect ratio of the generated images. Defaults to 1:1. | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | A text prompt describing what should not be included in the image. | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | The number of images to generate. Must be between 1 and 4. Defaults to 1. | ## ImagenGenerationConfig.aspectRatio +The aspect ratio of the generated images. Defaults to `1:1`. + Signature: ```typescript @@ -34,6 +38,8 @@ aspectRatio?: ImagenAspectRatio; ## ImagenGenerationConfig.negativePrompt +A text prompt describing what should not be included in the image. + Signature: ```typescript @@ -42,6 +48,8 @@ negativePrompt?: string; ## ImagenGenerationConfig.numberOfImages +The number of images to generate. Must be between 1 and 4. Defaults to 1. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md index 7ab7b9eb277..abdd561caa4 100644 --- a/docs-devsite/vertexai.imagenimageformat.md +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -9,22 +9,35 @@ overwritten. Changes should be made in the source code at https://github.com/firebase/firebase-js-sdk {% endcomment %} -# ImagenImageFormat interface +# ImagenImageFormat class +Defines the image format for images output by Imagen. + +Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + Signature: ```typescript -export interface ImagenImageFormat +export declare class ImagenImageFormat ``` ## Properties -| Property | Type | Description | +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | The level of compression (a number between 0 and 100). | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | The MIME type. | + +## Methods + +| Method | Modifiers | Description | | --- | --- | --- | -| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | number | | -| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | string | | +| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | Creates an ImagenImageFormat for a JPEG image. | +| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | Creates an ImageImageFormat for a PNG image. | ## ImagenImageFormat.compressionQuality +The level of compression (a number between 0 and 100). + Signature: ```typescript @@ -33,8 +46,59 @@ compressionQuality?: number; ## ImagenImageFormat.mimeType +The MIME type. + Signature: ```typescript mimeType: string; ``` + +## ImagenImageFormat.jpeg() + +Creates an ImagenImageFormat for a JPEG image. + +Signature: + +```typescript +static jpeg(compressionQuality: number): ImagenImageFormat; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| compressionQuality | number | The level of compression (a number between 0 and 100). | + +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) + +ImagenImageFormat + +## ImagenImageFormat.png() + +Creates an ImageImageFormat for a PNG image. + +Signature: + +```typescript +static png(): ImagenImageFormat; +``` +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) + +ImageImageFormat + +### Example + + +```javascript +const imagenModelParams = { + // ... other ImagenModelParams + imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. +} + +``` + diff --git a/docs-devsite/vertexai.imagenimagereponse.md b/docs-devsite/vertexai.imagenimagereponse.md deleted file mode 100644 index 14e34651a41..00000000000 --- a/docs-devsite/vertexai.imagenimagereponse.md +++ /dev/null @@ -1,40 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# ImagenImageReponse interface -Signature: - -```typescript -export interface ImagenImageReponse -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [filteredReason](./vertexai.imagenimagereponse.md#imagenimagereponsefilteredreason) | string | | -| [images](./vertexai.imagenimagereponse.md#imagenimagereponseimages) | ImagenImage\[\] | | - -## ImagenImageReponse.filteredReason - -Signature: - -```typescript -filteredReason?: string; -``` - -## ImagenImageReponse.images - -Signature: - -```typescript -images: ImagenImage[]; -``` diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md index fdfac65c15a..6d3a3972445 100644 --- a/docs-devsite/vertexai.imageninlineimage.md +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -10,25 +10,37 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenInlineImage interface -Image generated by Imagen to inline bytes. +An image generated by Imagen, represented as inline bytes. Signature: ```typescript -export interface ImagenInlineImage extends ImagenImage +export interface ImagenInlineImage ``` -Extends: ImagenImage ## Properties | Property | Type | Description | | --- | --- | --- | -| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | The image data encoded as a base64 string. | +| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | The MIME type of the image. | ## ImagenInlineImage.bytesBase64Encoded +The image data encoded as a base64 string. + Signature: ```typescript bytesBase64Encoded: string; ``` + +## ImagenInlineImage.mimeType + +The MIME type of the image. + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imageninlineimageresponse.md b/docs-devsite/vertexai.imageninlineimageresponse.md index 06cf3a2857f..c895dea6ea4 100644 --- a/docs-devsite/vertexai.imageninlineimageresponse.md +++ b/docs-devsite/vertexai.imageninlineimageresponse.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenInlineImageResponse interface -Imagen image response. +The response from a request to generate images to inline bytes. Signature: @@ -22,12 +22,12 @@ export interface ImagenInlineImageResponse | Property | Type | Description | | --- | --- | --- | -| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | -| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered, this will be empty. | +| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered out due to safety reasons, this array will be empty. | ## ImagenInlineImageResponse.filteredReason -The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). +The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). Signature: @@ -37,7 +37,7 @@ filteredReason?: string; ## ImagenInlineImageResponse.images -The images generated by Imagen. If all images were filtered, this will be empty. +The images generated by Imagen. If all images were filtered out due to safety reasons, this array will be empty. Signature: diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md index f1bff9689fe..190b9e7307a 100644 --- a/docs-devsite/vertexai.imagenmodel.md +++ b/docs-devsite/vertexai.imagenmodel.md @@ -12,34 +12,38 @@ https://github.com/firebase/firebase-js-sdk # ImagenModel class Class for Imagen model APIs. +This class provides methods for generating images using the Imagen model. You can generate images inline as base64-encoded strings, or directly to Google Cloud Storage (GCS). + Signature: ```typescript -export declare class ImagenModel +export declare class ImagenModel extends VertexAIModel ``` +Extends: [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) ## Constructors | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [model](./vertexai.imagenmodel.md#imagenmodelmodel) | | string | | +| [modelConfig](./vertexai.imagenmodel.md#imagenmodelmodelconfig) | | [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configurations to use when using Imagen. | +| [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | ## Methods | Method | Modifiers | Description | | --- | --- | --- | | [generateImages(prompt, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | -| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images using the Imagen model and returns them as base64-encoded strings. | +| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images to Google Cloud Storage (GCS) using the Imagen model. | ## ImagenModel.(constructor) -Constructs a new instance of the `ImagenModel` class +Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. Signature: @@ -51,16 +55,30 @@ constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: | Parameter | Type | Description | | --- | --- | --- | -| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | -| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | -| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making Imagen requests. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | Additional options to use when making requests. | + +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. + +## ImagenModel.modelConfig -## ImagenModel.model +Model-level configurations to use when using Imagen. Signature: ```typescript -model: string; +readonly modelConfig: ImagenModelConfig; +``` + +## ImagenModel.requestOptions + +Signature: + +```typescript +readonly requestOptions?: RequestOptions | undefined; ``` ## ImagenModel.generateImages() @@ -90,13 +108,13 @@ A promise that resolves to an [ImagenInlineImageResponse](./vertexai.imageninlin #### Exceptions -If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). +If the request to generate images fails. This happens if the prompt is blocked. ## ImagenModel.generateImagesGCS() -Generates images using the Imagen model and returns them as base64-encoded strings. +Generates images to Google Cloud Storage (GCS) using the Imagen model. -If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. +If one or more images are filtered due to safety reasons, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. Signature: @@ -109,16 +127,29 @@ generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenG | Parameter | Type | Description | | --- | --- | --- | | prompt | string | The text prompt used to generate the images. | -| gcsURI | string | The GCS URI where the images should be stored. | +| gcsURI | string | The GCS URI where the images should be stored. This should be a directory. For example, gs://my-bucket/my-directory/. | | imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | Returns: Promise<[ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface)> -A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the generated images. +A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the URLs of the generated images. #### Exceptions -If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). +If the request fails to generate images fails. This happens if the prompt is blocked. + +### Example + + +```javascript +const imagen = new ImagenModel(vertexAI, { + model: 'imagen-3.0-generate-001' +}); + +const response = await imagen.generateImages('A photo of a cat'); +console.log(response.images[0].bytesBase64Encoded); + +``` diff --git a/docs-devsite/vertexai.imagenmodelconfig.md b/docs-devsite/vertexai.imagenmodelconfig.md index 9923d0de504..e8948276eb6 100644 --- a/docs-devsite/vertexai.imagenmodelconfig.md +++ b/docs-devsite/vertexai.imagenmodelconfig.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModelConfig interface +Model-level configuration options for Imagen. + Signature: ```typescript @@ -20,12 +22,14 @@ export interface ImagenModelConfig | Property | Type | Description | | --- | --- | --- | -| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | | -| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | -| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | Whether to add a watermark to generated images. | +| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | The image format of the generated images. | +| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inapropriate content. | ## ImagenModelConfig.addWatermark +Whether to add a watermark to generated images. + Signature: ```typescript @@ -34,6 +38,8 @@ addWatermark?: boolean; ## ImagenModelConfig.imageFormat +The image format of the generated images. + Signature: ```typescript @@ -42,6 +48,8 @@ imageFormat?: ImagenImageFormat; ## ImagenModelConfig.safetySettings +Safety settings for filtering inapropriate content. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md index 2d1070a3585..256979e0498 100644 --- a/docs-devsite/vertexai.imagenmodelparams.md +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -10,13 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModelParams interface - Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). Signature: @@ -29,10 +23,12 @@ export interface ImagenModelParams extends ImagenModelConfig | Property | Type | Description | | --- | --- | --- | -| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | The Imagen model to use for generating images. For example: imagen-3.0-generate-001. | ## ImagenModelParams.model +The Imagen model to use for generating images. For example: `imagen-3.0-generate-001`. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md index 0d4a45a6d52..a544b9688bc 100644 --- a/docs-devsite/vertexai.imagensafetysettings.md +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenSafetySettings interface +Safety settings for Imagen. Signature: @@ -21,12 +22,12 @@ export interface ImagenSafetySettings | Property | Type | Description | | --- | --- | --- | -| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Generate people. | -| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter level | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | The person filter level to use. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | The safety filter level to use. | ## ImagenSafetySettings.personFilterLevel -Generate people. +The person filter level to use. Signature: @@ -36,7 +37,7 @@ personFilterLevel?: ImagenPersonFilterLevel; ## ImagenSafetySettings.safetyFilterLevel -Safety filter level +The safety filter level to use. Signature: diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index 827babbcafe..3814ccf42a4 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -21,10 +21,6 @@ The Vertex AI in Firebase Web SDK. | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | | [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. | -| function() | -| [png()](./vertexai.md#png) | Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | -| function(compressionQuality, ...) | -| [jpeg(compressionQuality)](./vertexai.md#jpeg_8e6711f) | Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | ## Classes @@ -34,13 +30,15 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | -| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs. | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | Defines the image format for images output by Imagen.Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs.This class provides methods for generating images using the Imagen model. You can generate images inline as base64-encoded strings, or directly to Google Cloud Storage (GCS). | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | | [Schema](./vertexai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with JSON.stringify() into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) | | [StringSchema](./vertexai.stringschema.md#stringschema_class) | Schema class for "string" types. Can be used with or without enum values. | | [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) | Error class for the Vertex AI in Firebase SDK. | +| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | Base class for Vertex AI in Firebase model APIs. | ## Enumerations @@ -54,9 +52,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | -| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | -| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | -| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | Aspect ratios for Imagen images. | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Person filter levels for Imagen. | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter levels for Imagen. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -92,23 +90,20 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | -| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | Image generated by Imagen, stored in Google Cloud Storage (GCS). | -| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | | -| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | -| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | -| [ImagenImageReponse](./vertexai.imagenimagereponse.md#imagenimagereponse_interface) | | -| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Image generated by Imagen to inline bytes. | -| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | Imagen image response. | -| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | | -| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Copyright 2024 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | -| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in Google Cloud Storage (GCS). | +| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | The response from a request to generate images to Google Cloud Storage (GCS). | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Request-level configuration options for generating images with Imagen. | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | An image generated by Imagen, represented as inline bytes. | +| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | The response from a request to generate images to inline bytes. | +| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configuration options for Imagen. | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for Imagen. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | -| [SafetyAttributes](./vertexai.safetyattributes.md#safetyattributes_interface) | | | [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | | [SchemaInterface](./vertexai.schemainterface.md#schemainterface_interface) | Interface for [Schema](./vertexai.schema.md#schema_class) class. | @@ -209,46 +204,6 @@ export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenMo [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) -## function() - -### png() {:#png} - -Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). - -Signature: - -```typescript -export declare function png(): ImagenImageFormat; -``` -Returns: - -[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) - - -## function(compressionQuality, ...) - -### jpeg(compressionQuality) {:#jpeg_8e6711f} - -Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). - -Signature: - -```typescript -export declare function jpeg(compressionQuality: number): ImagenImageFormat; -``` - -#### Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| compressionQuality | number | The level of compression. | - -Returns: - -[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) - -[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) - ## POSSIBLE\_ROLES Possible roles. @@ -447,6 +402,8 @@ export declare enum HarmSeverity ## ImagenAspectRatio +Aspect ratios for Imagen images. + Signature: ```typescript @@ -457,14 +414,15 @@ export declare enum ImagenAspectRatio | Member | Value | Description | | --- | --- | --- | -| CLASSIC\_LANDSCAPE | "4:3" | | -| CLASSIC\_PORTRAIT | "3:4" | | -| PORTRAIT | "9:16" | | -| SQUARE | "1:1" | | -| WIDESCREEN | "16:9" | | +| LANDSCAPE\_16x9 | "16:9" | Landscape 16:9 aspect ratio. | +| LANDSCAPE\_3x4 | "3:4" | Landscape 3:4 aspect ratio. | +| PORTRAIT\_4x3 | "4:3" | Portrait 4:3 aspect ratio. | +| PORTRAIT\_9x16 | "9:16" | Portrait 9:16 aspect ratio. | +| SQUARE | "1:1" | Square 1:1 aspect ratio. | ## ImagenPersonFilterLevel +Person filter levels for Imagen. Signature: @@ -476,12 +434,13 @@ export declare enum ImagenPersonFilterLevel | Member | Value | Description | | --- | --- | --- | -| ALLOW\_ADULT | "allow_adult" | | -| ALLOW\_ALL | "allow_all" | | -| BLOCK\_ALL | "dont_allow" | | +| ALLOW\_ADULT | "allow_adult" | Allow only adults in generated images. | +| ALLOW\_ALL | "allow_all" | Allow all person generation. | +| BLOCK\_ALL | "dont_allow" | Do not allow any person generation. | ## ImagenSafetyFilterLevel +Safety filter levels for Imagen. Signature: @@ -493,10 +452,10 @@ export declare enum ImagenSafetyFilterLevel | Member | Value | Description | | --- | --- | --- | -| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | -| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | -| BLOCK\_NONE | "block_none" | | -| BLOCK\_ONLY\_HIGH | "block_only_high" | | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | Block images with low or higher safety severity. | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | Block images with medium or higher safety severity. | +| BLOCK\_NONE | "block_none" | Do not block any images based on safety. | +| BLOCK\_ONLY\_HIGH | "block_only_high" | Block images with high safety severity. | ## SchemaType diff --git a/docs-devsite/vertexai.safetyattributes.md b/docs-devsite/vertexai.safetyattributes.md deleted file mode 100644 index 837850ab80f..00000000000 --- a/docs-devsite/vertexai.safetyattributes.md +++ /dev/null @@ -1,40 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# SafetyAttributes interface -Signature: - -```typescript -export interface SafetyAttributes -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [categories](./vertexai.safetyattributes.md#safetyattributescategories) | string\[\] | | -| [scores](./vertexai.safetyattributes.md#safetyattributesscores) | number\[\] | | - -## SafetyAttributes.categories - -Signature: - -```typescript -categories: string[]; -``` - -## SafetyAttributes.scores - -Signature: - -```typescript -scores: number[]; -``` diff --git a/docs-devsite/vertexai.vertexaimodel.md b/docs-devsite/vertexai.vertexaimodel.md new file mode 100644 index 00000000000..ad35e68dded --- /dev/null +++ b/docs-devsite/vertexai.vertexaimodel.md @@ -0,0 +1,66 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# VertexAIModel class +Base class for Vertex AI in Firebase model APIs. + +The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `VertexAIModel` class. + +Signature: + +```typescript +export declare class VertexAIModel +``` + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | The fully qualified model resource name to use for generating images (e.g. publishers/google/models/imagen-3.0-generate-001). | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [normalizeModelName(modelName)](./vertexai.vertexaimodel.md#vertexaimodelnormalizemodelname) | static | Normalizes the given model name to a fully qualified model resource name. | + +## VertexAIModel.model + +The fully qualified model resource name to use for generating images (e.g. `publishers/google/models/imagen-3.0-generate-001`). + +Signature: + +```typescript +readonly model: string; +``` + +## VertexAIModel.normalizeModelName() + +Normalizes the given model name to a fully qualified model resource name. + +Signature: + +```typescript +static normalizeModelName(modelName: string): string; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| modelName | string | The model name to normalize. | + +Returns: + +string + +The fully qualified model resource name. + diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index 0a05eb6fea6..e1ce26c6272 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -15,6 +15,9 @@ export class VertexAIModel { */ readonly model: string; + /** + * @internal + */ protected _apiSettings: ApiSettings; /** diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 6b7c84d8f2b..1b7b9eaf6ac 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -16,7 +16,13 @@ * @public */ export class ImagenImageFormat { + /** + * The MIME type. + */ mimeType: string; + /** + * The level of compression (a number between 0 and 100). + */ compressionQuality?: number; private constructor() { diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index a9a9c664b45..99b69ad09b8 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -136,9 +136,24 @@ export interface ImagenSafetySettings { * @public */ export enum ImagenAspectRatio { + /** + * Square 1:1 aspect ratio. + */ SQUARE = '1:1', + /** + * Landscape 3:4 aspect ratio. + */ LANDSCAPE_3x4 = '3:4', + /** + * Portrait 4:3 aspect ratio. + */ PORTRAIT_4x3 = '4:3', + /** + * Landscape 16:9 aspect ratio. + */ LANDSCAPE_16x9 = '16:9', + /** + * Portrait 9:16 aspect ratio. + */ PORTRAIT_9x16 = '9:16' } From 6215781a9b7adc09cd29155bf8a30cd10956058a Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:55:35 -0600 Subject: [PATCH 09/31] revert dataconnect change --- packages/data-connect/test/unit/transportoptions.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/data-connect/test/unit/transportoptions.test.ts b/packages/data-connect/test/unit/transportoptions.test.ts index a7136b5c408..91e090e6a54 100644 --- a/packages/data-connect/test/unit/transportoptions.test.ts +++ b/packages/data-connect/test/unit/transportoptions.test.ts @@ -16,8 +16,6 @@ */ import { expect } from 'chai'; - -import { queryRef } from '../../src'; import { TransportOptions, areTransportOptionsEqual, @@ -25,6 +23,7 @@ import { getDataConnect } from '../../src/api/DataConnect'; import { app } from '../util'; +import { queryRef } from '../../src'; describe('Transport Options', () => { it('should return false if transport options are not equal', () => { const transportOptions1: TransportOptions = { From 025035a679e04ac50fd947a85fb1ff0e643bdbb2 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 2 Jan 2025 12:56:16 -0600 Subject: [PATCH 10/31] formatting --- packages/vertexai/src/models/index.ts | 19 +++++++- .../src/models/vertexai-model.test.ts | 10 ++--- .../vertexai/src/models/vertexai-model.ts | 44 ++++++++++++------- .../src/requests/imagen-image-format.ts | 29 +++++++++--- .../vertexai/src/requests/request-helpers.ts | 2 +- .../vertexai/src/types/imagen/internal.ts | 4 +- 6 files changed, 78 insertions(+), 30 deletions(-) diff --git a/packages/vertexai/src/models/index.ts b/packages/vertexai/src/models/index.ts index a6ada0d894c..aec06be26fd 100644 --- a/packages/vertexai/src/models/index.ts +++ b/packages/vertexai/src/models/index.ts @@ -1,3 +1,20 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + export * from './vertexai-model'; export * from './generative-model'; -export * from './imagen-model'; \ No newline at end of file +export * from './imagen-model'; diff --git a/packages/vertexai/src/models/vertexai-model.test.ts b/packages/vertexai/src/models/vertexai-model.test.ts index 6b5cdd48e2a..d570230a64c 100644 --- a/packages/vertexai/src/models/vertexai-model.test.ts +++ b/packages/vertexai/src/models/vertexai-model.test.ts @@ -27,10 +27,7 @@ use(sinonChai); */ class TestModel extends VertexAIModel { /* eslint-disable @typescript-eslint/no-useless-constructor */ - constructor( - vertexAI: VertexAI, - modelName: string - ) { + constructor(vertexAI: VertexAI, modelName: string) { super(vertexAI, modelName); } } @@ -57,7 +54,10 @@ describe('VertexAIModel', () => { expect(testModel.model).to.equal('publishers/google/models/my-model'); }); it('handles full model name', () => { - const testModel = new TestModel(fakeVertexAI, 'publishers/google/models/my-model'); + const testModel = new TestModel( + fakeVertexAI, + 'publishers/google/models/my-model' + ); expect(testModel.model).to.equal('publishers/google/models/my-model'); }); it('handles prefixed tuned model name', () => { diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index e1ce26c6272..37099b032b8 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -1,7 +1,24 @@ -import { VertexAIError } from "../errors"; -import { VertexAI, VertexAIErrorCode } from "../public-types"; -import { VertexAIService } from "../service"; -import { ApiSettings } from "../types/internal"; +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAIError } from '../errors'; +import { VertexAI, VertexAIErrorCode } from '../public-types'; +import { VertexAIService } from '../service'; +import { ApiSettings } from '../types/internal'; /** * Base class for Vertex AI in Firebase model APIs. @@ -10,7 +27,7 @@ import { ApiSettings } from "../types/internal"; */ export class VertexAIModel { /** - * The fully qualified model resource name to use for generating images + * The fully qualified model resource name to use for generating images * (e.g. `publishers/google/models/imagen-3.0-generate-001`). */ readonly model: string; @@ -22,7 +39,7 @@ export class VertexAIModel { /** * Constructs a new instance of the {@link VertexAIModel} class. - * + * * This constructor should only be called from subclasses that provide * a model API. * @@ -30,17 +47,14 @@ export class VertexAIModel { * @param modelName - The name of the model being used. It can be in one of the following formats: * - `my-model` (short name, will resolve to `publishers/google/models/my-model`) * - `models/my-model` (will resolve to `publishers/google/models/my-model`) - * - `publishers/my-publisher/models/my-model` (fully qualified model name) - * + * - `publishers/my-publisher/models/my-model` (fully qualified model name) + * * @throws If the `apiKey` or `projectId` fields are missing in your * Firebase config. - * + * * @internal */ - protected constructor( - vertexAI: VertexAI, - modelName: string - ) { + protected constructor(vertexAI: VertexAI, modelName: string) { this.model = VertexAIModel.normalizeModelName(modelName); if (!vertexAI.app?.options?.apiKey) { @@ -73,7 +87,7 @@ export class VertexAIModel { /** * Normalizes the given model name to a fully qualified model resource name. - * + * * @param modelName - The model name to normalize. * @returns The fully qualified model resource name. */ @@ -94,4 +108,4 @@ export class VertexAIModel { return model; } -} \ No newline at end of file +} diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 1b7b9eaf6ac..8315947b732 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -1,8 +1,25 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** * Defines the image format for images output by Imagen. * - * Use this class to specify the desired format (JPEG or PNG) and compression quality - * for images generated by Imagen. This is typically included as part of + * Use this class to specify the desired format (JPEG or PNG) and compression quality + * for images generated by Imagen. This is typically included as part of * {@link ImagenModelParams}. * * @example @@ -26,7 +43,7 @@ export class ImagenImageFormat { compressionQuality?: number; private constructor() { - this.mimeType = "image/png"; + this.mimeType = 'image/png'; } /** @@ -38,7 +55,7 @@ export class ImagenImageFormat { * @public */ static jpeg(compressionQuality: number): ImagenImageFormat { - return { mimeType: "image/jpeg", compressionQuality }; + return { mimeType: 'image/jpeg', compressionQuality }; } /** @@ -49,6 +66,6 @@ export class ImagenImageFormat { * @public */ static png(): ImagenImageFormat { - return { mimeType: "image/png" }; + return { mimeType: 'image/png' }; } -} \ No newline at end of file +} diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 831bb6a26a5..49731f748b7 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -20,7 +20,7 @@ import { GenerateContentRequest, Part, VertexAIErrorCode, - ImagenAspectRatio, + ImagenAspectRatio } from '../types'; import { VertexAIError } from '../errors'; import { ImagenImageFormat } from './imagen-image-format'; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index ba7ee2c8f49..26a70201eea 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { ImagenGenerationConfig, ImagenModelConfig } from "./requests"; +import { ImagenGenerationConfig, ImagenModelConfig } from './requests'; /** * A response from the REST API is expected to look like this in the success case: @@ -130,4 +130,4 @@ export interface ImagenRequestConfig * (for GCS requests only). */ gcsURI?: string; -} \ No newline at end of file +} From 2f8b45de8d7a5f9e582eec850c1969d4619e03e0 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 3 Jan 2025 10:42:18 -0600 Subject: [PATCH 11/31] Fix ImagenRequestConfig comment --- packages/vertexai/src/types/imagen/internal.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 26a70201eea..3852d0eebf8 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -112,9 +112,9 @@ export interface PredictRequestBody { /** * Contains all possible REST API paramaters. - * This is the intersection of the model-level (`ImagenModelParams`), - * request-level (`ImagenGenerationConfig`) configurations, along with - * the other required parameters prompt and gcsURI (for GCS generation only). + * This is the union of model-level (`ImagenModelParams`), + * request-level (`ImagenGenerationConfig`) configurations, prompt, + * and gcsURI (for GCS generation only). * * @internal */ From 89a73de27dbc84e6ba209515425d7ecbbdb01c8d Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 3 Jan 2025 12:00:51 -0600 Subject: [PATCH 12/31] Improve tests --- .../vertexai/src/models/imagen-model.test.ts | 89 +++++++++++++------ 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index a0bc857a53b..731b775cfae 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -45,18 +45,18 @@ const fakeVertexAI: VertexAI = { describe('ImagenModel', () => { it('generateImages makes a request to predict with default parameters', async () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'my-model' - }); - const mockResponse = getMockResponse( 'unary-success-generate-images-base64.json' ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); + + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); const prompt = 'A photorealistic image of a toy boat at sea.'; - await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + await imagenModel.generateImages(prompt); expect(makeRequestStub).to.be.calledWith( 'publishers/google/models/my-model', request.Task.PREDICT, @@ -64,10 +64,10 @@ describe('ImagenModel', () => { false, match((value: string) => { return ( - value.includes('sampleCount') && - value.includes('includeRaiReason') && - value.includes('aspectRatio') && - value.includes('mimeType') + value.includes(`"prompt":"${prompt}"`) && + value.includes(`"mimeType":"image/png"`) && + value.includes(`"sampleCount":1`) && + value.includes(`"aspectRatio":"1:1"`) ); }), undefined @@ -100,14 +100,10 @@ describe('ImagenModel', () => { false, match((value: string) => { return ( - value.includes('safetyFilterLevel') && - value.includes('block_only_high') && - value.includes('personFilterLevel') && - value.includes('allow_adult') && - value.includes('mimeType') && - value.includes('image/jpeg') && - value.includes('addWatermark') && - value.includes('true') + value.includes(JSON.stringify(imagenModel.modelConfig.addWatermark)) && + value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.safetyFilterLevel)) && + value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.personFilterLevel)) && + value.includes(JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType)) ); }), undefined @@ -126,11 +122,53 @@ describe('ImagenModel', () => { mockResponse as Response ); const prompt = 'A photorealistic image of a toy boat at sea.'; - await imagenModel.generateImages(prompt, { + const requestConfig = { numberOfImages: 4, aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, negativePrompt: 'do not hallucinate' + }; + await imagenModel.generateImages(prompt, requestConfig); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes(`"sampleCount":${requestConfig.numberOfImages}`) && + value.includes(`"aspectRatio":"${requestConfig.aspectRatio}"`) && + value.includes(`"negativePrompt":"${requestConfig.negativePrompt}"`) + ); + }), + undefined + ); + + restore(); + }); + it('generateImages makes a request to predict with model-level and request-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model', + addWatermark: true, + imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + safetySettings: { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + } }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + const requestConfig = { + numberOfImages: 4, + aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, + negativePrompt: 'do not hallucinate' + }; + await imagenModel.generateImages(prompt, requestConfig); expect(makeRequestStub).to.be.calledWith( 'publishers/google/models/my-model', request.Task.PREDICT, @@ -138,12 +176,13 @@ describe('ImagenModel', () => { false, match((value: string) => { return ( - value.includes('sampleCount') && - value.includes('4') && - value.includes('aspectRatio') && - value.includes('16:9') && - value.includes('negativePrompt') && - value.includes('do not hallucinate') + value.includes(JSON.stringify(imagenModel.modelConfig.addWatermark)) && + value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.safetyFilterLevel)) && + value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.personFilterLevel)) && + value.includes(JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType)) && + value.includes(`"sampleCount":${requestConfig.numberOfImages}`) && + value.includes(`"aspectRatio":"${requestConfig.aspectRatio}"`) && + value.includes(`"negativePrompt":"${requestConfig.negativePrompt}"`) ); }), undefined @@ -158,7 +197,7 @@ describe('ImagenModel', () => { stub(globalThis, 'fetch').resolves({ ok: false, - status: 400, // TODO (dlarocque): Get this from the mock response + status: 400, statusText: 'Bad Request', json: mockResponse.json } as Response); From 80021f9407896d9fc83eb989d6df5c47afde9e20 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 3 Jan 2025 12:08:24 -0600 Subject: [PATCH 13/31] Formatting --- .../vertexai/src/models/imagen-model.test.ts | 40 +++++++++++++++---- .../vertexai/src/types/imagen/internal.ts | 4 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index 731b775cfae..caa9c5830f7 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -100,10 +100,22 @@ describe('ImagenModel', () => { false, match((value: string) => { return ( - value.includes(JSON.stringify(imagenModel.modelConfig.addWatermark)) && - value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.safetyFilterLevel)) && - value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.personFilterLevel)) && - value.includes(JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType)) + value.includes( + JSON.stringify(imagenModel.modelConfig.addWatermark) + ) && + value.includes( + JSON.stringify( + imagenModel.modelConfig.safetySettings?.safetyFilterLevel + ) + ) && + value.includes( + JSON.stringify( + imagenModel.modelConfig.safetySettings?.personFilterLevel + ) + ) && + value.includes( + JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType) + ) ); }), undefined @@ -176,10 +188,22 @@ describe('ImagenModel', () => { false, match((value: string) => { return ( - value.includes(JSON.stringify(imagenModel.modelConfig.addWatermark)) && - value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.safetyFilterLevel)) && - value.includes(JSON.stringify(imagenModel.modelConfig.safetySettings?.personFilterLevel)) && - value.includes(JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType)) && + value.includes( + JSON.stringify(imagenModel.modelConfig.addWatermark) + ) && + value.includes( + JSON.stringify( + imagenModel.modelConfig.safetySettings?.safetyFilterLevel + ) + ) && + value.includes( + JSON.stringify( + imagenModel.modelConfig.safetySettings?.personFilterLevel + ) + ) && + value.includes( + JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType) + ) && value.includes(`"sampleCount":${requestConfig.numberOfImages}`) && value.includes(`"aspectRatio":"${requestConfig.aspectRatio}"`) && value.includes(`"negativePrompt":"${requestConfig.negativePrompt}"`) diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 3852d0eebf8..95407a96d90 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -113,8 +113,8 @@ export interface PredictRequestBody { /** * Contains all possible REST API paramaters. * This is the union of model-level (`ImagenModelParams`), - * request-level (`ImagenGenerationConfig`) configurations, prompt, - * and gcsURI (for GCS generation only). + * request-level (`ImagenGenerationConfig`) configurations, prompt, + * and gcsURI (for GCS generation only). * * @internal */ From 858f260c0d5e5e95a9286e4fde9fb9839f35dfa4 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 9 Jan 2025 12:23:37 -0500 Subject: [PATCH 14/31] Add ImagenGenerationResponse --- common/api-review/vertexai.api.md | 22 ++++----- docs-devsite/_toc.yaml | 6 +-- ...d => vertexai.imagengenerationresponse.md} | 16 +++---- .../vertexai.imageninlineimageresponse.md | 46 ------------------- docs-devsite/vertexai.imagenmodel.md | 12 ++--- docs-devsite/vertexai.md | 3 +- packages/vertexai/src/models/imagen-model.ts | 13 +++--- .../vertexai/src/types/imagen/responses.ts | 28 ++--------- 8 files changed, 35 insertions(+), 111 deletions(-) rename docs-devsite/{vertexai.imagengcsimageresponse.md => vertexai.imagengenerationresponse.md} (50%) delete mode 100644 docs-devsite/vertexai.imageninlineimageresponse.md diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 8affc2a3922..97029c4b4d8 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -445,12 +445,6 @@ export interface ImagenGCSImage { mimeType: string; } -// @public -export interface ImagenGCSImageResponse { - filteredReason?: string; - images: ImagenGCSImage[]; -} - // @public export interface ImagenGenerationConfig { aspectRatio?: ImagenAspectRatio; @@ -458,6 +452,12 @@ export interface ImagenGenerationConfig { numberOfImages?: number; } +// @public +export interface ImagenGenerationResponse { + filteredReason?: string; + images: T[]; +} + // @public export class ImagenImageFormat { compressionQuality?: number; @@ -472,17 +472,11 @@ export interface ImagenInlineImage { mimeType: string; } -// @public -export interface ImagenInlineImageResponse { - filteredReason?: string; - images: ImagenInlineImage[]; -} - // @public export class ImagenModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); - generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; - generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; + generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; readonly modelConfig: ImagenModelConfig; // (undocumented) readonly requestOptions?: RequestOptions | undefined; diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 80444baaa1c..00ca7fe2686 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -530,16 +530,14 @@ toc: path: /docs/reference/js/vertexai.groundingmetadata.md - title: ImagenGCSImage path: /docs/reference/js/vertexai.imagengcsimage.md - - title: ImagenGCSImageResponse - path: /docs/reference/js/vertexai.imagengcsimageresponse.md - title: ImagenGenerationConfig path: /docs/reference/js/vertexai.imagengenerationconfig.md + - title: ImagenGenerationResponse + path: /docs/reference/js/vertexai.imagengenerationresponse.md - title: ImagenImageFormat path: /docs/reference/js/vertexai.imagenimageformat.md - title: ImagenInlineImage path: /docs/reference/js/vertexai.imageninlineimage.md - - title: ImagenInlineImageResponse - path: /docs/reference/js/vertexai.imageninlineimageresponse.md - title: ImagenModel path: /docs/reference/js/vertexai.imagenmodel.md - title: ImagenModelConfig diff --git a/docs-devsite/vertexai.imagengcsimageresponse.md b/docs-devsite/vertexai.imagengenerationresponse.md similarity index 50% rename from docs-devsite/vertexai.imagengcsimageresponse.md rename to docs-devsite/vertexai.imagengenerationresponse.md index cb1fa2cfe70..624698b5ff7 100644 --- a/docs-devsite/vertexai.imagengcsimageresponse.md +++ b/docs-devsite/vertexai.imagengenerationresponse.md @@ -9,23 +9,23 @@ overwritten. Changes should be made in the source code at https://github.com/firebase/firebase-js-sdk {% endcomment %} -# ImagenGCSImageResponse interface -The response from a request to generate images to Google Cloud Storage (GCS). +# ImagenGenerationResponse interface +The response from a request to generate images with Imagen. Signature: ```typescript -export interface ImagenGCSImageResponse +export interface ImagenGenerationResponse ``` ## Properties | Property | Type | Description | | --- | --- | --- | -| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | -| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. | +| [filteredReason](./vertexai.imagengenerationresponse.md#imagengenerationresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imagengenerationresponse.md#imagengenerationresponseimages) | T\[\] | The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. | -## ImagenGCSImageResponse.filteredReason +## ImagenGenerationResponse.filteredReason The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). @@ -35,12 +35,12 @@ The reason why any images were filtered. This field is only present if one or mo filteredReason?: string; ``` -## ImagenGCSImageResponse.images +## ImagenGenerationResponse.images The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. Signature: ```typescript -images: ImagenGCSImage[]; +images: T[]; ``` diff --git a/docs-devsite/vertexai.imageninlineimageresponse.md b/docs-devsite/vertexai.imageninlineimageresponse.md deleted file mode 100644 index c895dea6ea4..00000000000 --- a/docs-devsite/vertexai.imageninlineimageresponse.md +++ /dev/null @@ -1,46 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# ImagenInlineImageResponse interface -The response from a request to generate images to inline bytes. - -Signature: - -```typescript -export interface ImagenInlineImageResponse -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | -| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered out due to safety reasons, this array will be empty. | - -## ImagenInlineImageResponse.filteredReason - -The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). - -Signature: - -```typescript -filteredReason?: string; -``` - -## ImagenInlineImageResponse.images - -The images generated by Imagen. If all images were filtered out due to safety reasons, this array will be empty. - -Signature: - -```typescript -images: ImagenInlineImage[]; -``` diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md index 190b9e7307a..5b2345e8b8c 100644 --- a/docs-devsite/vertexai.imagenmodel.md +++ b/docs-devsite/vertexai.imagenmodel.md @@ -90,7 +90,7 @@ If one or more images are filtered, the returned object will have a defined `fil Signature: ```typescript -generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; ``` #### Parameters @@ -102,9 +102,9 @@ generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): P Returns: -Promise<[ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface)> +Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)>> -A promise that resolves to an [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) object containing the generated images. +A promise that resolves to an [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) object containing the generated images. #### Exceptions @@ -119,7 +119,7 @@ If one or more images are filtered due to safety reasons, the returned object wi Signature: ```typescript -generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; ``` #### Parameters @@ -132,9 +132,9 @@ generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenG Returns: -Promise<[ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface)> +Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)>> -A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the URLs of the generated images. +A promise that resolves to an [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) object containing the URLs of the generated images. #### Exceptions diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index 3814ccf42a4..2f28ccbc720 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -91,10 +91,9 @@ The Vertex AI in Firebase Web SDK. | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in Google Cloud Storage (GCS). | -| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | The response from a request to generate images to Google Cloud Storage (GCS). | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Request-level configuration options for generating images with Imagen. | +| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | The response from a request to generate images with Imagen. | | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | An image generated by Imagen, represented as inline bytes. | -| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | The response from a request to generate images to inline bytes. | | [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configuration options for Imagen. | | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for Imagen. | diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index 3434e1dd99d..f76323363a3 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -21,13 +21,12 @@ import { createPredictRequestBody } from '../requests/request-helpers'; import { handlePredictResponse } from '../requests/response-helpers'; import { ImagenGCSImage, - ImagenGCSImageResponse, ImagenGenerationConfig, ImagenInlineImage, RequestOptions, ImagenModelParams, - ImagenInlineImageResponse, - ImagenModelConfig + ImagenModelConfig, + ImagenGenerationResponse } from '../types'; import { VertexAIModel } from './vertexai-model'; @@ -84,7 +83,7 @@ export class ImagenModel extends VertexAIModel { * @param imagenRequestOptions - Configuration options for the Imagen * generation request. * See {@link ImagenGenerationConfig}. - * @returns A promise that resolves to an {@link ImagenInlineImageResponse} + * @returns A promise that resolves to an {@link ImagenGenerationResponse} * object containing the generated images. * * @throws If the request to generate images fails. This happens if the @@ -98,7 +97,7 @@ export class ImagenModel extends VertexAIModel { async generateImages( prompt: string, imagenRequestOptions?: ImagenGenerationConfig - ): Promise { + ): Promise> { const body = createPredictRequestBody({ prompt, ...imagenRequestOptions, @@ -123,7 +122,7 @@ export class ImagenModel extends VertexAIModel { * This should be a directory. For example, `gs://my-bucket/my-directory/`. * @param imagenRequestOptions - Configuration options for the Imagen * generation request. See {@link ImagenGenerationConfig}. - * @returns A promise that resolves to an {@link ImagenGCSImageResponse} + * @returns A promise that resolves to an {@link ImagenGenerationResponse} * object containing the URLs of the generated images. * * @throws If the request fails to generate images fails. This happens if @@ -138,7 +137,7 @@ export class ImagenModel extends VertexAIModel { prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig - ): Promise { + ): Promise> { const body = createPredictRequestBody({ prompt, gcsURI, diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index a05e3fda0f7..c363cc5dbfd 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -48,36 +48,16 @@ export interface ImagenGCSImage { } /** - * The response from a request to generate images to inline bytes. + * The response from a request to generate images with Imagen. * * @public */ -export interface ImagenInlineImageResponse { - /** - * The images generated by Imagen. - * If all images were filtered out due to safety reasons, this array will be empty. - */ - images: ImagenInlineImage[]; - /** - * The reason why any images were filtered. This field is only present if one - * or more images were filtered. - * For the mappings of error codes to reasons, see - * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. - */ - filteredReason?: string; -} - -/** - * The response from a request to generate images to Google Cloud Storage (GCS). - * - * @public - */ -export interface ImagenGCSImageResponse { +export interface ImagenGenerationResponse { /** * The images generated by Imagen. * If all images were filtered due to safety reasons, this array will be empty. */ - images: ImagenGCSImage[]; + images: T[]; /** * The reason why any images were filtered. This field is only present if one * or more images were filtered. @@ -85,4 +65,4 @@ export interface ImagenGCSImageResponse { * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. */ filteredReason?: string; -} + }; From bfc2c0f8376e26612c8d9b9174abc0e0b84947fe Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 9 Jan 2025 12:26:20 -0500 Subject: [PATCH 15/31] Formatting --- packages/vertexai/src/types/imagen/responses.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index c363cc5dbfd..b0d229e0246 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -52,7 +52,9 @@ export interface ImagenGCSImage { * * @public */ -export interface ImagenGenerationResponse { +export interface ImagenGenerationResponse< + T extends ImagenInlineImage | ImagenGCSImage +> { /** * The images generated by Imagen. * If all images were filtered due to safety reasons, this array will be empty. @@ -65,4 +67,4 @@ export interface ImagenGenerationResponse Date: Tue, 21 Jan 2025 13:38:52 -0500 Subject: [PATCH 16/31] Update to match API changes --- common/api-review/vertexai.api.md | 22 ++- docs-devsite/_toc.yaml | 2 - .../vertexai.imagengenerationconfig.md | 38 +++++- docs-devsite/vertexai.imagenimageformat.md | 2 +- docs-devsite/vertexai.imagenmodel.md | 46 ++++--- docs-devsite/vertexai.imagenmodelconfig.md | 57 -------- docs-devsite/vertexai.imagenmodelparams.md | 25 +++- docs-devsite/vertexai.md | 7 +- .../vertexai/src/models/imagen-model.test.ts | 125 +++--------------- packages/vertexai/src/models/imagen-model.ts | 67 +++++----- .../src/requests/imagen-image-format.ts | 2 +- .../src/requests/request-helpers.test.ts | 21 ++- .../vertexai/src/requests/request-helpers.ts | 44 +++--- .../vertexai/src/types/imagen/internal.ts | 32 ++--- .../vertexai/src/types/imagen/requests.ts | 45 +++---- 15 files changed, 218 insertions(+), 317 deletions(-) delete mode 100644 docs-devsite/vertexai.imagenmodelconfig.md diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 97029c4b4d8..05a3cd1aacd 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -447,7 +447,9 @@ export interface ImagenGCSImage { // @public export interface ImagenGenerationConfig { + addWatermark?: boolean; aspectRatio?: ImagenAspectRatio; + imageFormat?: ImagenImageFormat; negativePrompt?: string; numberOfImages?: number; } @@ -461,7 +463,7 @@ export interface ImagenGenerationResponse>; - generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; - readonly modelConfig: ImagenModelConfig; + generateImages(prompt: string): Promise>; + generateImagesGCS(prompt: string, gcsURI: string): Promise>; + readonly generationConfig?: ImagenGenerationConfig; // (undocumented) readonly requestOptions?: RequestOptions | undefined; + readonly safetySettings?: ImagenSafetySettings; } // @public -export interface ImagenModelConfig { - addWatermark?: boolean; - imageFormat?: ImagenImageFormat; - safetySettings?: ImagenSafetySettings; -} - -// @public -export interface ImagenModelParams extends ImagenModelConfig { +export interface ImagenModelParams { + generationConfig?: ImagenGenerationConfig; model: string; + safetySettings?: ImagenSafetySettings; } // @public diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 00ca7fe2686..1a18f8419b8 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -540,8 +540,6 @@ toc: path: /docs/reference/js/vertexai.imageninlineimage.md - title: ImagenModel path: /docs/reference/js/vertexai.imagenmodel.md - - title: ImagenModelConfig - path: /docs/reference/js/vertexai.imagenmodelconfig.md - title: ImagenModelParams path: /docs/reference/js/vertexai.imagenmodelparams.md - title: ImagenSafetySettings diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md index 9de48e45e81..d41260b2c07 100644 --- a/docs-devsite/vertexai.imagengenerationconfig.md +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -10,7 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationConfig interface -Request-level configuration options for generating images with Imagen. +Configuration options for generating images with Imagen. + +See the \[Google Cloud Docs\](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api\#rest\_1). Signature: @@ -22,13 +24,25 @@ export interface ImagenGenerationConfig | Property | Type | Description | | --- | --- | --- | -| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | The aspect ratio of the generated images. Defaults to 1:1. | -| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | A text prompt describing what should not be included in the image. | -| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | The number of images to generate. Must be between 1 and 4. Defaults to 1. | +| [addWatermark](./vertexai.imagengenerationconfig.md#imagengenerationconfigaddwatermark) | boolean | If true, adds a SynthID watermark to the generated images. | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | The aspect ratio of the generated images. The default value is 1:1. used. | +| [imageFormat](./vertexai.imagengenerationconfig.md#imagengenerationconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | The image format of the generated images. The default is PNG. | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | A description of what should be omitted from the generated images. | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | The number of images to generate. Must be between 1 and 4. The default value is 1. | + +## ImagenGenerationConfig.addWatermark + +If true, adds a SynthID watermark to the generated images. + +Signature: + +```typescript +addWatermark?: boolean; +``` ## ImagenGenerationConfig.aspectRatio -The aspect ratio of the generated images. Defaults to `1:1`. +The aspect ratio of the generated images. The default value is 1:1. used. Signature: @@ -36,9 +50,19 @@ The aspect ratio of the generated images. Defaults to `1:1`. aspectRatio?: ImagenAspectRatio; ``` +## ImagenGenerationConfig.imageFormat + +The image format of the generated images. The default is PNG. + +Signature: + +```typescript +imageFormat?: ImagenImageFormat; +``` + ## ImagenGenerationConfig.negativePrompt -A text prompt describing what should not be included in the image. +A description of what should be omitted from the generated images. Signature: @@ -48,7 +72,7 @@ negativePrompt?: string; ## ImagenGenerationConfig.numberOfImages -The number of images to generate. Must be between 1 and 4. Defaults to 1. +The number of images to generate. Must be between 1 and 4. The default value is 1. Signature: diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md index abdd561caa4..93d7f1d8fe7 100644 --- a/docs-devsite/vertexai.imagenimageformat.md +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -61,7 +61,7 @@ Creates an ImagenImageFormat for a JPEG image. Signature: ```typescript -static jpeg(compressionQuality: number): ImagenImageFormat; +static jpeg(compressionQuality?: number): ImagenImageFormat; ``` #### Parameters diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md index 5b2345e8b8c..b38684f7e12 100644 --- a/docs-devsite/vertexai.imagenmodel.md +++ b/docs-devsite/vertexai.imagenmodel.md @@ -31,15 +31,16 @@ export declare class ImagenModel extends VertexAIModel | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [modelConfig](./vertexai.imagenmodel.md#imagenmodelmodelconfig) | | [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configurations to use when using Imagen. | +| [generationConfig](./vertexai.imagenmodel.md#imagenmodelgenerationconfig) | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | The Imagen Generation Configuration. | | [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | +| [safetySettings](./vertexai.imagenmodel.md#imagenmodelsafetysettings) | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inappropriate content. | ## Methods | Method | Modifiers | Description | | --- | --- | --- | -| [generateImages(prompt, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | -| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images to Google Cloud Storage (GCS) using the Imagen model. | +| [generateImages(prompt)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | +| [generateImagesGCS(prompt, gcsURI)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images to Google Cloud Storage (GCS) using the Imagen model. | ## ImagenModel.(constructor) @@ -63,14 +64,14 @@ constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: If the `apiKey` or `projectId` fields are missing in your Firebase config. -## ImagenModel.modelConfig +## ImagenModel.generationConfig -Model-level configurations to use when using Imagen. +The Imagen Generation Configuration. Signature: ```typescript -readonly modelConfig: ImagenModelConfig; +readonly generationConfig?: ImagenGenerationConfig; ``` ## ImagenModel.requestOptions @@ -81,16 +82,26 @@ readonly modelConfig: ImagenModelConfig; readonly requestOptions?: RequestOptions | undefined; ``` +## ImagenModel.safetySettings + +Safety settings for filtering inappropriate content. + +Signature: + +```typescript +readonly safetySettings?: ImagenSafetySettings; +``` + ## ImagenModel.generateImages() Generates images using the Imagen model and returns them as base64-encoded strings. -If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. +If the prompt was not blocked, but one or more of the generated images were filtered, the returned object will have a `filteredReason` property. If all images are filtered, the `images` array will be empty. Signature: ```typescript -generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; +generateImages(prompt: string): Promise>; ``` #### Parameters @@ -98,7 +109,6 @@ generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): P | Parameter | Type | Description | | --- | --- | --- | | prompt | string | The text prompt used to generate the images. | -| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | Returns: @@ -114,12 +124,12 @@ If the request to generate images fails. This happens if the prompt is blocked. Generates images to Google Cloud Storage (GCS) using the Imagen model. -If one or more images are filtered due to safety reasons, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. +If the prompt was not blocked, but one or more of the generated images were filtered, the returned object will have a `filteredReason` property. If all images are filtered, the `images` array will be empty. Signature: ```typescript -generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise>; +generateImagesGCS(prompt: string, gcsURI: string): Promise>; ``` #### Parameters @@ -128,7 +138,6 @@ generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenG | --- | --- | --- | | prompt | string | The text prompt used to generate the images. | | gcsURI | string | The GCS URI where the images should be stored. This should be a directory. For example, gs://my-bucket/my-directory/. | -| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | Returns: @@ -144,12 +153,17 @@ If the request fails to generate images fails. This happens if the prompt is blo ```javascript -const imagen = new ImagenModel(vertexAI, { - model: 'imagen-3.0-generate-001' -}); +const imagen = new ImagenModel( + vertexAI, + { + model: 'imagen-3.0-generate-001' + } +); const response = await imagen.generateImages('A photo of a cat'); -console.log(response.images[0].bytesBase64Encoded); +if (response.images.length > 0) { + console.log(response.images[0].bytesBase64Encoded); +} ``` diff --git a/docs-devsite/vertexai.imagenmodelconfig.md b/docs-devsite/vertexai.imagenmodelconfig.md deleted file mode 100644 index e8948276eb6..00000000000 --- a/docs-devsite/vertexai.imagenmodelconfig.md +++ /dev/null @@ -1,57 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# ImagenModelConfig interface -Model-level configuration options for Imagen. - -Signature: - -```typescript -export interface ImagenModelConfig -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | Whether to add a watermark to generated images. | -| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | The image format of the generated images. | -| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inapropriate content. | - -## ImagenModelConfig.addWatermark - -Whether to add a watermark to generated images. - -Signature: - -```typescript -addWatermark?: boolean; -``` - -## ImagenModelConfig.imageFormat - -The image format of the generated images. - -Signature: - -```typescript -imageFormat?: ImagenImageFormat; -``` - -## ImagenModelConfig.safetySettings - -Safety settings for filtering inapropriate content. - -Signature: - -```typescript -safetySettings?: ImagenSafetySettings; -``` diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md index 256979e0498..1ad6fe03cd5 100644 --- a/docs-devsite/vertexai.imagenmodelparams.md +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -15,15 +15,26 @@ Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmode Signature: ```typescript -export interface ImagenModelParams extends ImagenModelConfig +export interface ImagenModelParams ``` -Extends: [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) ## Properties | Property | Type | Description | | --- | --- | --- | +| [generationConfig](./vertexai.imagenmodelparams.md#imagenmodelparamsgenerationconfig) | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | The Imagen Generation Configuration. | | [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | The Imagen model to use for generating images. For example: imagen-3.0-generate-001. | +| [safetySettings](./vertexai.imagenmodelparams.md#imagenmodelparamssafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inappropriate content. | + +## ImagenModelParams.generationConfig + +The Imagen Generation Configuration. + +Signature: + +```typescript +generationConfig?: ImagenGenerationConfig; +``` ## ImagenModelParams.model @@ -34,3 +45,13 @@ The Imagen model to use for generating images. For example: `imagen-3.0-generate ```typescript model: string; ``` + +## ImagenModelParams.safetySettings + +Safety settings for filtering inappropriate content. + +Signature: + +```typescript +safetySettings?: ImagenSafetySettings; +``` diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index 2f28ccbc720..f8016829c9f 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -54,7 +54,7 @@ The Vertex AI in Firebase Web SDK. | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | Aspect ratios for Imagen images. | | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Person filter levels for Imagen. | -| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter levels for Imagen. | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter levels for Imagen | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -91,10 +91,9 @@ The Vertex AI in Firebase Web SDK. | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in Google Cloud Storage (GCS). | -| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Request-level configuration options for generating images with Imagen. | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for generating images with Imagen.See the \[Google Cloud Docs\](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api\#rest\_1). | | [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | The response from a request to generate images with Imagen. | | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | An image generated by Imagen, represented as inline bytes. | -| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | Model-level configuration options for Imagen. | | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for Imagen. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | @@ -439,7 +438,7 @@ export declare enum ImagenPersonFilterLevel ## ImagenSafetyFilterLevel -Safety filter levels for Imagen. +Safety filter levels for Imagen Signature: diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index caa9c5830f7..b62c4a9e2ea 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -65,20 +65,23 @@ describe('ImagenModel', () => { match((value: string) => { return ( value.includes(`"prompt":"${prompt}"`) && - value.includes(`"mimeType":"image/png"`) && - value.includes(`"sampleCount":1`) && - value.includes(`"aspectRatio":"1:1"`) + value.includes(`"sampleCount":1`) ); }), undefined ); restore(); }); - it('generateImages makes a request to predict with model-level image configs', async () => { + it('generateImages makes a request to predict with generation config and safety settings', async () => { const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model', - addWatermark: true, - imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + generationConfig: { + negativePrompt: 'do not hallucinate', + numberOfImages: 4, + aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, + imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + addWatermark: true + }, safetySettings: { safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT @@ -92,7 +95,7 @@ describe('ImagenModel', () => { mockResponse as Response ); const prompt = 'A photorealistic image of a toy boat at sea.'; - await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + await imagenModel.generateImages(prompt); expect(makeRequestStub).to.be.calledWith( 'publishers/google/models/my-model', request.Task.PREDICT, @@ -101,117 +104,30 @@ describe('ImagenModel', () => { match((value: string) => { return ( value.includes( - JSON.stringify(imagenModel.modelConfig.addWatermark) + `"negativePrompt":"${imagenModel.generationConfig?.negativePrompt}"` ) && value.includes( - JSON.stringify( - imagenModel.modelConfig.safetySettings?.safetyFilterLevel - ) + `"sampleCount":${imagenModel.generationConfig?.numberOfImages}` ) && value.includes( - JSON.stringify( - imagenModel.modelConfig.safetySettings?.personFilterLevel - ) + `"aspectRatio":"${imagenModel.generationConfig?.aspectRatio}"` ) && value.includes( - JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType) - ) - ); - }), - undefined - ); - restore(); - }); - it('generateImages makes a request to predict with request-level image configs', async () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'my-model' - }); - - const mockResponse = getMockResponse( - 'unary-success-generate-images-base64.json' - ); - const makeRequestStub = stub(request, 'makeRequest').resolves( - mockResponse as Response - ); - const prompt = 'A photorealistic image of a toy boat at sea.'; - const requestConfig = { - numberOfImages: 4, - aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, - negativePrompt: 'do not hallucinate' - }; - await imagenModel.generateImages(prompt, requestConfig); - expect(makeRequestStub).to.be.calledWith( - 'publishers/google/models/my-model', - request.Task.PREDICT, - match.any, - false, - match((value: string) => { - return ( - value.includes(`"sampleCount":${requestConfig.numberOfImages}`) && - value.includes(`"aspectRatio":"${requestConfig.aspectRatio}"`) && - value.includes(`"negativePrompt":"${requestConfig.negativePrompt}"`) - ); - }), - undefined - ); - - restore(); - }); - it('generateImages makes a request to predict with model-level and request-level image configs', async () => { - const imagenModel = new ImagenModel(fakeVertexAI, { - model: 'my-model', - addWatermark: true, - imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, - safetySettings: { - safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, - personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT - } - }); - - const mockResponse = getMockResponse( - 'unary-success-generate-images-base64.json' - ); - const makeRequestStub = stub(request, 'makeRequest').resolves( - mockResponse as Response - ); - const prompt = 'A photorealistic image of a toy boat at sea.'; - const requestConfig = { - numberOfImages: 4, - aspectRatio: ImagenAspectRatio.LANDSCAPE_16x9, - negativePrompt: 'do not hallucinate' - }; - await imagenModel.generateImages(prompt, requestConfig); - expect(makeRequestStub).to.be.calledWith( - 'publishers/google/models/my-model', - request.Task.PREDICT, - match.any, - false, - match((value: string) => { - return ( - value.includes( - JSON.stringify(imagenModel.modelConfig.addWatermark) + JSON.stringify(imagenModel.generationConfig?.imageFormat?.mimeType) ) && value.includes( - JSON.stringify( - imagenModel.modelConfig.safetySettings?.safetyFilterLevel - ) + JSON.stringify(imagenModel.generationConfig?.addWatermark) ) && value.includes( - JSON.stringify( - imagenModel.modelConfig.safetySettings?.personFilterLevel - ) + JSON.stringify(imagenModel.safetySettings?.safetyFilterLevel) ) && value.includes( - JSON.stringify(imagenModel.modelConfig.imageFormat?.mimeType) - ) && - value.includes(`"sampleCount":${requestConfig.numberOfImages}`) && - value.includes(`"aspectRatio":"${requestConfig.aspectRatio}"`) && - value.includes(`"negativePrompt":"${requestConfig.negativePrompt}"`) + JSON.stringify(imagenModel.safetySettings?.personFilterLevel) + ) ); }), undefined ); - restore(); }); it('throws if prompt blocked', async () => { @@ -230,10 +146,7 @@ describe('ImagenModel', () => { model: 'my-model' }); try { - await imagenModel.generateImages( - 'A photorealistic image of a toy boat at sea.', - { numberOfImages: 1 } - ); + await imagenModel.generateImages('some inappropriate prompt.'); } catch (e) { expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); expect((e as VertexAIError).message).to.include('400'); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index f76323363a3..e5d384924e4 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -25,8 +25,8 @@ import { ImagenInlineImage, RequestOptions, ImagenModelParams, - ImagenModelConfig, - ImagenGenerationResponse + ImagenGenerationResponse, + ImagenSafetySettings } from '../types'; import { VertexAIModel } from './vertexai-model'; @@ -39,21 +39,30 @@ import { VertexAIModel } from './vertexai-model'; * * @example * ```javascript - * const imagen = new ImagenModel(vertexAI, { - * model: 'imagen-3.0-generate-001' - * }); + * const imagen = new ImagenModel( + * vertexAI, + * { + * model: 'imagen-3.0-generate-001' + * } + * ); * * const response = await imagen.generateImages('A photo of a cat'); - * console.log(response.images[0].bytesBase64Encoded); + * if (response.images.length > 0) { + * console.log(response.images[0].bytesBase64Encoded); + * } * ``` * * @public */ export class ImagenModel extends VertexAIModel { /** - * Model-level configurations to use when using Imagen. + * The Imagen Generation Configuration. */ - readonly modelConfig: ImagenModelConfig; + readonly generationConfig?: ImagenGenerationConfig; + /** + * Safety settings for filtering inappropriate content. + */ + readonly safetySettings?: ImagenSafetySettings; /** * Constructs a new instance of the {@link ImagenModel} class. @@ -70,9 +79,10 @@ export class ImagenModel extends VertexAIModel { modelParams: ImagenModelParams, readonly requestOptions?: RequestOptions ) { - const { model, ...modelConfig } = modelParams; + const { model, generationConfig, safetySettings } = modelParams; super(vertexAI, model); - this.modelConfig = modelConfig; + this.generationConfig = generationConfig; + this.safetySettings = safetySettings; } /** @@ -80,9 +90,6 @@ export class ImagenModel extends VertexAIModel { * base64-encoded strings. * * @param prompt - The text prompt used to generate the images. - * @param imagenRequestOptions - Configuration options for the Imagen - * generation request. - * See {@link ImagenGenerationConfig}. * @returns A promise that resolves to an {@link ImagenGenerationResponse} * object containing the generated images. * @@ -90,18 +97,16 @@ export class ImagenModel extends VertexAIModel { * prompt is blocked. * * @remarks - * If one or more images are filtered, the returned object will have a - * defined `filteredReason` property. If all images are filtered, the - * `images` array will be empty, and no error will be thrown. + * If the prompt was not blocked, but one or more of the generated images were filtered, the + * returned object will have a `filteredReason` property. + * If all images are filtered, the `images` array will be empty. */ async generateImages( - prompt: string, - imagenRequestOptions?: ImagenGenerationConfig + prompt: string ): Promise> { - const body = createPredictRequestBody({ - prompt, - ...imagenRequestOptions, - ...this.modelConfig + const body = createPredictRequestBody(prompt, { + ...this.generationConfig, + ...this.safetySettings }); const response = await makeRequest( this.model, @@ -120,8 +125,6 @@ export class ImagenModel extends VertexAIModel { * @param prompt - The text prompt used to generate the images. * @param gcsURI - The GCS URI where the images should be stored. * This should be a directory. For example, `gs://my-bucket/my-directory/`. - * @param imagenRequestOptions - Configuration options for the Imagen - * generation request. See {@link ImagenGenerationConfig}. * @returns A promise that resolves to an {@link ImagenGenerationResponse} * object containing the URLs of the generated images. * @@ -129,20 +132,18 @@ export class ImagenModel extends VertexAIModel { * the prompt is blocked. * * @remarks - * If one or more images are filtered due to safety reasons, the returned object - * will have a defined `filteredReason` property. If all images are filtered, - * the `images` array will be empty, and no error will be thrown. + * If the prompt was not blocked, but one or more of the generated images were filtered, the + * returned object will have a `filteredReason` property. + * If all images are filtered, the `images` array will be empty. */ async generateImagesGCS( prompt: string, - gcsURI: string, - imagenRequestOptions?: ImagenGenerationConfig + gcsURI: string ): Promise> { - const body = createPredictRequestBody({ - prompt, + const body = createPredictRequestBody(prompt, { gcsURI, - ...imagenRequestOptions, - ...this.modelConfig + ...this.generationConfig, + ...this.safetySettings }); const response = await makeRequest( this.model, diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 8315947b732..df7fb9f0576 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -54,7 +54,7 @@ export class ImagenImageFormat { * * @public */ - static jpeg(compressionQuality: number): ImagenImageFormat { + static jpeg(compressionQuality?: number): ImagenImageFormat { return { mimeType: 'image/jpeg', compressionQuality }; } diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 2317457aca4..2fd5ffb3d4b 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -210,20 +210,18 @@ describe('request formatting methods', () => { describe('createPredictRequestBody', () => { it('creates body with default request parameters', () => { const prompt = 'A photorealistic image of a toy boat at sea.'; - const body = createPredictRequestBody({ - prompt - }); + const body = createPredictRequestBody(prompt, {}); expect(body.instances[0].prompt).to.equal(prompt); expect(body.parameters.sampleCount).to.equal(1); - expect(body.parameters.mimeType).to.equal('image/png'); expect(body.parameters.includeRaiReason).to.be.true; - expect(body.parameters.aspectRatio).to.equal('1:1'); // Parameters without default values should be undefined expect(body.parameters.storageUri).to.be.undefined; expect(body.parameters.compressionQuality).to.be.undefined; expect(body.parameters.negativePrompt).to.be.undefined; expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.mimeType).to.be.undefined; + expect(body.parameters.aspectRatio).to.be.undefined; expect(body.parameters.addWatermark).to.be.undefined; expect(body.parameters.safetyFilterLevel).to.be.undefined; expect(body.parameters.personGeneration).to.be.undefined; @@ -240,16 +238,16 @@ describe('request formatting methods', () => { const numberOfImages = 4; const negativePrompt = 'do not hallucinate'; const aspectRatio = ImagenAspectRatio.LANDSCAPE_16x9; - const body = createPredictRequestBody({ - prompt, + const body = createPredictRequestBody(prompt, { numberOfImages, imageFormat, - safetySettings, addWatermark, negativePrompt, - aspectRatio + aspectRatio, + ...safetySettings }); expect(body.instances[0].prompt).to.equal(prompt); + console.log(body); expect(body.parameters).deep.equal({ sampleCount: numberOfImages, mimeType: imageFormat.mimeType, @@ -257,7 +255,7 @@ describe('request formatting methods', () => { addWatermark, negativePrompt, safetyFilterLevel: safetySettings.safetyFilterLevel, - personFilterLevel: safetySettings.personFilterLevel, + personGeneration: safetySettings.personFilterLevel, aspectRatio, includeRaiReason: true, storageUri: undefined @@ -266,8 +264,7 @@ describe('request formatting methods', () => { it('creates body with GCS URI', () => { const prompt = 'A photorealistic image of a toy boat at sea.'; const gcsURI = 'gcs-uri'; - const body = createPredictRequestBody({ - prompt, + const body = createPredictRequestBody(prompt, { gcsURI }); diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 49731f748b7..4c03dbeefdf 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -19,12 +19,10 @@ import { Content, GenerateContentRequest, Part, - VertexAIErrorCode, - ImagenAspectRatio + VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; -import { ImagenImageFormat } from './imagen-image-format'; -import { ImagenRequestConfig, PredictRequestBody } from '../types/internal'; +import { ImagenGenerationParams, PredictRequestBody } from '../types/internal'; export function formatSystemInstruction( input?: string | Part | Content @@ -128,22 +126,25 @@ export function formatGenerateContentInput( } /** - * Convert the user-defined parameters in {@link ImagenRequestConfig} to the format + * Convert the user-defined parameters in {@link ImagenGenerationParams} to the format * that is expected from the REST API. * * @internal */ -export function createPredictRequestBody({ - prompt, - gcsURI, - imageFormat = ImagenImageFormat.png(), - addWatermark, - safetySettings, - numberOfImages = 1, - negativePrompt, - aspectRatio = ImagenAspectRatio.SQUARE -}: ImagenRequestConfig): PredictRequestBody { - // Properties that are undefined will be omitted from the JSON string. +export function createPredictRequestBody( + prompt: string, + { + gcsURI, + imageFormat, + addWatermark, + numberOfImages = 1, + negativePrompt, + aspectRatio, + safetyFilterLevel, + personFilterLevel + }: ImagenGenerationParams +): PredictRequestBody { + // Properties that are undefined will be omitted from the JSON string that is sent in the request. const body: PredictRequestBody = { instances: [ { @@ -152,13 +153,14 @@ export function createPredictRequestBody({ ], parameters: { storageUri: gcsURI, + negativePrompt, + sampleCount: numberOfImages, + aspectRatio, ...imageFormat, addWatermark, - ...safetySettings, - sampleCount: numberOfImages, - includeRaiReason: true, - negativePrompt, - aspectRatio + safetyFilterLevel, + personGeneration: personFilterLevel, + includeRaiReason: true } }; return body; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 95407a96d90..3abc2b2713e 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { ImagenGenerationConfig, ImagenModelConfig } from './requests'; +import { ImagenGenerationConfig, ImagenSafetySettings } from './requests'; /** * A response from the REST API is expected to look like this in the success case: @@ -51,9 +51,10 @@ export interface ImagenResponseInternal { * The image data encoded as a base64 string. */ bytesBase64Encoded?: string; - + /** + * The GCS URI where the image was stored. + */ gcsUri?: string; - /** * The reason why the image was filtered. */ @@ -68,9 +69,6 @@ export interface ImagenResponseInternal { * We need a seperate internal-only interface for this because the REST * API expects different parameter names than what we show to our users. * - * This interface should be populated from the ImagenGenerationConfig that - * the user defines. - * * Sample request body JSON: * { * "instances": [ @@ -88,6 +86,8 @@ export interface ImagenResponseInternal { * } * } * + * See the Google Cloud docs: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#-drest + * * @internal */ export interface PredictRequestBody { @@ -98,8 +98,8 @@ export interface PredictRequestBody { ]; parameters: { sampleCount: number; // Maps to numberOfImages - aspectRatio: string; - mimeType: string; + aspectRatio?: string; + mimeType?: string; compressionQuality?: number; negativePrompt?: string; storageUri?: string; // Maps to gcsURI @@ -111,23 +111,15 @@ export interface PredictRequestBody { } /** - * Contains all possible REST API paramaters. - * This is the union of model-level (`ImagenModelParams`), - * request-level (`ImagenGenerationConfig`) configurations, prompt, - * and gcsURI (for GCS generation only). + * Contains all possible REST API paramaters that are provided by the caller. * * @internal */ -export interface ImagenRequestConfig - extends ImagenModelConfig, - ImagenGenerationConfig { - /** - * The text prompt used to generate the images. - */ - prompt: string; +export type ImagenGenerationParams = { /** * The Google Cloud Storage (GCS) URI where the images should be stored * (for GCS requests only). */ gcsURI?: string; -} +} & ImagenGenerationConfig & + ImagenSafetySettings; diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index 99b69ad09b8..c33f73b9424 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -22,56 +22,55 @@ import { ImagenImageFormat } from '../../requests/imagen-image-format'; * * @public */ -export interface ImagenModelParams extends ImagenModelConfig { +export interface ImagenModelParams { /** * The Imagen model to use for generating images. * For example: `imagen-3.0-generate-001`. */ model: string; -} - -/** - * Model-level configuration options for Imagen. - * - * @public - */ -export interface ImagenModelConfig { /** - * The image format of the generated images. + * The Imagen Generation Configuration. */ - imageFormat?: ImagenImageFormat; + generationConfig?: ImagenGenerationConfig; /** - * Whether to add a watermark to generated images. - */ - addWatermark?: boolean; - /** - * Safety settings for filtering inapropriate content. + * Safety settings for filtering inappropriate content. */ safetySettings?: ImagenSafetySettings; } /** - * Request-level configuration options for generating images with Imagen. + * Configuration options for generating images with Imagen. + * + * See the [Google Cloud Docs](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#rest_1). * * @public */ export interface ImagenGenerationConfig { /** - * The number of images to generate. Must be between 1 and 4. Defaults to 1. + * A description of what should be omitted from the generated images. */ - numberOfImages?: number; + negativePrompt?: string; /** - * A text prompt describing what should not be included in the image. + * The number of images to generate. Must be between 1 and 4. The default value is 1. */ - negativePrompt?: string; + numberOfImages?: number; /** - * The aspect ratio of the generated images. Defaults to `1:1`. + * The aspect ratio of the generated images. The default value is 1:1. + * used. */ aspectRatio?: ImagenAspectRatio; + /** + * The image format of the generated images. The default is PNG. + */ + imageFormat?: ImagenImageFormat; + /** + * If true, adds a SynthID watermark to the generated images. + */ + addWatermark?: boolean; } /** - * Safety filter levels for Imagen. + * Safety filter levels for Imagen * * @public */ From d967900fa5b3be208aec39954dcd74282ebaec17 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 3 Feb 2025 14:55:51 -0500 Subject: [PATCH 17/31] fixes --- packages/vertexai/src/models/imagen-model.test.ts | 2 +- packages/vertexai/src/models/imagen-model.ts | 2 +- packages/vertexai/src/models/vertexai-model.test.ts | 2 +- packages/vertexai/src/models/vertexai-model.ts | 2 +- packages/vertexai/src/requests/response-helpers.ts | 5 +++-- packages/vertexai/src/types/imagen/index.ts | 2 +- packages/vertexai/src/types/imagen/internal.ts | 2 +- packages/vertexai/src/types/imagen/requests.ts | 2 +- packages/vertexai/src/types/imagen/responses.ts | 2 +- 9 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts index b62c4a9e2ea..000b2f07f90 100644 --- a/packages/vertexai/src/models/imagen-model.test.ts +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index e5d384924e4..22d3421dbf4 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/vertexai/src/models/vertexai-model.test.ts b/packages/vertexai/src/models/vertexai-model.test.ts index d570230a64c..2aa36d56f0d 100644 --- a/packages/vertexai/src/models/vertexai-model.test.ts +++ b/packages/vertexai/src/models/vertexai-model.test.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index 3b9f09bb370..23a8466f32b 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -26,7 +26,7 @@ import { _isFirebaseServerApp } from '@firebase/app'; * * @public */ -export class VertexAIModel { +export abstract class VertexAIModel { /** * The fully qualified model resource name to use for generating images * (e.g. `publishers/google/models/imagen-3.0-generate-001`). diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index acdc6e741d1..dd6a71f5b6c 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -216,10 +216,11 @@ export async function handlePredictResponse< const images: T[] = []; let filteredReason: string | undefined = undefined; + // The backend should always send a non-empty array of predictions if the response was successful. if (!responseJson.predictions || responseJson.predictions?.length === 0) { throw new VertexAIError( - VertexAIErrorCode.ERROR, - "Predictions array is undefined or empty in response. Was 'includeRaiReason' enabled in the request?" + VertexAIErrorCode.RESPONSE_ERROR, + "No predictions or filtered reason received from Vertex AI. Please report this issue with the full error details at https://github.com/firebase/firebase-js-sdk/issues." ); } diff --git a/packages/vertexai/src/types/imagen/index.ts b/packages/vertexai/src/types/imagen/index.ts index 0037b185e50..546c64f13b1 100644 --- a/packages/vertexai/src/types/imagen/index.ts +++ b/packages/vertexai/src/types/imagen/index.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 3abc2b2713e..ec37646af9a 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index c33f73b9424..0df0ac12202 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index b0d229e0246..dcd43706f90 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -1,6 +1,6 @@ /** * @license - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From d53d3dd171865b67b9e8171ece505b3ecd73a084 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 3 Feb 2025 15:02:51 -0500 Subject: [PATCH 18/31] format --- packages/vertexai/src/requests/response-helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index dd6a71f5b6c..95bcfea20fc 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -220,7 +220,7 @@ export async function handlePredictResponse< if (!responseJson.predictions || responseJson.predictions?.length === 0) { throw new VertexAIError( VertexAIErrorCode.RESPONSE_ERROR, - "No predictions or filtered reason received from Vertex AI. Please report this issue with the full error details at https://github.com/firebase/firebase-js-sdk/issues." + 'No predictions or filtered reason received from Vertex AI. Please report this issue with the full error details at https://github.com/firebase/firebase-js-sdk/issues.' ); } From ae0f87e5a6ed3e1cf26a805603122cdb74fc9508 Mon Sep 17 00:00:00 2001 From: dlarocque Date: Mon, 3 Feb 2025 20:14:10 +0000 Subject: [PATCH 19/31] Update API reports --- common/api-review/vertexai.api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 05a3cd1aacd..65231050b9d 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -803,7 +803,7 @@ export const enum VertexAIErrorCode { } // @public -export class VertexAIModel { +export abstract class VertexAIModel { // @internal protected constructor(vertexAI: VertexAI, modelName: string); // @internal (undocumented) From 37aa71ab573f85b2f165e3c36f1d5b7232593cbe Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 3 Feb 2025 15:43:48 -0500 Subject: [PATCH 20/31] add changeset --- .changeset/violet-planets-impress.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/violet-planets-impress.md diff --git a/.changeset/violet-planets-impress.md b/.changeset/violet-planets-impress.md new file mode 100644 index 00000000000..8f5964d5572 --- /dev/null +++ b/.changeset/violet-planets-impress.md @@ -0,0 +1,5 @@ +--- +'@firebase/vertexai': minor +--- + +Add Image Generation support using Imagen. From ece898f8b99c64705f9dbd54625bd5a524b89f4b Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 6 Feb 2025 11:50:09 -0500 Subject: [PATCH 21/31] update --- .changeset/violet-planets-impress.md | 1 + packages/vertexai/src/api.ts | 4 ++++ packages/vertexai/src/models/imagen-model.ts | 14 +++++++++++--- .../src/requests/imagen-image-format.ts | 18 +++++++++++++----- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.changeset/violet-planets-impress.md b/.changeset/violet-planets-impress.md index 8f5964d5572..a7f09acd1a2 100644 --- a/.changeset/violet-planets-impress.md +++ b/.changeset/violet-planets-impress.md @@ -1,4 +1,5 @@ --- +'firebase': minor '@firebase/vertexai': minor --- diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 6ad61798c4f..e7f00584b23 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -84,6 +84,10 @@ export function getGenerativeModel( /** * Returns a {@link ImagenModel} class with methods for using Imagen. + * + * @beta + * This feature is in public preview and is not intended for production use. + * The API is subject to change. * * @public */ diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index 22d3421dbf4..a42be018ed7 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -58,11 +58,11 @@ export class ImagenModel extends VertexAIModel { /** * The Imagen Generation Configuration. */ - readonly generationConfig?: ImagenGenerationConfig; + generationConfig?: ImagenGenerationConfig; /** * Safety settings for filtering inappropriate content. */ - readonly safetySettings?: ImagenSafetySettings; + safetySettings?: ImagenSafetySettings; /** * Constructs a new instance of the {@link ImagenModel} class. @@ -77,7 +77,7 @@ export class ImagenModel extends VertexAIModel { constructor( vertexAI: VertexAI, modelParams: ImagenModelParams, - readonly requestOptions?: RequestOptions + public requestOptions?: RequestOptions ) { const { model, generationConfig, safetySettings } = modelParams; super(vertexAI, model); @@ -88,6 +88,10 @@ export class ImagenModel extends VertexAIModel { /** * Generates images using the Imagen model and returns them as * base64-encoded strings. + * + * @beta + * This feature is in public preview and is not intended for production use. + * The API is subject to change. * * @param prompt - The text prompt used to generate the images. * @returns A promise that resolves to an {@link ImagenGenerationResponse} @@ -121,6 +125,10 @@ export class ImagenModel extends VertexAIModel { /** * Generates images to Google Cloud Storage (GCS) using the Imagen model. + * + * @beta + * This feature is in public preview and is not intended for production use. + * The API is subject to change. * * @param prompt - The text prompt used to generate the images. * @param gcsURI - The GCS URI where the images should be stored. diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index df7fb9f0576..a44f3f4e3ed 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -20,7 +20,7 @@ * * Use this class to specify the desired format (JPEG or PNG) and compression quality * for images generated by Imagen. This is typically included as part of - * {@link ImagenModelParams}. + * {@link ImagenModelParams}. * * @example * ```javascript @@ -47,10 +47,14 @@ export class ImagenImageFormat { } /** - * Creates an ImagenImageFormat for a JPEG image. + * Creates an {@link ImagenImageFormat} for a JPEG image. + * + * @beta + * This feature is in public preview and is not intended for production use. + * The API is subject to change. * * @param compressionQuality - The level of compression (a number between 0 and 100). - * @returns ImagenImageFormat + * @returns An {@link ImagenImageFormat} object for a JPEG image. * * @public */ @@ -59,9 +63,13 @@ export class ImagenImageFormat { } /** - * Creates an ImageImageFormat for a PNG image. + * Creates an {@link ImagenImageFormat} for a PNG image. + * + * @beta + * This feature is in public preview and is not intended for production use. + * The API is subject to change. * - * @returns ImageImageFormat + * @returns An {@link ImagenImageFormat} object for a PNG image. * * @public */ From 967cd786262e35ead0d7ff097de091ad0542dfc9 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 6 Feb 2025 11:53:02 -0500 Subject: [PATCH 22/31] format --- packages/vertexai/src/api.ts | 2 +- packages/vertexai/src/models/imagen-model.ts | 4 ++-- packages/vertexai/src/requests/imagen-image-format.ts | 4 ++-- packages/vertexai/src/types/imagen/internal.ts | 2 ++ 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index e7f00584b23..cb1e6c2437d 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -84,7 +84,7 @@ export function getGenerativeModel( /** * Returns a {@link ImagenModel} class with methods for using Imagen. - * + * * @beta * This feature is in public preview and is not intended for production use. * The API is subject to change. diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index a42be018ed7..0668ddcdf3b 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -88,7 +88,7 @@ export class ImagenModel extends VertexAIModel { /** * Generates images using the Imagen model and returns them as * base64-encoded strings. - * + * * @beta * This feature is in public preview and is not intended for production use. * The API is subject to change. @@ -125,7 +125,7 @@ export class ImagenModel extends VertexAIModel { /** * Generates images to Google Cloud Storage (GCS) using the Imagen model. - * + * * @beta * This feature is in public preview and is not intended for production use. * The API is subject to change. diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index a44f3f4e3ed..63ac1e49c98 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -48,7 +48,7 @@ export class ImagenImageFormat { /** * Creates an {@link ImagenImageFormat} for a JPEG image. - * + * * @beta * This feature is in public preview and is not intended for production use. * The API is subject to change. @@ -64,7 +64,7 @@ export class ImagenImageFormat { /** * Creates an {@link ImagenImageFormat} for a PNG image. - * + * * @beta * This feature is in public preview and is not intended for production use. * The API is subject to change. diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index ec37646af9a..950a36b294e 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -40,6 +40,8 @@ import { ImagenGenerationConfig, ImagenSafetySettings } from './requests'; * } * ] * } + * + * @internal */ export interface ImagenResponseInternal { predictions?: Array<{ From b27a45e4fcae63676e62be9847a2c7944c46278f Mon Sep 17 00:00:00 2001 From: dlarocque Date: Thu, 6 Feb 2025 17:01:23 +0000 Subject: [PATCH 23/31] Update API reports --- common/api-review/vertexai.api.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 65231050b9d..bce0b106229 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -477,12 +477,14 @@ export interface ImagenInlineImage { // @public export class ImagenModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); + // @beta generateImages(prompt: string): Promise>; + // @beta generateImagesGCS(prompt: string, gcsURI: string): Promise>; - readonly generationConfig?: ImagenGenerationConfig; + generationConfig?: ImagenGenerationConfig; // (undocumented) - readonly requestOptions?: RequestOptions | undefined; - readonly safetySettings?: ImagenSafetySettings; + requestOptions?: RequestOptions | undefined; + safetySettings?: ImagenSafetySettings; } // @public From f0955630d319b05fd85af39bb8d149caf11481d5 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 7 Feb 2025 12:15:50 -0500 Subject: [PATCH 24/31] remove public docs --- common/api-review/vertexai.api.md | 61 ++++++++--- docs-devsite/vertexai.imagengcsimage.md | 10 +- .../vertexai.imagengenerationconfig.md | 24 +---- .../vertexai.imagengenerationresponse.md | 10 +- docs-devsite/vertexai.imagenimageformat.md | 41 ++----- docs-devsite/vertexai.imageninlineimage.md | 16 +-- docs-devsite/vertexai.imagenmodel.md | 80 +++----------- docs-devsite/vertexai.imagenmodelparams.md | 14 +-- docs-devsite/vertexai.imagensafetysettings.md | 10 +- docs-devsite/vertexai.md | 58 +++++----- docs-devsite/vertexai.vertexaimodel.md | 8 +- packages/vertexai/src/api.ts | 9 -- packages/vertexai/src/models/imagen-model.ts | 81 -------------- .../vertexai/src/models/vertexai-model.ts | 9 -- .../src/requests/imagen-image-format.ts | 46 -------- .../vertexai/src/types/imagen/requests.ts | 100 ------------------ .../vertexai/src/types/imagen/responses.ts | 37 ------- 17 files changed, 124 insertions(+), 490 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index bce0b106229..a1cd9cba066 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -346,7 +346,7 @@ export class GenerativeModel extends VertexAIModel { // @public export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; -// @public +// @public (undocumented) export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; // @public @@ -430,88 +430,122 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } -// @public +// @public (undocumented) export enum ImagenAspectRatio { + // (undocumented) LANDSCAPE_16x9 = "16:9", + // (undocumented) LANDSCAPE_3x4 = "3:4", + // (undocumented) PORTRAIT_4x3 = "4:3", + // (undocumented) PORTRAIT_9x16 = "9:16", + // (undocumented) SQUARE = "1:1" } -// @public +// @public (undocumented) export interface ImagenGCSImage { + // (undocumented) gcsURI: string; + // (undocumented) mimeType: string; } -// @public +// @public (undocumented) export interface ImagenGenerationConfig { + // (undocumented) addWatermark?: boolean; + // (undocumented) aspectRatio?: ImagenAspectRatio; + // (undocumented) imageFormat?: ImagenImageFormat; + // (undocumented) negativePrompt?: string; + // (undocumented) numberOfImages?: number; } -// @public +// @public (undocumented) export interface ImagenGenerationResponse { + // (undocumented) filteredReason?: string; + // (undocumented) images: T[]; } // @public export class ImagenImageFormat { + // (undocumented) compressionQuality?: number; + // (undocumented) static jpeg(compressionQuality?: number): ImagenImageFormat; + // (undocumented) mimeType: string; + // (undocumented) static png(): ImagenImageFormat; } // @public export interface ImagenInlineImage { + // (undocumented) bytesBase64Encoded: string; + // (undocumented) mimeType: string; } -// @public +// @public (undocumented) export class ImagenModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); - // @beta + // (undocumented) generateImages(prompt: string): Promise>; - // @beta + // (undocumented) generateImagesGCS(prompt: string, gcsURI: string): Promise>; + // (undocumented) generationConfig?: ImagenGenerationConfig; // (undocumented) requestOptions?: RequestOptions | undefined; + // (undocumented) safetySettings?: ImagenSafetySettings; } -// @public +// @public (undocumented) export interface ImagenModelParams { + // (undocumented) generationConfig?: ImagenGenerationConfig; + // (undocumented) model: string; + // (undocumented) safetySettings?: ImagenSafetySettings; } -// @public +// @public (undocumented) export enum ImagenPersonFilterLevel { + // (undocumented) ALLOW_ADULT = "allow_adult", + // (undocumented) ALLOW_ALL = "allow_all", + // (undocumented) BLOCK_ALL = "dont_allow" } -// @public +// @public (undocumented) export enum ImagenSafetyFilterLevel { + // (undocumented) BLOCK_LOW_AND_ABOVE = "block_low_and_above", + // (undocumented) BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", + // (undocumented) BLOCK_NONE = "block_none", + // (undocumented) BLOCK_ONLY_HIGH = "block_only_high" } -// @public +// @public (undocumented) export interface ImagenSafetySettings { + // (undocumented) personFilterLevel?: ImagenPersonFilterLevel; + // (undocumented) safetyFilterLevel?: ImagenSafetyFilterLevel; } @@ -804,12 +838,13 @@ export const enum VertexAIErrorCode { RESPONSE_ERROR = "response-error" } -// @public +// @public (undocumented) export abstract class VertexAIModel { // @internal protected constructor(vertexAI: VertexAI, modelName: string); // @internal (undocumented) protected _apiSettings: ApiSettings; + // (undocumented) readonly model: string; static normalizeModelName(modelName: string): string; } diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md index af9887ba9dd..af4717129f7 100644 --- a/docs-devsite/vertexai.imagengcsimage.md +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -10,8 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImage interface -An image generated by Imagen, stored in Google Cloud Storage (GCS). - Signature: ```typescript @@ -22,13 +20,11 @@ export interface ImagenGCSImage | Property | Type | Description | | --- | --- | --- | -| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI where the image is stored. | -| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image. | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | | +| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | | ## ImagenGCSImage.gcsURI -The Google Cloud Storage (GCS) URI where the image is stored. - Signature: ```typescript @@ -37,8 +33,6 @@ gcsURI: string; ## ImagenGCSImage.mimeType -The MIME type of the image. - Signature: ```typescript diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md index d41260b2c07..4b2e91f01a7 100644 --- a/docs-devsite/vertexai.imagengenerationconfig.md +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -10,10 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationConfig interface -Configuration options for generating images with Imagen. - -See the \[Google Cloud Docs\](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api\#rest\_1). - Signature: ```typescript @@ -24,16 +20,14 @@ export interface ImagenGenerationConfig | Property | Type | Description | | --- | --- | --- | -| [addWatermark](./vertexai.imagengenerationconfig.md#imagengenerationconfigaddwatermark) | boolean | If true, adds a SynthID watermark to the generated images. | -| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | The aspect ratio of the generated images. The default value is 1:1. used. | -| [imageFormat](./vertexai.imagengenerationconfig.md#imagengenerationconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | The image format of the generated images. The default is PNG. | -| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | A description of what should be omitted from the generated images. | -| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | The number of images to generate. Must be between 1 and 4. The default value is 1. | +| [addWatermark](./vertexai.imagengenerationconfig.md#imagengenerationconfigaddwatermark) | boolean | | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [imageFormat](./vertexai.imagengenerationconfig.md#imagengenerationconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | ## ImagenGenerationConfig.addWatermark -If true, adds a SynthID watermark to the generated images. - Signature: ```typescript @@ -42,8 +36,6 @@ addWatermark?: boolean; ## ImagenGenerationConfig.aspectRatio -The aspect ratio of the generated images. The default value is 1:1. used. - Signature: ```typescript @@ -52,8 +44,6 @@ aspectRatio?: ImagenAspectRatio; ## ImagenGenerationConfig.imageFormat -The image format of the generated images. The default is PNG. - Signature: ```typescript @@ -62,8 +52,6 @@ imageFormat?: ImagenImageFormat; ## ImagenGenerationConfig.negativePrompt -A description of what should be omitted from the generated images. - Signature: ```typescript @@ -72,8 +60,6 @@ negativePrompt?: string; ## ImagenGenerationConfig.numberOfImages -The number of images to generate. Must be between 1 and 4. The default value is 1. - Signature: ```typescript diff --git a/docs-devsite/vertexai.imagengenerationresponse.md b/docs-devsite/vertexai.imagengenerationresponse.md index 624698b5ff7..320e2b68533 100644 --- a/docs-devsite/vertexai.imagengenerationresponse.md +++ b/docs-devsite/vertexai.imagengenerationresponse.md @@ -10,8 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationResponse interface -The response from a request to generate images with Imagen. - Signature: ```typescript @@ -22,13 +20,11 @@ export interface ImagenGenerationResponse. | -| [images](./vertexai.imagengenerationresponse.md#imagengenerationresponseimages) | T\[\] | The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. | +| [filteredReason](./vertexai.imagengenerationresponse.md#imagengenerationresponsefilteredreason) | string | | +| [images](./vertexai.imagengenerationresponse.md#imagengenerationresponseimages) | T\[\] | | ## ImagenGenerationResponse.filteredReason -The reason why any images were filtered. This field is only present if one or more images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). - Signature: ```typescript @@ -37,8 +33,6 @@ filteredReason?: string; ## ImagenGenerationResponse.images -The images generated by Imagen. If all images were filtered due to safety reasons, this array will be empty. - Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md index 93d7f1d8fe7..f52af54d9f4 100644 --- a/docs-devsite/vertexai.imagenimageformat.md +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -10,9 +10,13 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenImageFormat class -Defines the image format for images output by Imagen. + Copyright 2025 Google LLC -Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Signature: @@ -24,20 +28,18 @@ export declare class ImagenImageFormat | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | The level of compression (a number between 0 and 100). | -| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | The MIME type. | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | | ## Methods | Method | Modifiers | Description | | --- | --- | --- | -| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | Creates an ImagenImageFormat for a JPEG image. | -| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | Creates an ImageImageFormat for a PNG image. | +| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | | +| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | | ## ImagenImageFormat.compressionQuality -The level of compression (a number between 0 and 100). - Signature: ```typescript @@ -46,8 +48,6 @@ compressionQuality?: number; ## ImagenImageFormat.mimeType -The MIME type. - Signature: ```typescript @@ -56,8 +56,6 @@ mimeType: string; ## ImagenImageFormat.jpeg() -Creates an ImagenImageFormat for a JPEG image. - Signature: ```typescript @@ -68,18 +66,14 @@ static jpeg(compressionQuality?: number): ImagenImageFormat; | Parameter | Type | Description | | --- | --- | --- | -| compressionQuality | number | The level of compression (a number between 0 and 100). | +| compressionQuality | number | | Returns: [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) -ImagenImageFormat - ## ImagenImageFormat.png() -Creates an ImageImageFormat for a PNG image. - Signature: ```typescript @@ -89,16 +83,3 @@ static png(): ImagenImageFormat; [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) -ImageImageFormat - -### Example - - -```javascript -const imagenModelParams = { - // ... other ImagenModelParams - imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. -} - -``` - diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md index 6d3a3972445..b1b2fb3e52c 100644 --- a/docs-devsite/vertexai.imageninlineimage.md +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -10,7 +10,13 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenInlineImage interface -An image generated by Imagen, represented as inline bytes. + Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Signature: @@ -22,13 +28,11 @@ export interface ImagenInlineImage | Property | Type | Description | | --- | --- | --- | -| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | The image data encoded as a base64 string. | -| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | The MIME type of the image. | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | +| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | | ## ImagenInlineImage.bytesBase64Encoded -The image data encoded as a base64 string. - Signature: ```typescript @@ -37,8 +41,6 @@ bytesBase64Encoded: string; ## ImagenInlineImage.mimeType -The MIME type of the image. - Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md index b38684f7e12..c2730170180 100644 --- a/docs-devsite/vertexai.imagenmodel.md +++ b/docs-devsite/vertexai.imagenmodel.md @@ -10,10 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModel class -Class for Imagen model APIs. - -This class provides methods for generating images using the Imagen model. You can generate images inline as base64-encoded strings, or directly to Google Cloud Storage (GCS). - Signature: ```typescript @@ -25,26 +21,26 @@ export declare class ImagenModel extends VertexAIModel | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [generationConfig](./vertexai.imagenmodel.md#imagenmodelgenerationconfig) | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | The Imagen Generation Configuration. | +| [generationConfig](./vertexai.imagenmodel.md#imagenmodelgenerationconfig) | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | | [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | -| [safetySettings](./vertexai.imagenmodel.md#imagenmodelsafetysettings) | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inappropriate content. | +| [safetySettings](./vertexai.imagenmodel.md#imagenmodelsafetysettings) | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | ## Methods | Method | Modifiers | Description | | --- | --- | --- | -| [generateImages(prompt)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | -| [generateImagesGCS(prompt, gcsURI)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images to Google Cloud Storage (GCS) using the Imagen model. | +| [generateImages(prompt)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | | +| [generateImagesGCS(prompt, gcsURI)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | | ## ImagenModel.(constructor) -Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. +Constructs a new instance of the `ImagenModel` class Signature: @@ -56,22 +52,16 @@ constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: | Parameter | Type | Description | | --- | --- | --- | -| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | -| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making Imagen requests. | -| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | Additional options to use when making requests. | - -#### Exceptions - -If the `apiKey` or `projectId` fields are missing in your Firebase config. +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | ## ImagenModel.generationConfig -The Imagen Generation Configuration. - Signature: ```typescript -readonly generationConfig?: ImagenGenerationConfig; +generationConfig?: ImagenGenerationConfig; ``` ## ImagenModel.requestOptions @@ -79,25 +69,19 @@ readonly generationConfig?: ImagenGenerationConfig; Signature: ```typescript -readonly requestOptions?: RequestOptions | undefined; +requestOptions?: RequestOptions | undefined; ``` ## ImagenModel.safetySettings -Safety settings for filtering inappropriate content. - Signature: ```typescript -readonly safetySettings?: ImagenSafetySettings; +safetySettings?: ImagenSafetySettings; ``` ## ImagenModel.generateImages() -Generates images using the Imagen model and returns them as base64-encoded strings. - -If the prompt was not blocked, but one or more of the generated images were filtered, the returned object will have a `filteredReason` property. If all images are filtered, the `images` array will be empty. - Signature: ```typescript @@ -108,24 +92,14 @@ generateImages(prompt: string): PromiseReturns: Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)>> -A promise that resolves to an [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) object containing the generated images. - -#### Exceptions - -If the request to generate images fails. This happens if the prompt is blocked. - ## ImagenModel.generateImagesGCS() -Generates images to Google Cloud Storage (GCS) using the Imagen model. - -If the prompt was not blocked, but one or more of the generated images were filtered, the returned object will have a `filteredReason` property. If all images are filtered, the `images` array will be empty. - Signature: ```typescript @@ -136,34 +110,10 @@ generateImagesGCS(prompt: string, gcsURI: string): Promisegs://my-bucket/my-directory/. | +| prompt | string | | +| gcsURI | string | | Returns: Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)>> -A promise that resolves to an [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) object containing the URLs of the generated images. - -#### Exceptions - -If the request fails to generate images fails. This happens if the prompt is blocked. - -### Example - - -```javascript -const imagen = new ImagenModel( - vertexAI, - { - model: 'imagen-3.0-generate-001' - } -); - -const response = await imagen.generateImages('A photo of a cat'); -if (response.images.length > 0) { - console.log(response.images[0].bytesBase64Encoded); -} - -``` - diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md index 1ad6fe03cd5..21dbd533006 100644 --- a/docs-devsite/vertexai.imagenmodelparams.md +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -10,8 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModelParams interface -Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). - Signature: ```typescript @@ -22,14 +20,12 @@ export interface ImagenModelParams | Property | Type | Description | | --- | --- | --- | -| [generationConfig](./vertexai.imagenmodelparams.md#imagenmodelparamsgenerationconfig) | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | The Imagen Generation Configuration. | -| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | The Imagen model to use for generating images. For example: imagen-3.0-generate-001. | -| [safetySettings](./vertexai.imagenmodelparams.md#imagenmodelparamssafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for filtering inappropriate content. | +| [generationConfig](./vertexai.imagenmodelparams.md#imagenmodelparamsgenerationconfig) | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | +| [safetySettings](./vertexai.imagenmodelparams.md#imagenmodelparamssafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | ## ImagenModelParams.generationConfig -The Imagen Generation Configuration. - Signature: ```typescript @@ -38,8 +34,6 @@ generationConfig?: ImagenGenerationConfig; ## ImagenModelParams.model -The Imagen model to use for generating images. For example: `imagen-3.0-generate-001`. - Signature: ```typescript @@ -48,8 +42,6 @@ model: string; ## ImagenModelParams.safetySettings -Safety settings for filtering inappropriate content. - Signature: ```typescript diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md index a544b9688bc..0f5ea97c3e4 100644 --- a/docs-devsite/vertexai.imagensafetysettings.md +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -10,8 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenSafetySettings interface -Safety settings for Imagen. - Signature: ```typescript @@ -22,13 +20,11 @@ export interface ImagenSafetySettings | Property | Type | Description | | --- | --- | --- | -| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | The person filter level to use. | -| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | The safety filter level to use. | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | ## ImagenSafetySettings.personFilterLevel -The person filter level to use. - Signature: ```typescript @@ -37,8 +33,6 @@ personFilterLevel?: ImagenPersonFilterLevel; ## ImagenSafetySettings.safetyFilterLevel -The safety filter level to use. - Signature: ```typescript diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index f8016829c9f..efc0cb92402 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -20,7 +20,7 @@ The Vertex AI in Firebase Web SDK. | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | -| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | | ## Classes @@ -30,15 +30,15 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | -| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | Defines the image format for images output by Imagen.Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | -| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs.This class provides methods for generating images using the Imagen model. You can generate images inline as base64-encoded strings, or directly to Google Cloud Storage (GCS). | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | Copyright 2025 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | | [Schema](./vertexai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with JSON.stringify() into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) | | [StringSchema](./vertexai.stringschema.md#stringschema_class) | Schema class for "string" types. Can be used with or without enum values. | | [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) | Error class for the Vertex AI in Firebase SDK. | -| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | Base class for Vertex AI in Firebase model APIs. | +| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | | ## Enumerations @@ -52,9 +52,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | -| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | Aspect ratios for Imagen images. | -| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Person filter levels for Imagen. | -| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter levels for Imagen | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -90,12 +90,12 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | -| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in Google Cloud Storage (GCS). | -| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for generating images with Imagen.See the \[Google Cloud Docs\](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api\#rest\_1). | -| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | The response from a request to generate images with Imagen. | -| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | An image generated by Imagen, represented as inline bytes. | -| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | -| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | Safety settings for Imagen. | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | +| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Copyright 2025 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | @@ -182,8 +182,6 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode ### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} -Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. - Signature: ```typescript @@ -400,8 +398,6 @@ export declare enum HarmSeverity ## ImagenAspectRatio -Aspect ratios for Imagen images. - Signature: ```typescript @@ -412,16 +408,14 @@ export declare enum ImagenAspectRatio | Member | Value | Description | | --- | --- | --- | -| LANDSCAPE\_16x9 | "16:9" | Landscape 16:9 aspect ratio. | -| LANDSCAPE\_3x4 | "3:4" | Landscape 3:4 aspect ratio. | -| PORTRAIT\_4x3 | "4:3" | Portrait 4:3 aspect ratio. | -| PORTRAIT\_9x16 | "9:16" | Portrait 9:16 aspect ratio. | -| SQUARE | "1:1" | Square 1:1 aspect ratio. | +| LANDSCAPE\_16x9 | "16:9" | | +| LANDSCAPE\_3x4 | "3:4" | | +| PORTRAIT\_4x3 | "4:3" | | +| PORTRAIT\_9x16 | "9:16" | | +| SQUARE | "1:1" | | ## ImagenPersonFilterLevel -Person filter levels for Imagen. - Signature: ```typescript @@ -432,14 +426,12 @@ export declare enum ImagenPersonFilterLevel | Member | Value | Description | | --- | --- | --- | -| ALLOW\_ADULT | "allow_adult" | Allow only adults in generated images. | -| ALLOW\_ALL | "allow_all" | Allow all person generation. | -| BLOCK\_ALL | "dont_allow" | Do not allow any person generation. | +| ALLOW\_ADULT | "allow_adult" | | +| ALLOW\_ALL | "allow_all" | | +| BLOCK\_ALL | "dont_allow" | | ## ImagenSafetyFilterLevel -Safety filter levels for Imagen - Signature: ```typescript @@ -450,10 +442,10 @@ export declare enum ImagenSafetyFilterLevel | Member | Value | Description | | --- | --- | --- | -| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | Block images with low or higher safety severity. | -| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | Block images with medium or higher safety severity. | -| BLOCK\_NONE | "block_none" | Do not block any images based on safety. | -| BLOCK\_ONLY\_HIGH | "block_only_high" | Block images with high safety severity. | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | +| BLOCK\_NONE | "block_none" | | +| BLOCK\_ONLY\_HIGH | "block_only_high" | | ## SchemaType diff --git a/docs-devsite/vertexai.vertexaimodel.md b/docs-devsite/vertexai.vertexaimodel.md index ad35e68dded..31d7e1d9c32 100644 --- a/docs-devsite/vertexai.vertexaimodel.md +++ b/docs-devsite/vertexai.vertexaimodel.md @@ -10,21 +10,19 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # VertexAIModel class -Base class for Vertex AI in Firebase model APIs. - The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `VertexAIModel` class. Signature: ```typescript -export declare class VertexAIModel +export declare abstract class VertexAIModel ``` ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | The fully qualified model resource name to use for generating images (e.g. publishers/google/models/imagen-3.0-generate-001). | +| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | | ## Methods @@ -34,8 +32,6 @@ export declare class VertexAIModel ## VertexAIModel.model -The fully qualified model resource name to use for generating images (e.g. `publishers/google/models/imagen-3.0-generate-001`). - Signature: ```typescript diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index cb1e6c2437d..8b7477f306d 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -82,15 +82,6 @@ export function getGenerativeModel( return new GenerativeModel(vertexAI, modelParams, requestOptions); } -/** - * Returns a {@link ImagenModel} class with methods for using Imagen. - * - * @beta - * This feature is in public preview and is not intended for production use. - * The API is subject to change. - * - * @public - */ export function getImagenModel( vertexAI: VertexAI, modelParams: ImagenModelParams, diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index 0668ddcdf3b..8d9ab93b6ba 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -30,50 +30,10 @@ import { } from '../types'; import { VertexAIModel } from './vertexai-model'; -/** - * Class for Imagen model APIs. - * - * This class provides methods for generating images using the Imagen model. - * You can generate images inline as base64-encoded strings, or directly to - * Google Cloud Storage (GCS). - * - * @example - * ```javascript - * const imagen = new ImagenModel( - * vertexAI, - * { - * model: 'imagen-3.0-generate-001' - * } - * ); - * - * const response = await imagen.generateImages('A photo of a cat'); - * if (response.images.length > 0) { - * console.log(response.images[0].bytesBase64Encoded); - * } - * ``` - * - * @public - */ export class ImagenModel extends VertexAIModel { - /** - * The Imagen Generation Configuration. - */ generationConfig?: ImagenGenerationConfig; - /** - * Safety settings for filtering inappropriate content. - */ safetySettings?: ImagenSafetySettings; - /** - * Constructs a new instance of the {@link ImagenModel} class. - * - * @param vertexAI - An instance of the Vertex AI in Firebase SDK. - * @param modelParams - Parameters to use when making Imagen requests. - * @param requestOptions - Additional options to use when making requests. - * - * @throws If the `apiKey` or `projectId` fields are missing in your - * Firebase config. - */ constructor( vertexAI: VertexAI, modelParams: ImagenModelParams, @@ -85,26 +45,6 @@ export class ImagenModel extends VertexAIModel { this.safetySettings = safetySettings; } - /** - * Generates images using the Imagen model and returns them as - * base64-encoded strings. - * - * @beta - * This feature is in public preview and is not intended for production use. - * The API is subject to change. - * - * @param prompt - The text prompt used to generate the images. - * @returns A promise that resolves to an {@link ImagenGenerationResponse} - * object containing the generated images. - * - * @throws If the request to generate images fails. This happens if the - * prompt is blocked. - * - * @remarks - * If the prompt was not blocked, but one or more of the generated images were filtered, the - * returned object will have a `filteredReason` property. - * If all images are filtered, the `images` array will be empty. - */ async generateImages( prompt: string ): Promise> { @@ -123,27 +63,6 @@ export class ImagenModel extends VertexAIModel { return handlePredictResponse(response); } - /** - * Generates images to Google Cloud Storage (GCS) using the Imagen model. - * - * @beta - * This feature is in public preview and is not intended for production use. - * The API is subject to change. - * - * @param prompt - The text prompt used to generate the images. - * @param gcsURI - The GCS URI where the images should be stored. - * This should be a directory. For example, `gs://my-bucket/my-directory/`. - * @returns A promise that resolves to an {@link ImagenGenerationResponse} - * object containing the URLs of the generated images. - * - * @throws If the request fails to generate images fails. This happens if - * the prompt is blocked. - * - * @remarks - * If the prompt was not blocked, but one or more of the generated images were filtered, the - * returned object will have a `filteredReason` property. - * If all images are filtered, the `images` array will be empty. - */ async generateImagesGCS( prompt: string, gcsURI: string diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index 23a8466f32b..5bed60d6279 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -21,16 +21,7 @@ import { VertexAIService } from '../service'; import { ApiSettings } from '../types/internal'; import { _isFirebaseServerApp } from '@firebase/app'; -/** - * Base class for Vertex AI in Firebase model APIs. - * - * @public - */ export abstract class VertexAIModel { - /** - * The fully qualified model resource name to use for generating images - * (e.g. `publishers/google/models/imagen-3.0-generate-001`). - */ readonly model: string; /** diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 63ac1e49c98..faf08b424ab 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -15,64 +15,18 @@ * limitations under the License. */ -/** - * Defines the image format for images output by Imagen. - * - * Use this class to specify the desired format (JPEG or PNG) and compression quality - * for images generated by Imagen. This is typically included as part of - * {@link ImagenModelParams}. - * - * @example - * ```javascript - * const imagenModelParams = { - * // ... other ImagenModelParams - * imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. - * } - * ``` - * - * @public - */ export class ImagenImageFormat { - /** - * The MIME type. - */ mimeType: string; - /** - * The level of compression (a number between 0 and 100). - */ compressionQuality?: number; private constructor() { this.mimeType = 'image/png'; } - /** - * Creates an {@link ImagenImageFormat} for a JPEG image. - * - * @beta - * This feature is in public preview and is not intended for production use. - * The API is subject to change. - * - * @param compressionQuality - The level of compression (a number between 0 and 100). - * @returns An {@link ImagenImageFormat} object for a JPEG image. - * - * @public - */ static jpeg(compressionQuality?: number): ImagenImageFormat { return { mimeType: 'image/jpeg', compressionQuality }; } - /** - * Creates an {@link ImagenImageFormat} for a PNG image. - * - * @beta - * This feature is in public preview and is not intended for production use. - * The API is subject to change. - * - * @returns An {@link ImagenImageFormat} object for a PNG image. - * - * @public - */ static png(): ImagenImageFormat { return { mimeType: 'image/png' }; } diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index 0df0ac12202..cf72391dc41 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -17,142 +17,42 @@ import { ImagenImageFormat } from '../../requests/imagen-image-format'; -/** - * Parameters for configuring an {@link ImagenModel}. - * - * @public - */ export interface ImagenModelParams { - /** - * The Imagen model to use for generating images. - * For example: `imagen-3.0-generate-001`. - */ model: string; - /** - * The Imagen Generation Configuration. - */ generationConfig?: ImagenGenerationConfig; - /** - * Safety settings for filtering inappropriate content. - */ safetySettings?: ImagenSafetySettings; } -/** - * Configuration options for generating images with Imagen. - * - * See the [Google Cloud Docs](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#rest_1). - * - * @public - */ export interface ImagenGenerationConfig { - /** - * A description of what should be omitted from the generated images. - */ negativePrompt?: string; - /** - * The number of images to generate. Must be between 1 and 4. The default value is 1. - */ numberOfImages?: number; - /** - * The aspect ratio of the generated images. The default value is 1:1. - * used. - */ aspectRatio?: ImagenAspectRatio; - /** - * The image format of the generated images. The default is PNG. - */ imageFormat?: ImagenImageFormat; - /** - * If true, adds a SynthID watermark to the generated images. - */ addWatermark?: boolean; } -/** - * Safety filter levels for Imagen - * - * @public - */ export enum ImagenSafetyFilterLevel { - /** - * Block images with low or higher safety severity. - */ BLOCK_LOW_AND_ABOVE = 'block_low_and_above', - /** - * Block images with medium or higher safety severity. - */ BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', - /** - * Block images with high safety severity. - */ BLOCK_ONLY_HIGH = 'block_only_high', - /** - * Do not block any images based on safety. - */ BLOCK_NONE = 'block_none' } -/** - * Person filter levels for Imagen. - * - * @public - */ export enum ImagenPersonFilterLevel { - /** - * Do not allow any person generation. - */ BLOCK_ALL = 'dont_allow', - /** - * Allow only adults in generated images. - */ ALLOW_ADULT = 'allow_adult', - /** - * Allow all person generation. - */ ALLOW_ALL = 'allow_all' } -/** - * Safety settings for Imagen. - * - * @public - */ export interface ImagenSafetySettings { - /** - * The safety filter level to use. - */ safetyFilterLevel?: ImagenSafetyFilterLevel; - /** - * The person filter level to use. - */ personFilterLevel?: ImagenPersonFilterLevel; } -/** - * Aspect ratios for Imagen images. - * - * @public - */ export enum ImagenAspectRatio { - /** - * Square 1:1 aspect ratio. - */ SQUARE = '1:1', - /** - * Landscape 3:4 aspect ratio. - */ LANDSCAPE_3x4 = '3:4', - /** - * Portrait 4:3 aspect ratio. - */ PORTRAIT_4x3 = '4:3', - /** - * Landscape 16:9 aspect ratio. - */ LANDSCAPE_16x9 = '16:9', - /** - * Portrait 9:16 aspect ratio. - */ PORTRAIT_9x16 = '9:16' } diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index dcd43706f90..514acc601b1 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -15,56 +15,19 @@ * limitations under the License. */ -/** - * An image generated by Imagen, represented as inline bytes. - * - * @public - */ export interface ImagenInlineImage { - /** - * The MIME type of the image. - */ mimeType: string; - /** - * The image data encoded as a base64 string. - */ bytesBase64Encoded: string; } -/** - * An image generated by Imagen, stored in Google Cloud Storage (GCS). - * - * @public - */ export interface ImagenGCSImage { - /** - * The MIME type of the image. - */ mimeType: string; - /** - * The Google Cloud Storage (GCS) URI where the image is stored. - */ gcsURI: string; } -/** - * The response from a request to generate images with Imagen. - * - * @public - */ export interface ImagenGenerationResponse< T extends ImagenInlineImage | ImagenGCSImage > { - /** - * The images generated by Imagen. - * If all images were filtered due to safety reasons, this array will be empty. - */ images: T[]; - /** - * The reason why any images were filtered. This field is only present if one - * or more images were filtered. - * For the mappings of error codes to reasons, see - * {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. - */ filteredReason?: string; } From d6a4f24023aca4ac4cf0462ff7309227515f1f09 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 10 Feb 2025 15:53:18 -0500 Subject: [PATCH 25/31] Nest imageFormat in outputOptions --- .../vertexai/src/requests/request-helpers.test.ts | 11 ++++++----- packages/vertexai/src/requests/request-helpers.ts | 2 +- packages/vertexai/src/types/imagen/internal.ts | 6 ++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 2fd5ffb3d4b..4c704b8722d 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -217,10 +217,9 @@ describe('request formatting methods', () => { // Parameters without default values should be undefined expect(body.parameters.storageUri).to.be.undefined; - expect(body.parameters.compressionQuality).to.be.undefined; - expect(body.parameters.negativePrompt).to.be.undefined; expect(body.parameters.storageUri).to.be.undefined; - expect(body.parameters.mimeType).to.be.undefined; + expect(body.parameters.outputOptions).to.be.undefined; + expect(body.parameters.negativePrompt).to.be.undefined; expect(body.parameters.aspectRatio).to.be.undefined; expect(body.parameters.addWatermark).to.be.undefined; expect(body.parameters.safetyFilterLevel).to.be.undefined; @@ -250,8 +249,10 @@ describe('request formatting methods', () => { console.log(body); expect(body.parameters).deep.equal({ sampleCount: numberOfImages, - mimeType: imageFormat.mimeType, - compressionQuality: imageFormat.compressionQuality, + outputOptions: { + mimeType: imageFormat.mimeType, + compressionQuality: imageFormat.compressionQuality + }, addWatermark, negativePrompt, safetyFilterLevel: safetySettings.safetyFilterLevel, diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 4c03dbeefdf..33d2fab266a 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -156,7 +156,7 @@ export function createPredictRequestBody( negativePrompt, sampleCount: numberOfImages, aspectRatio, - ...imageFormat, + outputOptions: imageFormat, addWatermark, safetyFilterLevel, personGeneration: personFilterLevel, diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index 950a36b294e..fb81e634617 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -101,8 +101,10 @@ export interface PredictRequestBody { parameters: { sampleCount: number; // Maps to numberOfImages aspectRatio?: string; - mimeType?: string; - compressionQuality?: number; + outputOptions?: { + mimeType: string; + compressionQuality?: number; + }; negativePrompt?: string; storageUri?: string; // Maps to gcsURI addWatermark?: boolean; From 66b3b9f7e050b130bccc23f7c971797508a8010c Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 10 Feb 2025 15:56:34 -0500 Subject: [PATCH 26/31] remove unecessary console.log --- packages/vertexai/src/requests/request-helpers.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 4c704b8722d..d8337850925 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -246,7 +246,6 @@ describe('request formatting methods', () => { ...safetySettings }); expect(body.instances[0].prompt).to.equal(prompt); - console.log(body); expect(body.parameters).deep.equal({ sampleCount: numberOfImages, outputOptions: { From 48b658d31f54bdfec01187d7828db9f3445af000 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 11 Feb 2025 15:29:58 -0500 Subject: [PATCH 27/31] Log warning if compressionQuality outside of range --- packages/vertexai/src/requests/imagen-image-format.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index faf08b424ab..9778d3da3e4 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -15,6 +15,8 @@ * limitations under the License. */ +import { logger } from "../logger"; + export class ImagenImageFormat { mimeType: string; compressionQuality?: number; @@ -24,6 +26,9 @@ export class ImagenImageFormat { } static jpeg(compressionQuality?: number): ImagenImageFormat { + if (compressionQuality && (compressionQuality < 0 || compressionQuality > 100)) { + logger.warn(`Invalid JPEG compression quality of ${compressionQuality} specified; the supported range is [0, 100].`); + } return { mimeType: 'image/jpeg', compressionQuality }; } From c3173a9cf3beb1c188c017dd03a6a996c6798519 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 11 Feb 2025 15:31:48 -0500 Subject: [PATCH 28/31] format --- packages/vertexai/src/requests/imagen-image-format.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index 9778d3da3e4..e6dbab041a6 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { logger } from "../logger"; +import { logger } from '../logger'; export class ImagenImageFormat { mimeType: string; @@ -26,8 +26,13 @@ export class ImagenImageFormat { } static jpeg(compressionQuality?: number): ImagenImageFormat { - if (compressionQuality && (compressionQuality < 0 || compressionQuality > 100)) { - logger.warn(`Invalid JPEG compression quality of ${compressionQuality} specified; the supported range is [0, 100].`); + if ( + compressionQuality && + (compressionQuality < 0 || compressionQuality > 100) + ) { + logger.warn( + `Invalid JPEG compression quality of ${compressionQuality} specified; the supported range is [0, 100].` + ); } return { mimeType: 'image/jpeg', compressionQuality }; } From 0900f8208e500e38194fd7ae49324cc7d845df50 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 20 Feb 2025 09:51:44 -0500 Subject: [PATCH 29/31] Imagen Documentation (#8776) * Revert "remove public docs" This reverts commit f0955630d319b05fd85af39bb8d149caf11481d5. * update docs * update docs * format * minor fixes * format * mark all APIs beta instead of public * review fixes * format * Replace (BETA) with (Public Preview) in docs * fix links * another pass * format * review fixes --- common/api-review/vertexai.api.md | 64 ++------ docs-devsite/vertexai.imagengcsimage.md | 18 +- .../vertexai.imagengenerationconfig.md | 54 +++++- .../vertexai.imagengenerationresponse.md | 25 ++- docs-devsite/vertexai.imagenimageformat.md | 54 ++++-- docs-devsite/vertexai.imageninlineimage.md | 25 ++- docs-devsite/vertexai.imagenmodel.md | 84 +++++++--- docs-devsite/vertexai.imagenmodelparams.md | 30 +++- docs-devsite/vertexai.imagensafetysettings.md | 21 ++- docs-devsite/vertexai.md | 90 ++++++---- docs-devsite/vertexai.vertexaimodel.md | 6 +- packages/vertexai/src/api.ts | 14 ++ packages/vertexai/src/models/imagen-model.ts | 75 +++++++++ .../vertexai/src/models/vertexai-model.ts | 9 + .../src/requests/imagen-image-format.ts | 38 +++++ .../vertexai/src/requests/request-helpers.ts | 2 +- .../vertexai/src/types/imagen/internal.ts | 2 +- .../vertexai/src/types/imagen/requests.ts | 154 ++++++++++++++++++ .../vertexai/src/types/imagen/responses.ts | 50 ++++++ 19 files changed, 679 insertions(+), 136 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index a1cd9cba066..4521aaf2b70 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -346,7 +346,7 @@ export class GenerativeModel extends VertexAIModel { // @public export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; -// @public (undocumented) +// @beta export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; // @public @@ -430,122 +430,87 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } -// @public (undocumented) +// @beta export enum ImagenAspectRatio { - // (undocumented) LANDSCAPE_16x9 = "16:9", - // (undocumented) LANDSCAPE_3x4 = "3:4", - // (undocumented) PORTRAIT_4x3 = "4:3", - // (undocumented) PORTRAIT_9x16 = "9:16", - // (undocumented) SQUARE = "1:1" } -// @public (undocumented) +// @public export interface ImagenGCSImage { - // (undocumented) gcsURI: string; - // (undocumented) mimeType: string; } -// @public (undocumented) +// @beta export interface ImagenGenerationConfig { - // (undocumented) addWatermark?: boolean; - // (undocumented) aspectRatio?: ImagenAspectRatio; - // (undocumented) imageFormat?: ImagenImageFormat; - // (undocumented) negativePrompt?: string; - // (undocumented) numberOfImages?: number; } -// @public (undocumented) +// @beta export interface ImagenGenerationResponse { - // (undocumented) filteredReason?: string; - // (undocumented) images: T[]; } -// @public +// @beta export class ImagenImageFormat { - // (undocumented) compressionQuality?: number; - // (undocumented) static jpeg(compressionQuality?: number): ImagenImageFormat; - // (undocumented) mimeType: string; - // (undocumented) static png(): ImagenImageFormat; } -// @public +// @beta export interface ImagenInlineImage { - // (undocumented) bytesBase64Encoded: string; - // (undocumented) mimeType: string; } -// @public (undocumented) +// @beta export class ImagenModel extends VertexAIModel { constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); - // (undocumented) generateImages(prompt: string): Promise>; - // (undocumented) + // @internal generateImagesGCS(prompt: string, gcsURI: string): Promise>; - // (undocumented) generationConfig?: ImagenGenerationConfig; // (undocumented) requestOptions?: RequestOptions | undefined; - // (undocumented) safetySettings?: ImagenSafetySettings; } -// @public (undocumented) +// @beta export interface ImagenModelParams { - // (undocumented) generationConfig?: ImagenGenerationConfig; - // (undocumented) model: string; - // (undocumented) safetySettings?: ImagenSafetySettings; } -// @public (undocumented) +// @beta export enum ImagenPersonFilterLevel { - // (undocumented) ALLOW_ADULT = "allow_adult", - // (undocumented) ALLOW_ALL = "allow_all", - // (undocumented) BLOCK_ALL = "dont_allow" } -// @public (undocumented) +// @beta export enum ImagenSafetyFilterLevel { - // (undocumented) BLOCK_LOW_AND_ABOVE = "block_low_and_above", - // (undocumented) BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", - // (undocumented) BLOCK_NONE = "block_none", - // (undocumented) BLOCK_ONLY_HIGH = "block_only_high" } -// @public (undocumented) +// @beta export interface ImagenSafetySettings { - // (undocumented) personFilterLevel?: ImagenPersonFilterLevel; - // (undocumented) safetyFilterLevel?: ImagenSafetyFilterLevel; } @@ -838,13 +803,12 @@ export const enum VertexAIErrorCode { RESPONSE_ERROR = "response-error" } -// @public (undocumented) +// @public export abstract class VertexAIModel { // @internal protected constructor(vertexAI: VertexAI, modelName: string); // @internal (undocumented) protected _apiSettings: ApiSettings; - // (undocumented) readonly model: string; static normalizeModelName(modelName: string): string; } diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md index af4717129f7..b094e63c1d8 100644 --- a/docs-devsite/vertexai.imagengcsimage.md +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -10,6 +10,10 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImage interface +An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. + +This feature is not available yet. + Signature: ```typescript @@ -20,19 +24,29 @@ export interface ImagenGCSImage | Property | Type | Description | | --- | --- | --- | -| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | | -| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The URI of the file stored in a Cloud Storage for Firebase bucket. | +| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | ## ImagenGCSImage.gcsURI +The URI of the file stored in a Cloud Storage for Firebase bucket. + Signature: ```typescript gcsURI: string; ``` +### Example + +`"gs://bucket-name/path/sample_0.jpg"`. + ## ImagenGCSImage.mimeType +The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + +To request a different format, set the `imageFormat` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md index 4b2e91f01a7..cee7734f789 100644 --- a/docs-devsite/vertexai.imagengenerationconfig.md +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -10,6 +10,13 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationConfig interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Configuration options for generating images with Imagen. + +See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. + Signature: ```typescript @@ -20,14 +27,23 @@ export interface ImagenGenerationConfig | Property | Type | Description | | --- | --- | --- | -| [addWatermark](./vertexai.imagengenerationconfig.md#imagengenerationconfigaddwatermark) | boolean | | -| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | -| [imageFormat](./vertexai.imagengenerationconfig.md#imagengenerationconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | | -| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | -| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | +| [addWatermark](./vertexai.imagengenerationconfig.md#imagengenerationconfigaddwatermark) | boolean | (Public Preview) Whether to add an invisible watermark to generated images.If set to true, an invisible SynthID watermark is embedded in generated images to indicate that they are AI generated. If set to false, watermarking will be disabled.For Imagen 3 models, the default value is true; see the addWatermark documentation for more details. | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) The aspect ratio of the generated images. The default value is square 1:1. Supported aspect ratios depend on the Imagen model, see [ImagenAspectRatio](./vertexai.md#imagenaspectratio) for more details. | +| [imageFormat](./vertexai.imagengenerationconfig.md#imagengenerationconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | (Public Preview) The image format of the generated images. The default is PNG.See [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for more details. | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | (Public Preview) A description of what should be omitted from the generated images.Support for negative prompts depends on the Imagen model.See the [documentation](http://firebase.google.com/docs/vertex-ai/model-parameters#imagen) for more details. | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | (Public Preview) The number of images to generate. The default value is 1.The number of sample images that may be generated in each request depends on the model (typically up to 4); see the sampleCount documentation for more details. | ## ImagenGenerationConfig.addWatermark +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Whether to add an invisible watermark to generated images. + +If set to `true`, an invisible SynthID watermark is embedded in generated images to indicate that they are AI generated. If set to `false`, watermarking will be disabled. + +For Imagen 3 models, the default value is `true`; see the addWatermark documentation for more details. + Signature: ```typescript @@ -36,6 +52,11 @@ addWatermark?: boolean; ## ImagenGenerationConfig.aspectRatio +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The aspect ratio of the generated images. The default value is square 1:1. Supported aspect ratios depend on the Imagen model, see [ImagenAspectRatio](./vertexai.md#imagenaspectratio) for more details. + Signature: ```typescript @@ -44,6 +65,13 @@ aspectRatio?: ImagenAspectRatio; ## ImagenGenerationConfig.imageFormat +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The image format of the generated images. The default is PNG. + +See [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for more details. + Signature: ```typescript @@ -52,6 +80,15 @@ imageFormat?: ImagenImageFormat; ## ImagenGenerationConfig.negativePrompt +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A description of what should be omitted from the generated images. + +Support for negative prompts depends on the Imagen model. + +See the [documentation](http://firebase.google.com/docs/vertex-ai/model-parameters#imagen) for more details. + Signature: ```typescript @@ -60,6 +97,13 @@ negativePrompt?: string; ## ImagenGenerationConfig.numberOfImages +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The number of images to generate. The default value is 1. + +The number of sample images that may be generated in each request depends on the model (typically up to 4); see the sampleCount documentation for more details. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagengenerationresponse.md b/docs-devsite/vertexai.imagengenerationresponse.md index 320e2b68533..32ed69718f9 100644 --- a/docs-devsite/vertexai.imagengenerationresponse.md +++ b/docs-devsite/vertexai.imagengenerationresponse.md @@ -10,6 +10,11 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGenerationResponse interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The response from a request to generate images with Imagen. + Signature: ```typescript @@ -20,11 +25,20 @@ export interface ImagenGenerationResponse(Public Preview) The reason that images were filtered out. This property will only be defined if one or more images were filtered.Images may be filtered out due to the [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel), [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel), or filtering included in the model. The filter levels may be adjusted in your [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface).See the [Responsible AI and usage guidelines for Imagen](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen) for more details. | +| [images](./vertexai.imagengenerationresponse.md#imagengenerationresponseimages) | T\[\] | (Public Preview) The images generated by Imagen.The number of images generated may be fewer than the number requested if one or more were filtered out; see filteredReason. | ## ImagenGenerationResponse.filteredReason +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The reason that images were filtered out. This property will only be defined if one or more images were filtered. + +Images may be filtered out due to the [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel), [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel), or filtering included in the model. The filter levels may be adjusted in your [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface). + +See the [Responsible AI and usage guidelines for Imagen](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen) for more details. + Signature: ```typescript @@ -33,6 +47,13 @@ filteredReason?: string; ## ImagenGenerationResponse.images +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The images generated by Imagen. + +The number of images generated may be fewer than the number requested if one or more were filtered out; see `filteredReason`. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md index f52af54d9f4..785c7c726fc 100644 --- a/docs-devsite/vertexai.imagenimageformat.md +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -10,13 +10,12 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenImageFormat class - Copyright 2025 Google LLC +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +Defines the image format for images generated by Imagen. -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). Signature: @@ -28,18 +27,23 @@ export declare class ImagenImageFormat | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | | -| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | | number | (Public Preview) The level of compression (a number between 0 and 100). | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | | string | (Public Preview) The MIME type. | ## Methods | Method | Modifiers | Description | | --- | --- | --- | -| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | | -| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | | +| [jpeg(compressionQuality)](./vertexai.imagenimageformat.md#imagenimageformatjpeg) | static | (Public Preview) Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a JPEG image. | +| [png()](./vertexai.imagenimageformat.md#imagenimageformatpng) | static | (Public Preview) Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a PNG image. | ## ImagenImageFormat.compressionQuality +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The level of compression (a number between 0 and 100). + Signature: ```typescript @@ -48,6 +52,11 @@ compressionQuality?: number; ## ImagenImageFormat.mimeType +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The MIME type. + Signature: ```typescript @@ -56,6 +65,11 @@ mimeType: string; ## ImagenImageFormat.jpeg() +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a JPEG image. + Signature: ```typescript @@ -66,14 +80,21 @@ static jpeg(compressionQuality?: number): ImagenImageFormat; | Parameter | Type | Description | | --- | --- | --- | -| compressionQuality | number | | +| compressionQuality | number | The level of compression (a number between 0 and 100). | Returns: [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) +An [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) object for a JPEG image. + ## ImagenImageFormat.png() +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) for a PNG image. + Signature: ```typescript @@ -83,3 +104,16 @@ static png(): ImagenImageFormat; [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) +An [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) object for a PNG image. + +### Example + + +```javascript +const imagenModelParams = { + // ... other ImagenModelParams + imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. +} + +``` + diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md index b1b2fb3e52c..19fe8a67764 100644 --- a/docs-devsite/vertexai.imageninlineimage.md +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -10,13 +10,10 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenInlineImage interface - Copyright 2025 Google LLC +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +An image generated by Imagen, represented as inline data. Signature: @@ -28,11 +25,16 @@ export interface ImagenInlineImage | Property | Type | Description | | --- | --- | --- | -| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | -| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | (Public Preview) The base64-encoded image data. | +| [mimeType](./vertexai.imageninlineimage.md#imageninlineimagemimetype) | string | (Public Preview) The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | ## ImagenInlineImage.bytesBase64Encoded +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The base64-encoded image data. + Signature: ```typescript @@ -41,6 +43,13 @@ bytesBase64Encoded: string; ## ImagenInlineImage.mimeType +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + +To request a different format, set the `imageFormat` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md index c2730170180..63e15ff133a 100644 --- a/docs-devsite/vertexai.imagenmodel.md +++ b/docs-devsite/vertexai.imagenmodel.md @@ -10,6 +10,13 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModel class +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Class for Imagen model APIs. + +This class provides methods for generating images using the Imagen model. + Signature: ```typescript @@ -21,26 +28,28 @@ export declare class ImagenModel extends VertexAIModel | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | (Public Preview) Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [generationConfig](./vertexai.imagenmodel.md#imagenmodelgenerationconfig) | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | -| [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | -| [safetySettings](./vertexai.imagenmodel.md#imagenmodelsafetysettings) | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [generationConfig](./vertexai.imagenmodel.md#imagenmodelgenerationconfig) | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) The Imagen generation configuration. | +| [requestOptions](./vertexai.imagenmodel.md#imagenmodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | (Public Preview) | +| [safetySettings](./vertexai.imagenmodel.md#imagenmodelsafetysettings) | | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Safety settings for filtering inappropriate content. | ## Methods | Method | Modifiers | Description | | --- | --- | --- | -| [generateImages(prompt)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | | -| [generateImagesGCS(prompt, gcsURI)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | | +| [generateImages(prompt)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | (Public Preview) Generates images using the Imagen model and returns them as base64-encoded strings. | ## ImagenModel.(constructor) -Constructs a new instance of the `ImagenModel` class +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Constructs a new instance of the [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class. Signature: @@ -52,12 +61,21 @@ constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: | Parameter | Type | Description | | --- | --- | --- | -| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | -| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | -| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making requests to Imagen. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | Additional options to use when making requests. | + +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. ## ImagenModel.generationConfig +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The Imagen generation configuration. + Signature: ```typescript @@ -66,6 +84,9 @@ generationConfig?: ImagenGenerationConfig; ## ImagenModel.requestOptions +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + Signature: ```typescript @@ -74,6 +95,11 @@ requestOptions?: RequestOptions | undefined; ## ImagenModel.safetySettings +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Safety settings for filtering inappropriate content. + Signature: ```typescript @@ -82,6 +108,13 @@ safetySettings?: ImagenSafetySettings; ## ImagenModel.generateImages() +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If the prompt was not blocked, but one or more of the generated images were filtered, the returned object will have a `filteredReason` property. If all images are filtered, the `images` array will be empty. + Signature: ```typescript @@ -92,28 +125,33 @@ generateImages(prompt: string): PromiseReturns: Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)>> -## ImagenModel.generateImagesGCS() +A promise that resolves to an [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) object containing the generated images. -Signature: +#### Exceptions -```typescript -generateImagesGCS(prompt: string, gcsURI: string): Promise>; -``` +If the request to generate images fails. This happens if the prompt is blocked. -#### Parameters +### Example -| Parameter | Type | Description | -| --- | --- | --- | -| prompt | string | | -| gcsURI | string | | -Returns: +```javascript +const imagen = new ImagenModel( + vertexAI, + { + model: 'imagen-3.0-generate-002' + } +); -Promise<[ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface)<[ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)>> +const response = await imagen.generateImages('A photo of a cat'); +if (response.images.length > 0) { + console.log(response.images[0].bytesBase64Encoded); +} + +``` diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md index 21dbd533006..66c4bb0bfe6 100644 --- a/docs-devsite/vertexai.imagenmodelparams.md +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -10,6 +10,11 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenModelParams interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). + Signature: ```typescript @@ -20,12 +25,17 @@ export interface ImagenModelParams | Property | Type | Description | | --- | --- | --- | -| [generationConfig](./vertexai.imagenmodelparams.md#imagenmodelparamsgenerationconfig) | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | -| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | -| [safetySettings](./vertexai.imagenmodelparams.md#imagenmodelparamssafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [generationConfig](./vertexai.imagenmodelparams.md#imagenmodelparamsgenerationconfig) | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen. | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | (Public Preview) The Imagen model to use for generating images. For example: imagen-3.0-generate-002.Only Imagen 3 models (named imagen-3.0-*) are supported.See [model versions](https://firebase.google.com/docs/vertex-ai/models) for a full list of supported Imagen 3 models. | +| [safetySettings](./vertexai.imagenmodelparams.md#imagenmodelparamssafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Safety settings for filtering potentially inappropriate content. | ## ImagenModelParams.generationConfig +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Configuration options for generating images with Imagen. + Signature: ```typescript @@ -34,6 +44,15 @@ generationConfig?: ImagenGenerationConfig; ## ImagenModelParams.model +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +The Imagen model to use for generating images. For example: `imagen-3.0-generate-002`. + +Only Imagen 3 models (named `imagen-3.0-*`) are supported. + +See [model versions](https://firebase.google.com/docs/vertex-ai/models) for a full list of supported Imagen 3 models. + Signature: ```typescript @@ -42,6 +61,11 @@ model: string; ## ImagenModelParams.safetySettings +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Safety settings for filtering potentially inappropriate content. + Signature: ```typescript diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md index 0f5ea97c3e4..3cf7931a959 100644 --- a/docs-devsite/vertexai.imagensafetysettings.md +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -10,6 +10,13 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenSafetySettings interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Settings for controlling the aggressiveness of filtering out sensitive content. + +See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. + Signature: ```typescript @@ -20,11 +27,16 @@ export interface ImagenSafetySettings | Property | Type | Description | | --- | --- | --- | -| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | -| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressive to filter out sensitive content from generated images. | ## ImagenSafetySettings.personFilterLevel +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling whether generation of images containing people or faces is allowed. + Signature: ```typescript @@ -33,6 +45,11 @@ personFilterLevel?: ImagenPersonFilterLevel; ## ImagenSafetySettings.safetyFilterLevel +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling how aggressive to filter out sensitive content from generated images. + Signature: ```typescript diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index efc0cb92402..1d284345c39 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -20,7 +20,7 @@ The Vertex AI in Firebase Web SDK. | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | -| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | (Public Preview) Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.Only Imagen 3 models (named imagen-3.0-*) are supported. | ## Classes @@ -30,15 +30,15 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | -| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | Copyright 2025 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | -| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | (Public Preview) Defines the image format for images generated by Imagen.Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | (Public Preview) Class for Imagen model APIs.This class provides methods for generating images using the Imagen model. | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | | [Schema](./vertexai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with JSON.stringify() into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) | | [StringSchema](./vertexai.stringschema.md#stringschema_class) | Schema class for "string" types. Can be used with or without enum values. | | [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) | Error class for the Vertex AI in Firebase SDK. | -| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | | +| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | Base class for Vertex AI in Firebase model APIs. | ## Enumerations @@ -52,9 +52,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | -| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | -| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | -| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface).See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -90,12 +90,12 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | -| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | | -| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | -| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | | -| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Copyright 2025 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | -| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | -| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | +| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | (Public Preview) An image generated by Imagen, represented as inline data. | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | @@ -182,6 +182,13 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode ### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. + +Only Imagen 3 models (named `imagen-3.0-*`) are supported. + Signature: ```typescript @@ -192,14 +199,18 @@ export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenMo | Parameter | Type | Description | | --- | --- | --- | -| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | -| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | -| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making Imagen requests. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Additional options to use when making requests. | Returns: [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. + ## POSSIBLE\_ROLES Possible roles. @@ -398,6 +409,15 @@ export declare enum HarmSeverity ## ImagenAspectRatio +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Aspect ratios for Imagen images. + +To specify an aspect ratio for generated images, set the `aspectRatio` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + +See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. + Signature: ```typescript @@ -408,14 +428,21 @@ export declare enum ImagenAspectRatio | Member | Value | Description | | --- | --- | --- | -| LANDSCAPE\_16x9 | "16:9" | | -| LANDSCAPE\_3x4 | "3:4" | | -| PORTRAIT\_4x3 | "4:3" | | -| PORTRAIT\_9x16 | "9:16" | | -| SQUARE | "1:1" | | +| LANDSCAPE\_16x9 | "16:9" | (Public Preview) Landscape (16:9) aspect ratio. | +| LANDSCAPE\_3x4 | "3:4" | (Public Preview) Landscape (3:4) aspect ratio. | +| PORTRAIT\_4x3 | "4:3" | (Public Preview) Portrait (4:3) aspect ratio. | +| PORTRAIT\_9x16 | "9:16" | (Public Preview) Portrait (9:16) aspect ratio. | +| SQUARE | "1:1" | (Public Preview) Square (1:1) aspect ratio. | ## ImagenPersonFilterLevel +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling whether generation of images containing people or faces is allowed. + +See the personGeneration documentation for more details. + Signature: ```typescript @@ -426,12 +453,19 @@ export declare enum ImagenPersonFilterLevel | Member | Value | Description | | --- | --- | --- | -| ALLOW\_ADULT | "allow_adult" | | -| ALLOW\_ALL | "allow_all" | | -| BLOCK\_ALL | "dont_allow" | | +| ALLOW\_ADULT | "allow_adult" | (Public Preview) Allow generation of images containing adults only; images of children are filtered out.Generation of images containing people or faces may require your use case to be reviewed and approved by Cloud support; see the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) for more details. | +| ALLOW\_ALL | "allow_all" | (Public Preview) Allow generation of images containing adults only; images of children are filtered out.Generation of images containing people or faces may require your use case to be reviewed and approved by Cloud support; see the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) for more details. | +| BLOCK\_ALL | "dont_allow" | (Public Preview) Disallow generation of images containing people or faces; images of people are filtered out. | ## ImagenSafetyFilterLevel +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling how aggressively to filter sensitive content. + +Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, `violence`, `sexual`, `derogatory`, and `toxic`). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. + Signature: ```typescript @@ -442,10 +476,10 @@ export declare enum ImagenSafetyFilterLevel | Member | Value | Description | | --- | --- | --- | -| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | -| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | -| BLOCK\_NONE | "block_none" | | -| BLOCK\_ONLY\_HIGH | "block_only_high" | | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | (Public Preview) The most aggressive filtering level; most strict blocking. | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | (Public Preview) Blocks some sensitive prompts and responses. | +| BLOCK\_NONE | "block_none" | (Public Preview) The least aggressive filtering level; blocks very few sensitive prompts and responses.Access to this feature is restricted and may require your case to be reviewed and approved by Cloud support. | +| BLOCK\_ONLY\_HIGH | "block_only_high" | (Public Preview) Blocks few sensitive prompts and responses. | ## SchemaType diff --git a/docs-devsite/vertexai.vertexaimodel.md b/docs-devsite/vertexai.vertexaimodel.md index 31d7e1d9c32..5c3244fe1e5 100644 --- a/docs-devsite/vertexai.vertexaimodel.md +++ b/docs-devsite/vertexai.vertexaimodel.md @@ -10,6 +10,8 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # VertexAIModel class +Base class for Vertex AI in Firebase model APIs. + The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `VertexAIModel` class. Signature: @@ -22,7 +24,7 @@ export declare abstract class VertexAIModel | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | | +| [model](./vertexai.vertexaimodel.md#vertexaimodelmodel) | | string | The fully qualified model resource name to use for generating images (for example, publishers/google/models/imagen-3.0-generate-002). | ## Methods @@ -32,6 +34,8 @@ export declare abstract class VertexAIModel ## VertexAIModel.model +The fully qualified model resource name to use for generating images (for example, `publishers/google/models/imagen-3.0-generate-002`). + Signature: ```typescript diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 8b7477f306d..752e75c7e23 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -82,6 +82,20 @@ export function getGenerativeModel( return new GenerativeModel(vertexAI, modelParams, requestOptions); } +/** + * Returns an {@link ImagenModel} class with methods for using Imagen. + * + * Only Imagen 3 models (named `imagen-3.0-*`) are supported. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelParams - Parameters to use when making Imagen requests. + * @param requestOptions - Additional options to use when making requests. + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + * + * @beta + */ export function getImagenModel( vertexAI: VertexAI, modelParams: ImagenModelParams, diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts index 8d9ab93b6ba..89c740852a3 100644 --- a/packages/vertexai/src/models/imagen-model.ts +++ b/packages/vertexai/src/models/imagen-model.ts @@ -30,10 +30,48 @@ import { } from '../types'; import { VertexAIModel } from './vertexai-model'; +/** + * Class for Imagen model APIs. + * + * This class provides methods for generating images using the Imagen model. + * + * @example + * ```javascript + * const imagen = new ImagenModel( + * vertexAI, + * { + * model: 'imagen-3.0-generate-002' + * } + * ); + * + * const response = await imagen.generateImages('A photo of a cat'); + * if (response.images.length > 0) { + * console.log(response.images[0].bytesBase64Encoded); + * } + * ``` + * + * @beta + */ export class ImagenModel extends VertexAIModel { + /** + * The Imagen generation configuration. + */ generationConfig?: ImagenGenerationConfig; + /** + * Safety settings for filtering inappropriate content. + */ safetySettings?: ImagenSafetySettings; + /** + * Constructs a new instance of the {@link ImagenModel} class. + * + * @param vertexAI - An instance of the Vertex AI in Firebase SDK. + * @param modelParams - Parameters to use when making requests to Imagen. + * @param requestOptions - Additional options to use when making requests. + * + * @throws If the `apiKey` or `projectId` fields are missing in your + * Firebase config. + */ constructor( vertexAI: VertexAI, modelParams: ImagenModelParams, @@ -45,6 +83,24 @@ export class ImagenModel extends VertexAIModel { this.safetySettings = safetySettings; } + /** + * Generates images using the Imagen model and returns them as + * base64-encoded strings. + * + * @param prompt - A text prompt describing the image(s) to generate. + * @returns A promise that resolves to an {@link ImagenGenerationResponse} + * object containing the generated images. + * + * @throws If the request to generate images fails. This happens if the + * prompt is blocked. + * + * @remarks + * If the prompt was not blocked, but one or more of the generated images were filtered, the + * returned object will have a `filteredReason` property. + * If all images are filtered, the `images` array will be empty. + * + * @beta + */ async generateImages( prompt: string ): Promise> { @@ -63,6 +119,25 @@ export class ImagenModel extends VertexAIModel { return handlePredictResponse(response); } + /** + * Generates images to Cloud Storage for Firebase using the Imagen model. + * + * @internal This method is temporarily internal. + * + * @param prompt - A text prompt describing the image(s) to generate. + * @param gcsURI - The URI of file stored in a Cloud Storage for Firebase bucket. + * This should be a directory. For example, `gs://my-bucket/my-directory/`. + * @returns A promise that resolves to an {@link ImagenGenerationResponse} + * object containing the URLs of the generated images. + * + * @throws If the request fails to generate images fails. This happens if + * the prompt is blocked. + * + * @remarks + * If the prompt was not blocked, but one or more of the generated images were filtered, the + * returned object will have a `filteredReason` property. + * If all images are filtered, the `images` array will be empty. + */ async generateImagesGCS( prompt: string, gcsURI: string diff --git a/packages/vertexai/src/models/vertexai-model.ts b/packages/vertexai/src/models/vertexai-model.ts index 5bed60d6279..4e211c0cf94 100644 --- a/packages/vertexai/src/models/vertexai-model.ts +++ b/packages/vertexai/src/models/vertexai-model.ts @@ -21,7 +21,16 @@ import { VertexAIService } from '../service'; import { ApiSettings } from '../types/internal'; import { _isFirebaseServerApp } from '@firebase/app'; +/** + * Base class for Vertex AI in Firebase model APIs. + * + * @public + */ export abstract class VertexAIModel { + /** + * The fully qualified model resource name to use for generating images + * (for example, `publishers/google/models/imagen-3.0-generate-002`). + */ readonly model: string; /** diff --git a/packages/vertexai/src/requests/imagen-image-format.ts b/packages/vertexai/src/requests/imagen-image-format.ts index e6dbab041a6..283dc80bfaf 100644 --- a/packages/vertexai/src/requests/imagen-image-format.ts +++ b/packages/vertexai/src/requests/imagen-image-format.ts @@ -17,14 +17,45 @@ import { logger } from '../logger'; +/** + * Defines the image format for images generated by Imagen. + * + * Use this class to specify the desired format (JPEG or PNG) and compression quality + * for images generated by Imagen. This is typically included as part of + * {@link ImagenModelParams}. + * + * @example + * ```javascript + * const imagenModelParams = { + * // ... other ImagenModelParams + * imageFormat: ImagenImageFormat.jpeg(75) // JPEG with a compression level of 75. + * } + * ``` + * + * @beta + */ export class ImagenImageFormat { + /** + * The MIME type. + */ mimeType: string; + /** + * The level of compression (a number between 0 and 100). + */ compressionQuality?: number; private constructor() { this.mimeType = 'image/png'; } + /** + * Creates an {@link ImagenImageFormat} for a JPEG image. + * + * @param compressionQuality - The level of compression (a number between 0 and 100). + * @returns An {@link ImagenImageFormat} object for a JPEG image. + * + * @beta + */ static jpeg(compressionQuality?: number): ImagenImageFormat { if ( compressionQuality && @@ -37,6 +68,13 @@ export class ImagenImageFormat { return { mimeType: 'image/jpeg', compressionQuality }; } + /** + * Creates an {@link ImagenImageFormat} for a PNG image. + * + * @returns An {@link ImagenImageFormat} object for a PNG image. + * + * @beta + */ static png(): ImagenImageFormat { return { mimeType: 'image/png' }; } diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 33d2fab266a..b99c327c044 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -126,7 +126,7 @@ export function formatGenerateContentInput( } /** - * Convert the user-defined parameters in {@link ImagenGenerationParams} to the format + * Convert the user-defined parameters in {@link ImagenGenerationParams} to the format * that is expected from the REST API. * * @internal diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts index fb81e634617..02a8a55e01c 100644 --- a/packages/vertexai/src/types/imagen/internal.ts +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -121,7 +121,7 @@ export interface PredictRequestBody { */ export type ImagenGenerationParams = { /** - * The Google Cloud Storage (GCS) URI where the images should be stored + * The Cloud Storage for Firebase bucket URI where the images should be stored * (for GCS requests only). */ gcsURI?: string; diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts index cf72391dc41..ac37488dfb5 100644 --- a/packages/vertexai/src/types/imagen/requests.ts +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -17,42 +17,196 @@ import { ImagenImageFormat } from '../../requests/imagen-image-format'; +/** + * Parameters for configuring an {@link ImagenModel}. + * + * @beta + */ export interface ImagenModelParams { + /** + * The Imagen model to use for generating images. + * For example: `imagen-3.0-generate-002`. + * + * Only Imagen 3 models (named `imagen-3.0-*`) are supported. + * + * See {@link https://firebase.google.com/docs/vertex-ai/models | model versions} + * for a full list of supported Imagen 3 models. + */ model: string; + /** + * Configuration options for generating images with Imagen. + */ generationConfig?: ImagenGenerationConfig; + /** + * Safety settings for filtering potentially inappropriate content. + */ safetySettings?: ImagenSafetySettings; } +/** + * Configuration options for generating images with Imagen. + * + * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images-imagen | documentation} for + * more details. + * + * @beta + */ export interface ImagenGenerationConfig { + /** + * A description of what should be omitted from the generated images. + * + * Support for negative prompts depends on the Imagen model. + * + * See the {@link http://firebase.google.com/docs/vertex-ai/model-parameters#imagen | documentation} for more details. + */ negativePrompt?: string; + /** + * The number of images to generate. The default value is 1. + * + * The number of sample images that may be generated in each request depends on the model + * (typically up to 4); see the sampleCount + * documentation for more details. + */ numberOfImages?: number; + /** + * The aspect ratio of the generated images. The default value is square 1:1. + * Supported aspect ratios depend on the Imagen model, see {@link ImagenAspectRatio} + * for more details. + */ aspectRatio?: ImagenAspectRatio; + /** + * The image format of the generated images. The default is PNG. + * + * See {@link ImagenImageFormat} for more details. + */ imageFormat?: ImagenImageFormat; + /** + * Whether to add an invisible watermark to generated images. + * + * If set to `true`, an invisible SynthID watermark is embedded in generated images to indicate + * that they are AI generated. If set to `false`, watermarking will be disabled. + * + * For Imagen 3 models, the default value is `true`; see the addWatermark + * documentation for more details. + */ addWatermark?: boolean; } +/** + * A filter level controlling how aggressively to filter sensitive content. + * + * Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI + * are assessed against a list of safety filters, which include 'harmful categories' (for example, + * `violence`, `sexual`, `derogatory`, and `toxic`). This filter level controls how aggressively to + * filter out potentially harmful content from responses. See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation } + * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines} + * for more details. + * + * @beta + */ export enum ImagenSafetyFilterLevel { + /** + * The most aggressive filtering level; most strict blocking. + */ BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + /** + * Blocks some sensitive prompts and responses. + */ BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + /** + * Blocks few sensitive prompts and responses. + */ BLOCK_ONLY_HIGH = 'block_only_high', + /** + * The least aggressive filtering level; blocks very few sensitive prompts and responses. + * + * Access to this feature is restricted and may require your case to be reviewed and approved by + * Cloud support. + */ BLOCK_NONE = 'block_none' } +/** + * A filter level controlling whether generation of images containing people or faces is allowed. + * + * See the personGeneration + * documentation for more details. + * + * @beta + */ export enum ImagenPersonFilterLevel { + /** + * Disallow generation of images containing people or faces; images of people are filtered out. + */ BLOCK_ALL = 'dont_allow', + /** + * Allow generation of images containing adults only; images of children are filtered out. + * + * Generation of images containing people or faces may require your use case to be + * reviewed and approved by Cloud support; see the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen | Responsible AI and usage guidelines} + * for more details. + */ ALLOW_ADULT = 'allow_adult', + /** + * Allow generation of images containing adults only; images of children are filtered out. + * + * Generation of images containing people or faces may require your use case to be + * reviewed and approved by Cloud support; see the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen | Responsible AI and usage guidelines} + * for more details. + */ ALLOW_ALL = 'allow_all' } +/** + * Settings for controlling the aggressiveness of filtering out sensitive content. + * + * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation } + * for more details. + * + * @beta + */ export interface ImagenSafetySettings { + /** + * A filter level controlling how aggressive to filter out sensitive content from generated + * images. + */ safetyFilterLevel?: ImagenSafetyFilterLevel; + /** + * A filter level controlling whether generation of images containing people or faces is allowed. + */ personFilterLevel?: ImagenPersonFilterLevel; } +/** + * Aspect ratios for Imagen images. + * + * To specify an aspect ratio for generated images, set the `aspectRatio` property in your + * {@link ImagenGenerationConfig}. + * + * See the the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation } + * for more details and examples of the supported aspect ratios. + * + * @beta + */ export enum ImagenAspectRatio { + /** + * Square (1:1) aspect ratio. + */ SQUARE = '1:1', + /** + * Landscape (3:4) aspect ratio. + */ LANDSCAPE_3x4 = '3:4', + /** + * Portrait (4:3) aspect ratio. + */ PORTRAIT_4x3 = '4:3', + /** + * Landscape (16:9) aspect ratio. + */ LANDSCAPE_16x9 = '16:9', + /** + * Portrait (9:16) aspect ratio. + */ PORTRAIT_9x16 = '9:16' } diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts index 514acc601b1..c5cf5dd9057 100644 --- a/packages/vertexai/src/types/imagen/responses.ts +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -15,19 +15,69 @@ * limitations under the License. */ +/** + * An image generated by Imagen, represented as inline data. + * + * @beta + */ export interface ImagenInlineImage { + /** + * The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + * + * To request a different format, set the `imageFormat` property in your {@link ImagenGenerationConfig}. + */ mimeType: string; + /** + * The base64-encoded image data. + */ bytesBase64Encoded: string; } +/** + * An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. + * + * This feature is not available yet. + */ export interface ImagenGCSImage { + /** + * The MIME type of the image; either `"image/png"` or `"image/jpeg"`. + * + * To request a different format, set the `imageFormat` property in your {@link ImagenGenerationConfig}. + */ mimeType: string; + /** + * The URI of the file stored in a Cloud Storage for Firebase bucket. + * + * @example `"gs://bucket-name/path/sample_0.jpg"`. + */ gcsURI: string; } +/** + * The response from a request to generate images with Imagen. + * + * @beta + */ export interface ImagenGenerationResponse< T extends ImagenInlineImage | ImagenGCSImage > { + /** + * The images generated by Imagen. + * + * The number of images generated may be fewer than the number requested if one or more were + * filtered out; see `filteredReason`. + */ images: T[]; + /** + * The reason that images were filtered out. This property will only be defined if one + * or more images were filtered. + * + * Images may be filtered out due to the {@link ImagenSafetyFilterLevel}, + * {@link ImagenPersonFilterLevel}, or filtering included in the model. + * The filter levels may be adjusted in your {@link ImagenSafetySettings}. + * + * See the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen | Responsible AI and usage guidelines for Imagen} + * for more details. + */ filteredReason?: string; } From 5e7df2f2fbf8206a023ab51ad9749a30e8573853 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 20 Feb 2025 10:57:30 -0500 Subject: [PATCH 30/31] Cleanup --- packages/vertexai/src/requests/request-helpers.ts | 9 +++++---- packages/vertexai/src/requests/response-helpers.ts | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index b99c327c044..f69c88fca92 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -50,11 +50,11 @@ export function formatNewContent( if (typeof request === 'string') { newParts = [{ text: request }]; } else { - for (const elem of request) { - if (typeof elem === 'string') { - newParts.push({ text: elem }); + for (const partOrString of request) { + if (typeof partOrString === 'string') { + newParts.push({ text: partOrString }); } else { - newParts.push(elem); + newParts.push(partOrString); } } } @@ -114,6 +114,7 @@ export function formatGenerateContentInput( if ((params as GenerateContentRequest).contents) { formattedRequest = params as GenerateContentRequest; } else { + // Array or string const content = formatNewContent(params as string | Array); formattedRequest = { contents: [content] }; } diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 95bcfea20fc..1ba0986f3f9 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -207,7 +207,6 @@ export function formatBlockErrorMessage( * * @internal */ - export async function handlePredictResponse< T extends ImagenInlineImage | ImagenGCSImage >(response: Response): Promise<{ images: T[]; filteredReason?: string }> { From 4a306faa9c294cda428cc97daf13a48de8866e2d Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 20 Feb 2025 13:41:11 -0500 Subject: [PATCH 31/31] update changeset --- .changeset/violet-planets-impress.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/violet-planets-impress.md b/.changeset/violet-planets-impress.md index a7f09acd1a2..62427699f26 100644 --- a/.changeset/violet-planets-impress.md +++ b/.changeset/violet-planets-impress.md @@ -3,4 +3,4 @@ '@firebase/vertexai': minor --- -Add Image Generation support using Imagen. +**Public Preview** Added support for generating images using the Imagen 3 model.