From c1d397f50fb1bbb875ae69737d7c53caf6b0582b Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 9 Dec 2024 11:57:18 -0500 Subject: [PATCH] WIP --- common/api-review/vertexai.api.md | 184 +++ docs-devsite/_toc.yaml | 24 + docs-devsite/vertexai.imagengcsimage.md | 36 + .../vertexai.imagengcsimageresponse.md | 40 + .../vertexai.imagengenerationconfig.md | 49 + docs-devsite/vertexai.imagenimageformat.md | 40 + docs-devsite/vertexai.imagenimagereponse.md | 40 + docs-devsite/vertexai.imageninlineimage.md | 34 + .../vertexai.imageninlineimageresponse.md | 46 + docs-devsite/vertexai.imagenmodel.md | 124 ++ docs-devsite/vertexai.imagenmodelconfig.md | 49 + docs-devsite/vertexai.imagenmodelparams.md | 40 + docs-devsite/vertexai.imagensafetysettings.md | 45 + docs-devsite/vertexai.md | 135 +++ docs-devsite/vertexai.safetyattributes.md | 40 + .../test/unit/transportoptions.test.ts | 3 +- packages/vertexai/api-extractor.json | 18 +- packages/vertexai/patch.txt | 1027 +++++++++++++++++ packages/vertexai/src/api.test.ts | 54 +- packages/vertexai/src/api.ts | 31 +- .../vertexai/src/models/imagen-model.test.ts | 246 ++++ packages/vertexai/src/models/imagen-model.ts | 199 ++++ .../src/requests/request-helpers.test.ts | 79 +- .../vertexai/src/requests/request-helpers.ts | 53 +- packages/vertexai/src/requests/request.ts | 3 +- .../src/requests/response-helpers.test.ts | 78 +- .../vertexai/src/requests/response-helpers.ts | 52 + packages/vertexai/src/types/imagen/index.ts | 19 + .../vertexai/src/types/imagen/internal.ts | 54 + .../vertexai/src/types/imagen/requests.ts | 142 +++ .../vertexai/src/types/imagen/responses.ts | 73 ++ packages/vertexai/src/types/index.ts | 1 + packages/vertexai/src/types/internal.ts | 2 + packages/vertexai/src/types/responses.ts | 5 + 34 files changed, 3041 insertions(+), 24 deletions(-) create mode 100644 docs-devsite/vertexai.imagengcsimage.md create mode 100644 docs-devsite/vertexai.imagengcsimageresponse.md create mode 100644 docs-devsite/vertexai.imagengenerationconfig.md create mode 100644 docs-devsite/vertexai.imagenimageformat.md create mode 100644 docs-devsite/vertexai.imagenimagereponse.md create mode 100644 docs-devsite/vertexai.imageninlineimage.md create mode 100644 docs-devsite/vertexai.imageninlineimageresponse.md create mode 100644 docs-devsite/vertexai.imagenmodel.md create mode 100644 docs-devsite/vertexai.imagenmodelconfig.md create mode 100644 docs-devsite/vertexai.imagenmodelparams.md create mode 100644 docs-devsite/vertexai.imagensafetysettings.md create mode 100644 docs-devsite/vertexai.safetyattributes.md create mode 100644 packages/vertexai/patch.txt create mode 100644 packages/vertexai/src/models/imagen-model.test.ts create mode 100644 packages/vertexai/src/models/imagen-model.ts create mode 100644 packages/vertexai/src/types/imagen/index.ts create mode 100644 packages/vertexai/src/types/imagen/internal.ts create mode 100644 packages/vertexai/src/types/imagen/requests.ts create mode 100644 packages/vertexai/src/types/imagen/responses.ts diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 041bc62451f..aa35a3f024f 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -348,6 +348,9 @@ export class GenerativeModel { // @public export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +// @public +export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; + // @public export function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI; @@ -429,6 +432,148 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } +// @public (undocumented) +export enum ImagenAspectRatio { + // (undocumented) + CLASSIC_LANDSCAPE = "4:3", + // (undocumented) + CLASSIC_PORTRAIT = "3:4", + // (undocumented) + PORTRAIT = "9:16", + // (undocumented) + SQUARE = "1:1", + // (undocumented) + WIDESCREEN = "16:9" +} + +// Warning: (ae-incompatible-release-tags) The symbol "ImagenGCSImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal +// +// @public +export interface ImagenGCSImage extends ImagenImage { + gcsURI: string; +} + +// @public (undocumented) +export interface ImagenGCSImageResponse { + // (undocumented) + filteredReason?: string; + // (undocumented) + images: ImagenGCSImage[]; +} + +// @public (undocumented) +export interface ImagenGenerationConfig { + // (undocumented) + aspectRatio?: ImagenAspectRatio; + // (undocumented) + negativePrompt?: string; + // (undocumented) + numberOfImages?: number; +} + +// Warning: (ae-internal-missing-underscore) The name "ImagenImage" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface ImagenImage { + // (undocumented) + mimeType: string; +} + +// @public (undocumented) +export interface ImagenImageFormat { + // (undocumented) + compressionQuality?: number; + // (undocumented) + mimeType: string; +} + +// @public (undocumented) +export interface ImagenImageReponse { + // (undocumented) + filteredReason?: string; + // Warning: (ae-incompatible-release-tags) The symbol "images" is marked as @public, but its signature references "ImagenImage" which is marked as @internal + // + // (undocumented) + images: ImagenImage[]; +} + +// Warning: (ae-incompatible-release-tags) The symbol "ImagenInlineImage" is marked as @public, but its signature references "ImagenImage" which is marked as @internal +// +// @public +export interface ImagenInlineImage extends ImagenImage { + // (undocumented) + bytesBase64Encoded: string; +} + +// @public +export interface ImagenInlineImageResponse { + filteredReason?: string; + images: ImagenInlineImage[]; +} + +// @public +export class ImagenModel { + constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); + generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; + // (undocumented) + model: string; + } + +// @public (undocumented) +export interface ImagenModelConfig { + // (undocumented) + addWatermark?: boolean; + // (undocumented) + imageFormat?: ImagenImageFormat; + // (undocumented) + safetySettings?: ImagenSafetySettings; +} + +// @public +export interface ImagenModelParams extends ImagenModelConfig { + // (undocumented) + model: string; +} + +// @public (undocumented) +export enum ImagenPersonFilterLevel { + // (undocumented) + ALLOW_ADULT = "allow_adult", + // (undocumented) + ALLOW_ALL = "allow_all", + // (undocumented) + BLOCK_ALL = "dont_allow" +} + +// Warning: (ae-internal-missing-underscore) The name "ImagenRequestConfig" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface ImagenRequestConfig extends ImagenModelConfig, ImagenGenerationConfig { + // (undocumented) + gcsURI?: string; + // (undocumented) + prompt: string; +} + +// @public (undocumented) +export enum ImagenSafetyFilterLevel { + // (undocumented) + BLOCK_LOW_AND_ABOVE = "block_low_and_above", + // (undocumented) + BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", + // (undocumented) + BLOCK_NONE = "block_none", + // (undocumented) + BLOCK_ONLY_HIGH = "block_only_high" +} + +// @public (undocumented) +export interface ImagenSafetySettings { + personFilterLevel?: ImagenPersonFilterLevel; + safetyFilterLevel?: ImagenSafetyFilterLevel; +} + // @public export interface InlineDataPart { // (undocumented) @@ -447,6 +592,9 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export function jpeg(compressionQuality: number): ImagenImageFormat; + // @public export interface ModelParams extends BaseParams { // (undocumented) @@ -490,9 +638,37 @@ export interface ObjectSchemaInterface extends SchemaInterface { // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +// @public +export function png(): ImagenImageFormat; + // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; +// Warning: (ae-internal-missing-underscore) The name "PredictRequestBody" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export interface PredictRequestBody { + // (undocumented) + instances: [ + { + prompt: string; + } + ]; + // (undocumented) + parameters: { + sampleCount: number; + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; + includeRaiReason: boolean; + }; +} + // @public export interface PromptFeedback { // (undocumented) @@ -520,6 +696,14 @@ export interface RetrievedContextAttribution { // @public export type Role = (typeof POSSIBLE_ROLES)[number]; +// @public (undocumented) +export interface SafetyAttributes { + // (undocumented) + categories: string[]; + // (undocumented) + scores: number[]; +} + // @public export interface SafetyRating { // (undocumented) diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 9d60c12906c..6156c51c3f9 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -528,6 +528,28 @@ toc: path: /docs/reference/js/vertexai.groundingattribution.md - title: GroundingMetadata path: /docs/reference/js/vertexai.groundingmetadata.md + - title: ImagenGCSImage + path: /docs/reference/js/vertexai.imagengcsimage.md + - title: ImagenGCSImageResponse + path: /docs/reference/js/vertexai.imagengcsimageresponse.md + - title: ImagenGenerationConfig + path: /docs/reference/js/vertexai.imagengenerationconfig.md + - title: ImagenImageFormat + path: /docs/reference/js/vertexai.imagenimageformat.md + - title: ImagenImageReponse + path: /docs/reference/js/vertexai.imagenimagereponse.md + - title: ImagenInlineImage + path: /docs/reference/js/vertexai.imageninlineimage.md + - title: ImagenInlineImageResponse + path: /docs/reference/js/vertexai.imageninlineimageresponse.md + - title: ImagenModel + path: /docs/reference/js/vertexai.imagenmodel.md + - title: ImagenModelConfig + path: /docs/reference/js/vertexai.imagenmodelconfig.md + - title: ImagenModelParams + path: /docs/reference/js/vertexai.imagenmodelparams.md + - title: ImagenSafetySettings + path: /docs/reference/js/vertexai.imagensafetysettings.md - title: InlineDataPart path: /docs/reference/js/vertexai.inlinedatapart.md - title: IntegerSchema @@ -546,6 +568,8 @@ toc: path: /docs/reference/js/vertexai.requestoptions.md - title: RetrievedContextAttribution path: /docs/reference/js/vertexai.retrievedcontextattribution.md + - title: SafetyAttributes + path: /docs/reference/js/vertexai.safetyattributes.md - title: SafetyRating path: /docs/reference/js/vertexai.safetyrating.md - title: SafetySetting diff --git a/docs-devsite/vertexai.imagengcsimage.md b/docs-devsite/vertexai.imagengcsimage.md new file mode 100644 index 00000000000..eb2e9b76556 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimage.md @@ -0,0 +1,36 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImage interface +Image generated by Imagen, stored in Google Cloud Storage (GCS). + +Signature: + +```typescript +export interface ImagenGCSImage extends ImagenImage +``` +Extends: ImagenImage + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The Google Cloud Storage (GCS) URI at which the generated image is stored. | + +## ImagenGCSImage.gcsURI + +The Google Cloud Storage (GCS) URI at which the generated image is stored. + +Signature: + +```typescript +gcsURI: string; +``` diff --git a/docs-devsite/vertexai.imagengcsimageresponse.md b/docs-devsite/vertexai.imagengcsimageresponse.md new file mode 100644 index 00000000000..0c20cb1e752 --- /dev/null +++ b/docs-devsite/vertexai.imagengcsimageresponse.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGCSImageResponse interface +Signature: + +```typescript +export interface ImagenGCSImageResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagengcsimageresponse.md#imagengcsimageresponsefilteredreason) | string | | +| [images](./vertexai.imagengcsimageresponse.md#imagengcsimageresponseimages) | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface)\[\] | | + +## ImagenGCSImageResponse.filteredReason + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenGCSImageResponse.images + +Signature: + +```typescript +images: ImagenGCSImage[]; +``` diff --git a/docs-devsite/vertexai.imagengenerationconfig.md b/docs-devsite/vertexai.imagengenerationconfig.md new file mode 100644 index 00000000000..bf28bc291ae --- /dev/null +++ b/docs-devsite/vertexai.imagengenerationconfig.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenGenerationConfig interface +Signature: + +```typescript +export interface ImagenGenerationConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [aspectRatio](./vertexai.imagengenerationconfig.md#imagengenerationconfigaspectratio) | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [negativePrompt](./vertexai.imagengenerationconfig.md#imagengenerationconfignegativeprompt) | string | | +| [numberOfImages](./vertexai.imagengenerationconfig.md#imagengenerationconfignumberofimages) | number | | + +## ImagenGenerationConfig.aspectRatio + +Signature: + +```typescript +aspectRatio?: ImagenAspectRatio; +``` + +## ImagenGenerationConfig.negativePrompt + +Signature: + +```typescript +negativePrompt?: string; +``` + +## ImagenGenerationConfig.numberOfImages + +Signature: + +```typescript +numberOfImages?: number; +``` diff --git a/docs-devsite/vertexai.imagenimageformat.md b/docs-devsite/vertexai.imagenimageformat.md new file mode 100644 index 00000000000..7ab7b9eb277 --- /dev/null +++ b/docs-devsite/vertexai.imagenimageformat.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageFormat interface +Signature: + +```typescript +export interface ImagenImageFormat +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [compressionQuality](./vertexai.imagenimageformat.md#imagenimageformatcompressionquality) | number | | +| [mimeType](./vertexai.imagenimageformat.md#imagenimageformatmimetype) | string | | + +## ImagenImageFormat.compressionQuality + +Signature: + +```typescript +compressionQuality?: number; +``` + +## ImagenImageFormat.mimeType + +Signature: + +```typescript +mimeType: string; +``` diff --git a/docs-devsite/vertexai.imagenimagereponse.md b/docs-devsite/vertexai.imagenimagereponse.md new file mode 100644 index 00000000000..14e34651a41 --- /dev/null +++ b/docs-devsite/vertexai.imagenimagereponse.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenImageReponse interface +Signature: + +```typescript +export interface ImagenImageReponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imagenimagereponse.md#imagenimagereponsefilteredreason) | string | | +| [images](./vertexai.imagenimagereponse.md#imagenimagereponseimages) | ImagenImage\[\] | | + +## ImagenImageReponse.filteredReason + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenImageReponse.images + +Signature: + +```typescript +images: ImagenImage[]; +``` diff --git a/docs-devsite/vertexai.imageninlineimage.md b/docs-devsite/vertexai.imageninlineimage.md new file mode 100644 index 00000000000..fdfac65c15a --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimage.md @@ -0,0 +1,34 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImage interface +Image generated by Imagen to inline bytes. + +Signature: + +```typescript +export interface ImagenInlineImage extends ImagenImage +``` +Extends: ImagenImage + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [bytesBase64Encoded](./vertexai.imageninlineimage.md#imageninlineimagebytesbase64encoded) | string | | + +## ImagenInlineImage.bytesBase64Encoded + +Signature: + +```typescript +bytesBase64Encoded: string; +``` diff --git a/docs-devsite/vertexai.imageninlineimageresponse.md b/docs-devsite/vertexai.imageninlineimageresponse.md new file mode 100644 index 00000000000..06cf3a2857f --- /dev/null +++ b/docs-devsite/vertexai.imageninlineimageresponse.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenInlineImageResponse interface +Imagen image response. + +Signature: + +```typescript +export interface ImagenInlineImageResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [filteredReason](./vertexai.imageninlineimageresponse.md#imageninlineimageresponsefilteredreason) | string | The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). | +| [images](./vertexai.imageninlineimageresponse.md#imageninlineimageresponseimages) | [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface)\[\] | The images generated by Imagen. If all images were filtered, this will be empty. | + +## ImagenInlineImageResponse.filteredReason + +The reason the missing images were filtered. For the mappings of error codes to reasons, see [https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen\#safety-categories](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories). + +Signature: + +```typescript +filteredReason?: string; +``` + +## ImagenInlineImageResponse.images + +The images generated by Imagen. If all images were filtered, this will be empty. + +Signature: + +```typescript +images: ImagenInlineImage[]; +``` diff --git a/docs-devsite/vertexai.imagenmodel.md b/docs-devsite/vertexai.imagenmodel.md new file mode 100644 index 00000000000..f1bff9689fe --- /dev/null +++ b/docs-devsite/vertexai.imagenmodel.md @@ -0,0 +1,124 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModel class +Class for Imagen model APIs. + +Signature: + +```typescript +export declare class ImagenModel +``` + +## Constructors + +| Constructor | Modifiers | Description | +| --- | --- | --- | +| [(constructor)(vertexAI, modelParams, requestOptions)](./vertexai.imagenmodel.md#imagenmodelconstructor) | | Constructs a new instance of the ImagenModel class | + +## Properties + +| Property | Modifiers | Type | Description | +| --- | --- | --- | --- | +| [model](./vertexai.imagenmodel.md#imagenmodelmodel) | | string | | + +## Methods + +| Method | Modifiers | Description | +| --- | --- | --- | +| [generateImages(prompt, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimages) | | Generates images using the Imagen model and returns them as base64-encoded strings. | +| [generateImagesGCS(prompt, gcsURI, imagenRequestOptions)](./vertexai.imagenmodel.md#imagenmodelgenerateimagesgcs) | | Generates images using the Imagen model and returns them as base64-encoded strings. | + +## ImagenModel.(constructor) + +Constructs a new instance of the `ImagenModel` class + +Signature: + +```typescript +constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined); +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) \| undefined | | + +## ImagenModel.model + +Signature: + +```typescript +model: string; +``` + +## ImagenModel.generateImages() + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. + +Signature: + +```typescript +generateImages(prompt: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | The text prompt used to generate the images. | +| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +Returns: + +Promise<[ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface)> + +A promise that resolves to an [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) object containing the generated images. + +#### Exceptions + +If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). + +## ImagenModel.generateImagesGCS() + +Generates images using the Imagen model and returns them as base64-encoded strings. + +If one or more images are filtered, the returned object will have a defined `filteredReason` property. If all images are filtered, the `images` array will be empty, and no error will be thrown. + +Signature: + +```typescript +generateImagesGCS(prompt: string, gcsURI: string, imagenRequestOptions?: ImagenGenerationConfig): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| prompt | string | The text prompt used to generate the images. | +| gcsURI | string | The GCS URI where the images should be stored. | +| imagenRequestOptions | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for the Imagen generation request. See [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). | + +Returns: + +Promise<[ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface)> + +A promise that resolves to an [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) object containing the generated images. + +#### Exceptions + +If the request fails or if the prompt is blocked, throws a [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class). + diff --git a/docs-devsite/vertexai.imagenmodelconfig.md b/docs-devsite/vertexai.imagenmodelconfig.md new file mode 100644 index 00000000000..9923d0de504 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelconfig.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelConfig interface +Signature: + +```typescript +export interface ImagenModelConfig +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [addWatermark](./vertexai.imagenmodelconfig.md#imagenmodelconfigaddwatermark) | boolean | | +| [imageFormat](./vertexai.imagenmodelconfig.md#imagenmodelconfigimageformat) | [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | +| [safetySettings](./vertexai.imagenmodelconfig.md#imagenmodelconfigsafetysettings) | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | + +## ImagenModelConfig.addWatermark + +Signature: + +```typescript +addWatermark?: boolean; +``` + +## ImagenModelConfig.imageFormat + +Signature: + +```typescript +imageFormat?: ImagenImageFormat; +``` + +## ImagenModelConfig.safetySettings + +Signature: + +```typescript +safetySettings?: ImagenSafetySettings; +``` diff --git a/docs-devsite/vertexai.imagenmodelparams.md b/docs-devsite/vertexai.imagenmodelparams.md new file mode 100644 index 00000000000..2d1070a3585 --- /dev/null +++ b/docs-devsite/vertexai.imagenmodelparams.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenModelParams interface + Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +Signature: + +```typescript +export interface ImagenModelParams extends ImagenModelConfig +``` +Extends: [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [model](./vertexai.imagenmodelparams.md#imagenmodelparamsmodel) | string | | + +## ImagenModelParams.model + +Signature: + +```typescript +model: string; +``` diff --git a/docs-devsite/vertexai.imagensafetysettings.md b/docs-devsite/vertexai.imagensafetysettings.md new file mode 100644 index 00000000000..0d4a45a6d52 --- /dev/null +++ b/docs-devsite/vertexai.imagensafetysettings.md @@ -0,0 +1,45 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ImagenSafetySettings interface + +Signature: + +```typescript +export interface ImagenSafetySettings +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [personFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingspersonfilterlevel) | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | Generate people. | +| [safetyFilterLevel](./vertexai.imagensafetysettings.md#imagensafetysettingssafetyfilterlevel) | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | Safety filter level | + +## ImagenSafetySettings.personFilterLevel + +Generate people. + +Signature: + +```typescript +personFilterLevel?: ImagenPersonFilterLevel; +``` + +## ImagenSafetySettings.safetyFilterLevel + +Safety filter level + +Signature: + +```typescript +safetyFilterLevel?: ImagenSafetyFilterLevel; +``` diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index d9e26eabc5d..827babbcafe 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -20,6 +20,11 @@ The Vertex AI in Firebase Web SDK. | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | | [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. | +| function() | +| [png()](./vertexai.md#png) | Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| function(compressionQuality, ...) | +| [jpeg(compressionQuality)](./vertexai.md#jpeg_8e6711f) | Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | ## Classes @@ -29,6 +34,7 @@ The Vertex AI in Firebase Web SDK. | [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | | [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | | [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | Class for Imagen model APIs. | | [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | | [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | | [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | @@ -48,6 +54,9 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -83,12 +92,23 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | Image generated by Imagen, stored in Google Cloud Storage (GCS). | +| [ImagenGCSImageResponse](./vertexai.imagengcsimageresponse.md#imagengcsimageresponse_interface) | | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) | | +| [ImagenImageReponse](./vertexai.imagenimagereponse.md#imagenimagereponse_interface) | | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | Image generated by Imagen to inline bytes. | +| [ImagenInlineImageResponse](./vertexai.imageninlineimageresponse.md#imageninlineimageresponse_interface) | Imagen image response. | +| [ImagenModelConfig](./vertexai.imagenmodelconfig.md#imagenmodelconfig_interface) | | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Copyright 2024 Google LLCLicensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | +| [SafetyAttributes](./vertexai.safetyattributes.md#safetyattributes_interface) | | | [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | | [SchemaInterface](./vertexai.schemainterface.md#schemainterface_interface) | Interface for [Schema](./vertexai.schema.md#schema_class) class. | @@ -167,6 +187,68 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) +### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} + +Returns a [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. + +Signature: + +```typescript +export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | + +Returns: + +[ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) + +## function() + +### png() {:#png} + +Creates an for a PNG image, to be included in a [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare function png(): ImagenImageFormat; +``` +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + + +## function(compressionQuality, ...) + +### jpeg(compressionQuality) {:#jpeg_8e6711f} + +Creates an [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) for a JPEG image, to be included in an [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). + +Signature: + +```typescript +export declare function jpeg(compressionQuality: number): ImagenImageFormat; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| compressionQuality | number | The level of compression. | + +Returns: + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + +[ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_interface) + ## POSSIBLE\_ROLES Possible roles. @@ -363,6 +445,59 @@ export declare enum HarmSeverity | HARM\_SEVERITY\_MEDIUM | "HARM_SEVERITY_MEDIUM" | | | HARM\_SEVERITY\_NEGLIGIBLE | "HARM_SEVERITY_NEGLIGIBLE" | | +## ImagenAspectRatio + +Signature: + +```typescript +export declare enum ImagenAspectRatio +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| CLASSIC\_LANDSCAPE | "4:3" | | +| CLASSIC\_PORTRAIT | "3:4" | | +| PORTRAIT | "9:16" | | +| SQUARE | "1:1" | | +| WIDESCREEN | "16:9" | | + +## ImagenPersonFilterLevel + + +Signature: + +```typescript +export declare enum ImagenPersonFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ALLOW\_ADULT | "allow_adult" | | +| ALLOW\_ALL | "allow_all" | | +| BLOCK\_ALL | "dont_allow" | | + +## ImagenSafetyFilterLevel + + +Signature: + +```typescript +export declare enum ImagenSafetyFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | | +| BLOCK\_NONE | "block_none" | | +| BLOCK\_ONLY\_HIGH | "block_only_high" | | + ## SchemaType Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) diff --git a/docs-devsite/vertexai.safetyattributes.md b/docs-devsite/vertexai.safetyattributes.md new file mode 100644 index 00000000000..837850ab80f --- /dev/null +++ b/docs-devsite/vertexai.safetyattributes.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# SafetyAttributes interface +Signature: + +```typescript +export interface SafetyAttributes +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [categories](./vertexai.safetyattributes.md#safetyattributescategories) | string\[\] | | +| [scores](./vertexai.safetyattributes.md#safetyattributesscores) | number\[\] | | + +## SafetyAttributes.categories + +Signature: + +```typescript +categories: string[]; +``` + +## SafetyAttributes.scores + +Signature: + +```typescript +scores: number[]; +``` diff --git a/packages/data-connect/test/unit/transportoptions.test.ts b/packages/data-connect/test/unit/transportoptions.test.ts index 91e090e6a54..a7136b5c408 100644 --- a/packages/data-connect/test/unit/transportoptions.test.ts +++ b/packages/data-connect/test/unit/transportoptions.test.ts @@ -16,6 +16,8 @@ */ import { expect } from 'chai'; + +import { queryRef } from '../../src'; import { TransportOptions, areTransportOptionsEqual, @@ -23,7 +25,6 @@ import { getDataConnect } from '../../src/api/DataConnect'; import { app } from '../util'; -import { queryRef } from '../../src'; describe('Transport Options', () => { it('should return false if transport options are not equal', () => { const transportOptions1: TransportOptions = { diff --git a/packages/vertexai/api-extractor.json b/packages/vertexai/api-extractor.json index 8a3c6cb251e..799c1fcf397 100644 --- a/packages/vertexai/api-extractor.json +++ b/packages/vertexai/api-extractor.json @@ -1,10 +1,10 @@ { - "extends": "../../config/api-extractor.json", - // Point it to your entry point d.ts file. - "mainEntryPointFilePath": "/dist/src/index.d.ts", - "dtsRollup": { - "enabled": true, - "untrimmedFilePath": "/dist/.d.ts", - "publicTrimmedFilePath": "/dist/-public.d.ts" - } -} \ No newline at end of file + "extends": "../../config/api-extractor.json", + // Point it to your entry point d.ts file. + "mainEntryPointFilePath": "/dist/src/index.d.ts", + "dtsRollup": { + "enabled": true, + "untrimmedFilePath": "/dist/.d.ts", + "publicTrimmedFilePath": "/dist/-public.d.ts" + } +} diff --git a/packages/vertexai/patch.txt b/packages/vertexai/patch.txt new file mode 100644 index 00000000000..548a03980fe --- /dev/null +++ b/packages/vertexai/patch.txt @@ -0,0 +1,1027 @@ +commit 69c0690b61c2b22ef86d0dc4f122aa22bf630356 +Author: Daniel La Rocque +Date: Mon Dec 9 11:57:18 2024 -0500 + + WIP + +diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md +index 041bc6245..bb0ec36f9 100644 +--- a/common/api-review/vertexai.api.md ++++ b/common/api-review/vertexai.api.md +@@ -348,6 +348,11 @@ export class GenerativeModel { + // @public + export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; + ++// Warning: (ae-forgotten-export) The symbol "ImagenModel" needs to be exported by the entry point index.d.ts ++// ++// @public ++export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; ++ + // @public + export function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI; + +@@ -429,6 +434,144 @@ export enum HarmSeverity { + HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" + } + ++// @public (undocumented) ++export enum ImagenAspectRatio { ++ // (undocumented) ++ CLASSIC_LANDSCAPE = "4:3", ++ // (undocumented) ++ CLASSIC_PORTRAIT = "3:4", ++ // (undocumented) ++ PORTRAIT = "9:16", ++ // (undocumented) ++ SQUARE = "1:1", ++ // (undocumented) ++ WIDESCREEN = "16:9" ++} ++ ++// @public (undocumented) ++export interface ImagenGCSImage extends ImagenImage { ++ // (undocumented) ++ gcsURI: string; ++} ++ ++// @public (undocumented) ++export interface ImagenGCSImageResponse { ++ // (undocumented) ++ filteredReason?: string; ++ // (undocumented) ++ images: ImagenGCSImage[]; ++} ++ ++// @public (undocumented) ++export interface ImagenGenerationConfig { ++ // (undocumented) ++ addWatermark?: boolean; ++ // (undocumented) ++ aspectRatio?: ImagenAspectRatio; ++ // (undocumented) ++ imageFormat?: ImagenImageFormat; ++ // (undocumented) ++ negativePrompt?: string; ++ // (undocumented) ++ numberOfImages?: number; ++} ++ ++// @public ++export interface ImagenImage { ++ // (undocumented) ++ mimeType: string; ++} ++ ++// @public (undocumented) ++export interface ImagenImageFormat { ++ // (undocumented) ++ compressionQuality?: number; ++ // (undocumented) ++ mimeType: string; ++} ++ ++// @public (undocumented) ++export interface ImagenImageReponse { ++ // (undocumented) ++ filteredReason?: string; ++ // (undocumented) ++ images: ImagenImage[]; ++} ++ ++// @public (undocumented) ++export interface ImagenInlineImage extends ImagenImage { ++ // (undocumented) ++ bytesBase64Encoded: string; ++} ++ ++// @public (undocumented) ++export interface ImagenInlineImageResponse { ++ // (undocumented) ++ filteredReason?: string; ++ // (undocumented) ++ images: ImagenInlineImage[]; ++} ++ ++// @public ++export interface ImagenModelParams { ++ // (undocumented) ++ imageFormat?: ImagenImageFormat; ++ // (undocumented) ++ modelName: string; ++ // (undocumented) ++ safetySettings?: ImagenSafetySettings; ++} ++ ++// @public (undocumented) ++export enum ImagenPersonGeneration { ++ // (undocumented) ++ ALLOW_ADULT = "allow_adult", ++ // (undocumented) ++ ALLOW_ALL = "allow_all", ++ // (undocumented) ++ BLOCK_ALL = "dont_allow" ++} ++ ++// Warning: (ae-internal-missing-underscore) The name "ImagenRequestParameters" should be prefixed with an underscore because the declaration is marked as @internal ++// ++// @internal ++export interface ImagenRequestParameters { ++ // (undocumented) ++ addWatermark?: boolean; ++ // (undocumented) ++ aspectRatio: string; ++ // (undocumented) ++ includeRaiReason: boolean; ++ // (undocumented) ++ negativePrompt?: string; ++ // (undocumented) ++ personGeneration?: string; ++ // (undocumented) ++ safetyFilterLevel?: string; ++ // (undocumented) ++ sampleCount: number; ++ // (undocumented) ++ storageUri?: string; ++} ++ ++// @public (undocumented) ++export enum ImagenSafetyFilterLevel { ++ // (undocumented) ++ BLOCK_LOW_AND_ABOVE = "block_low_and_above", ++ // (undocumented) ++ BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above", ++ // (undocumented) ++ BLOCK_NONE = "block_none", ++ // (undocumented) ++ BLOCK_ONLY_HIGH = "block_only_high" ++} ++ ++// @public (undocumented) ++export interface ImagenSafetySettings { ++ personGeneration?: ImagenPersonGeneration; ++ safetyFilterLevel?: ImagenSafetyFilterLevel; ++} ++ + // @public + export interface InlineDataPart { + // (undocumented) +@@ -447,6 +590,9 @@ export class IntegerSchema extends Schema { + constructor(schemaParams?: SchemaParams); + } + ++// @public (undocumented) ++export function jpeg(compressionQuality: number): ImagenImageFormat; ++ + // @public + export interface ModelParams extends BaseParams { + // (undocumented) +@@ -490,6 +636,9 @@ export interface ObjectSchemaInterface extends SchemaInterface { + // @public + export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; + ++// @public (undocumented) ++export function png(): ImagenImageFormat; ++ + // @public + export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; + +@@ -520,6 +669,14 @@ export interface RetrievedContextAttribution { + // @public + export type Role = (typeof POSSIBLE_ROLES)[number]; + ++// @public (undocumented) ++export interface SafetyAttributes { ++ // (undocumented) ++ categories: string[]; ++ // (undocumented) ++ scores: number[]; ++} ++ + // @public + export interface SafetyRating { + // (undocumented) +diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts +index 42f33ed2a..4d42e772c 100644 +--- a/packages/vertexai/src/api.ts ++++ b/packages/vertexai/src/api.ts +@@ -21,14 +21,22 @@ import { getModularInstance } from '@firebase/util'; + import { DEFAULT_LOCATION, VERTEX_TYPE } from './constants'; + import { VertexAIService } from './service'; + import { VertexAI, VertexAIOptions } from './public-types'; +-import { ModelParams, RequestOptions, VertexAIErrorCode } from './types'; ++import { ++ ImagenModelParams, ++ ModelParams, ++ RequestOptions, ++ VertexAIErrorCode ++} from './types'; + import { VertexAIError } from './errors'; + import { GenerativeModel } from './models/generative-model'; ++import { ImagenModel, jpeg, png } from './models/imagen-model'; + + export { ChatSession } from './methods/chat-session'; + export * from './requests/schema-builder'; + +-export { GenerativeModel }; ++export { jpeg, png }; ++ ++export { GenerativeModel, ImagenModel }; + + export { VertexAIError }; + +@@ -77,3 +85,22 @@ export function getGenerativeModel( + } + return new GenerativeModel(vertexAI, modelParams, requestOptions); + } ++ ++/** ++ * Returns a {@link ImagenModel} class with methods for using Imagen. ++ * ++ * @public ++ */ ++export function getImagenModel( ++ vertexAI: VertexAI, ++ modelParams: ImagenModelParams, ++ requestOptions?: RequestOptions ++): ImagenModel { ++ if (!modelParams.modelName) { ++ throw new VertexAIError( ++ VertexAIErrorCode.NO_MODEL, ++ `Must provide a model name. Example: getGenerativeModel({ model: 'my-model-name' })` ++ ); ++ } ++ return new ImagenModel(vertexAI, modelParams, requestOptions); ++} +diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts +new file mode 100644 +index 000000000..901ad1ee8 +--- /dev/null ++++ b/packages/vertexai/src/models/imagen-model.test.ts +@@ -0,0 +1,59 @@ ++/** ++ * @license ++ * Copyright 2024 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++import { use, expect } from 'chai'; ++import { GenerativeModel } from './generative-model'; ++import { VertexAI } from '../public-types'; ++import sinonChai from 'sinon-chai'; ++ ++use(sinonChai); ++ ++const fakeVertexAI: VertexAI = { ++ app: { ++ name: 'DEFAULT', ++ automaticDataCollectionEnabled: true, ++ options: { ++ apiKey: 'key', ++ projectId: 'my-project' ++ } ++ }, ++ location: 'us-central1' ++}; ++ ++describe('ImagenModel', () => { ++ it('handles plain model name', () => { ++ const genModel = new GenerativeModel(fakeVertexAI, { model: 'my-model' }); ++ expect(genModel.model).to.equal('publishers/google/models/my-model'); ++}); ++it('handles models/ prefixed model name', () => { ++ const genModel = new GenerativeModel(fakeVertexAI, { ++ model: 'models/my-model' ++ }); ++ expect(genModel.model).to.equal('publishers/google/models/my-model'); ++}); ++it('handles full model name', () => { ++ const genModel = new GenerativeModel(fakeVertexAI, { ++ model: 'publishers/google/models/my-model' ++ }); ++ expect(genModel.model).to.equal('publishers/google/models/my-model'); ++}); ++it('handles prefixed tuned model name', () => { ++ const genModel = new GenerativeModel(fakeVertexAI, { ++ model: 'tunedModels/my-model' ++ }); ++ expect(genModel.model).to.equal('tunedModels/my-model'); ++}); ++}); +\ No newline at end of file +diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts +new file mode 100644 +index 000000000..08d2957c4 +--- /dev/null ++++ b/packages/vertexai/src/models/imagen-model.ts +@@ -0,0 +1,201 @@ ++/** ++ * @license ++ * Copyright 2024 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++import { VertexAIError } from '../errors'; ++import { VertexAI } from '../public-types'; ++import { Task, makeRequest } from '../requests/request'; ++import { ++ createPredictRequestBody, ++ handlePredictResponse ++} from '../requests/request-helpers'; ++import { VertexAIService } from '../service'; ++import { ++ ImagenGCSImage, ++ ImagenGCSImageResponse, ++ ImagenImageFormat, ++ ImagenGenerationConfig, ++ ImagenInlineImage, ++ RequestOptions, ++ VertexAIErrorCode, ++ ImagenModelParams, ++ ImagenInlineImageResponse, ++ ImagenModelConfig ++} from '../types'; ++import { ApiSettings } from '../types/internal'; ++ ++/** ++ * Class for Imagen model APIs. ++ * @public ++ */ ++export class ImagenModel { ++ model: string; ++ private _apiSettings: ApiSettings; ++ private modelConfig: ImagenModelConfig; ++ ++ /** ++ * ++ * @param vertexAI ++ * @param modelParams ++ * @param requestOptions ++ */ ++ constructor( ++ vertexAI: VertexAI, ++ modelParams: ImagenModelParams, ++ private requestOptions?: RequestOptions ++ ) { ++ const { modelName, ...modelConfig } = modelParams; ++ this.modelConfig = modelConfig; ++ if (modelName.includes('/')) { ++ if (modelName.startsWith('models/')) { ++ // Add "publishers/google" if the user is only passing in 'models/model-name'. ++ this.model = `publishers/google/${modelName}`; ++ } else { ++ // Any other custom format (e.g. tuned models) must be passed in correctly. ++ this.model = modelName; ++ } ++ } else { ++ // If path is not included, assume it's a non-tuned model. ++ this.model = `publishers/google/models/${modelName}`; ++ } ++ ++ if (!vertexAI.app?.options?.apiKey) { ++ throw new VertexAIError( ++ VertexAIErrorCode.NO_API_KEY, ++ `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` ++ ); ++ } else if (!vertexAI.app?.options?.projectId) { ++ throw new VertexAIError( ++ VertexAIErrorCode.NO_PROJECT_ID, ++ `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` ++ ); ++ } else { ++ this._apiSettings = { ++ apiKey: vertexAI.app.options.apiKey, ++ project: vertexAI.app.options.projectId, ++ location: vertexAI.location ++ }; ++ if ((vertexAI as VertexAIService).appCheck) { ++ this._apiSettings.getAppCheckToken = () => ++ (vertexAI as VertexAIService).appCheck!.getToken(); ++ } ++ ++ if ((vertexAI as VertexAIService).auth) { ++ this._apiSettings.getAuthToken = () => ++ (vertexAI as VertexAIService).auth!.getToken(); ++ } ++ } ++ } ++ ++ /** ++ * Generates images using the Imagen model and returns them as base64-encoded strings. ++ * ++ * @param prompt The text prompt used to generate the images. ++ * @param imagenRequestOptions Configuration options for the Imagen generation request. ++ * See {@link ImagenGenerationConfig}. ++ * @returns A promise that resolves to an {@link ImagenInlineImageResponse} object containing the generated images. ++ * ++ * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. ++ * ++ * @remarks ++ * If one or more images are filtered, the returned object will have a defined `filteredReason` property. ++ * If all images are filtered, the `images` array will be empty, and no error will be thrown. ++ */ ++ async generateImages( ++ prompt: string, ++ imagenRequestOptions: ImagenGenerationConfig ++ ): Promise { ++ const body = createPredictRequestBody({ ++ prompt, ++ ...imagenRequestOptions, ++ ...this.modelConfig ++ }); ++ const response = await makeRequest( ++ this.model, ++ Task.PREDICT, ++ this._apiSettings, ++ /* stream */ false, ++ JSON.stringify(body), ++ this.requestOptions ++ ); ++ return handlePredictResponse(response); ++ } ++ ++ /** ++ * Generates images using the Imagen model and returns them as base64-encoded strings. ++ * ++ * @param prompt The text prompt used to generate the images. ++ * @param gcsURI The GCS URI where the images should be stored. ++ * @param imagenRequestOptions Configuration options for the Imagen generation request. ++ * See {@link ImagenGenerationConfig}. ++ * @returns A promise that resolves to an {@link ImagenGCSImageResponse} object containing the generated images. ++ * ++ * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. ++ * ++ * @remarks ++ * If one or more images are filtered, the returned object will have a defined `filteredReason` property. ++ * If all images are filtered, the `images` array will be empty, and no error will be thrown. ++ */ ++ async generateImagesGCS( ++ prompt: string, ++ gcsURI: string, ++ imagenRequestOptions: ImagenGenerationConfig ++ ): Promise { ++ const body = createPredictRequestBody({ ++ prompt, ++ gcsURI, ++ ...imagenRequestOptions, ++ ...this.modelConfig ++ }); ++ const response = await makeRequest( ++ this.model, ++ Task.PREDICT, ++ this._apiSettings, ++ /* stream */ false, ++ JSON.stringify(body), ++ this.requestOptions ++ ); ++ return handlePredictResponse(response); ++ } ++} ++ ++/** ++ * Creates an {@link ImagenImageFormat} for a JPEG image, to be included in an {@link ImagenModelParams}. ++ * ++ * @param compressionQuality The level of compression. ++ * @returns {@link ImagenImageFormat} ++ * ++ * @public ++ */ ++export function jpeg(compressionQuality: number): ImagenImageFormat { ++ return { ++ mimeType: 'image/jpeg', ++ compressionQuality ++ }; ++} ++ ++/** ++ * Creates an {@link ImageImageFormat} for a PNG image, to be included in a {@link ImagenModelParams}. ++ * ++ * @returns {@link ImageImageFormat} ++ * ++ * @public ++ */ ++export function png(): ImagenImageFormat { ++ return { ++ mimeType: 'image/png' ++ }; ++} +diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts +index 9e525b2a8..96df278dd 100644 +--- a/packages/vertexai/src/requests/request-helpers.ts ++++ b/packages/vertexai/src/requests/request-helpers.ts +@@ -18,10 +18,16 @@ + import { + Content, + GenerateContentRequest, ++ PredictRequestBody, + Part, +- VertexAIErrorCode ++ VertexAIErrorCode, ++ ImagenAspectRatio, ++ ImagenInlineImage, ++ ImagenGCSImage, ++ ImagenRequestConfig + } from '../types'; + import { VertexAIError } from '../errors'; ++import { ImagenResponseInternal } from '../types/internal'; + + export function formatSystemInstruction( + input?: string | Part | Content +@@ -49,11 +55,13 @@ export function formatNewContent( + if (typeof request === 'string') { + newParts = [{ text: request }]; + } else { +- for (const partOrString of request) { +- if (typeof partOrString === 'string') { +- newParts.push({ text: partOrString }); ++ for (const elem of request) { ++ // This throws an error if request is not iterable ++ if (typeof elem === 'string') { ++ newParts.push({ text: elem }); + } else { +- newParts.push(partOrString); ++ // We assume this is a Part, but it could be anything. ++ newParts.push(elem); // This could be + } + } + } +@@ -114,6 +122,7 @@ export function formatGenerateContentInput( + formattedRequest = params as GenerateContentRequest; + } else { + // Array or string ++ // ... or something else + const content = formatNewContent(params as string | Array); + formattedRequest = { contents: [content] }; + } +@@ -124,3 +133,88 @@ export function formatGenerateContentInput( + } + return formattedRequest; + } ++ ++/** ++ * Convert the user-defined parameters in {@link ImagenRequestConfig} to the format ++ * that is expected from the REST API. ++ * ++ * @internal ++ */ ++export function createPredictRequestBody({ ++ prompt, ++ gcsURI, ++ imageFormat = { mimeType: 'image/png' }, ++ addWatermark, ++ safetySettings, ++ numberOfImages = 1, ++ negativePrompt, ++ aspectRatio = ImagenAspectRatio.SQUARE ++}: ImagenRequestConfig): PredictRequestBody { ++ // Properties that are undefined will be omitted from the JSON string. ++ const body: PredictRequestBody = { ++ instances: [ ++ { ++ prompt ++ } ++ ], ++ parameters: { ++ storageUri: gcsURI, ++ ...imageFormat, ++ addWatermark, ++ ...safetySettings, ++ sampleCount: numberOfImages, ++ includeRaiReason: true, ++ negativePrompt, ++ aspectRatio ++ } ++ }; ++ return body; ++} ++ ++/** ++ * Convert a generic successful fetch {@link Response} body to an Imagen response object ++ * that can be returned to the user. This converts the REST APIs response format to our ++ * representation of a response. ++ * ++ * @internal ++ */ ++export async function handlePredictResponse< ++ T extends ImagenInlineImage | ImagenGCSImage ++>(response: Response): Promise<{ images: T[]; filteredReason?: string }> { ++ const responseJson: ImagenResponseInternal = await response.json(); ++ ++ const images: T[] = []; ++ let filteredReason: string | undefined = undefined; ++ ++ if (!responseJson.predictions || responseJson.predictions?.length === 0) { ++ throw new VertexAIError( ++ VertexAIErrorCode.ERROR, ++ "Predictions array is undefined or empty in response. Was 'includeRaiReason' enabled in the request?" ++ ); ++ } ++ ++ for (const prediction of responseJson.predictions) { ++ if (prediction.raiFilteredReason) { ++ filteredReason = prediction.raiFilteredReason; ++ } else if (prediction.mimeType && prediction.bytesBase64Encoded) { ++ images.push({ ++ mimeType: prediction.mimeType, ++ bytesBase64Encoded: prediction.bytesBase64Encoded ++ } as T); ++ } else if (prediction.mimeType && prediction.gcsUri) { ++ images.push({ ++ mimeType: prediction.mimeType, ++ gcsURI: prediction.gcsUri ++ } as T); ++ } else { ++ throw new VertexAIError( ++ VertexAIErrorCode.RESPONSE_ERROR, ++ `Predictions array in response has missing properties. Response: ${JSON.stringify( ++ responseJson ++ )}` ++ ); ++ } ++ } ++ ++ return { images, filteredReason }; ++} +diff --git a/packages/vertexai/src/requests/request.ts b/packages/vertexai/src/requests/request.ts +index f81b40635..9b9465db7 100644 +--- a/packages/vertexai/src/requests/request.ts ++++ b/packages/vertexai/src/requests/request.ts +@@ -30,7 +30,8 @@ import { logger } from '../logger'; + export enum Task { + GENERATE_CONTENT = 'generateContent', + STREAM_GENERATE_CONTENT = 'streamGenerateContent', +- COUNT_TOKENS = 'countTokens' ++ COUNT_TOKENS = 'countTokens', ++ PREDICT = 'predict' + } + + export class RequestUrl { +diff --git a/packages/vertexai/src/types/imagen/index.ts b/packages/vertexai/src/types/imagen/index.ts +new file mode 100644 +index 000000000..0037b185e +--- /dev/null ++++ b/packages/vertexai/src/types/imagen/index.ts +@@ -0,0 +1,19 @@ ++/** ++ * @license ++ * Copyright 2024 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++export * from './requests'; ++export * from './responses'; +diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts +new file mode 100644 +index 000000000..1171df812 +--- /dev/null ++++ b/packages/vertexai/src/types/imagen/internal.ts +@@ -0,0 +1,54 @@ ++/** ++ * @license ++ * Copyright 2024 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++// Internal Imagen types ++ ++/** ++ * A response from the REST API is expected to look like this in the success case: ++ * { ++ * "predictions": [ ++ * { ++ * "mimeType": "image/png", ++ * "bytesBase64Encoded": "iVBORw0KG..." ++ * }, ++ * { ++ * "mimeType": "image/png", ++ * "bytesBase64Encoded": "i4BOtw0KG..." ++ * } ++ * ] ++ * } ++ * ++ * And like this in the failure case: ++ * { ++ * "predictions": [ ++ * { ++ * "raiFilteredReason": "..." ++ * } ++ * ] ++ * } ++ */ ++export interface ImagenResponseInternal { ++ predictions?: Array<{ ++ // Defined if the prediction was not filtered ++ mimeType?: string; ++ bytesBase64Encoded?: string; ++ gcsUri?: string; ++ ++ // Defined if the prediction was filtered, and there is no image ++ raiFilteredReason?: string; ++ }>; ++} +diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts +new file mode 100644 +index 000000000..d3d2d3562 +--- /dev/null ++++ b/packages/vertexai/src/types/imagen/requests.ts +@@ -0,0 +1,141 @@ ++/** ++ * @license ++ * Copyright 2024 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++export interface ImagenModelParams extends ImagenModelConfig { ++ modelName: string; ++} ++ ++export interface ImagenModelConfig { ++ imageFormat?: ImagenImageFormat; ++ addWatermark?: boolean; ++ safetySettings?: ImagenSafetySettings; ++} ++ ++export interface ImagenGenerationConfig { ++ numberOfImages?: number; // Default to 1. Possible values are [1...4] ++ negativePrompt?: string; // Default to null ++ aspectRatio?: ImagenAspectRatio; // Default to "1:1" ++} ++ ++/** ++ * Contains all possible REST API paramaters. ++ * This is the intersection of the model-level (`ImagenModelParams`), ++ * request-level (`ImagenGenerationConfig`) configurations, along with ++ * the other required parameters prompt and gcsURI (for GCS generation only). ++ * ++ * @internal ++ */ ++export interface ImagenRequestConfig ++ extends ImagenModelConfig, ++ ImagenGenerationConfig { ++ prompt: string; ++ gcsURI?: string; ++} ++ ++export interface ImagenImageFormat { ++ mimeType: string; // image/png, or image/jpeg, default image/png ++ compressionQuality?: number; // 0-100, default 75. Only for image/jpeg ++} ++ ++/** ++ * @public ++ */ ++export enum ImagenSafetyFilterLevel { ++ BLOCK_LOW_AND_ABOVE = 'block_low_and_above', ++ BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', ++ BLOCK_ONLY_HIGH = 'block_only_high', ++ BLOCK_NONE = 'block_none' ++} ++ ++/** ++ * @public ++ */ ++export enum ImagenPersonGeneration { ++ BLOCK_ALL = 'dont_allow', ++ ALLOW_ADULT = 'allow_adult', ++ ALLOW_ALL = 'allow_all' ++} ++ ++/** ++ * @public ++ */ ++export interface ImagenSafetySettings { ++ /** ++ * Safety filter level ++ */ ++ safetyFilterLevel?: ImagenSafetyFilterLevel; ++ /** ++ * Generate people. ++ */ ++ personGeneration?: ImagenPersonGeneration; ++} ++ ++export enum ImagenAspectRatio { ++ SQUARE = '1:1', ++ CLASSIC_PORTRAIT = '3:4', ++ CLASSIC_LANDSCAPE = '4:3', ++ WIDESCREEN = '16:9', ++ PORTRAIT = '9:16' ++} ++ ++/** ++ * The parameters to be sent in the request body of the HTTP call ++ * to the Vertex AI backend. ++ * ++ * We need a seperate internal-only interface for this because the REST ++ * API expects different parameter names than what we show to our users. ++ * ++ * This interface should be populated from the {@link ImagenGenerationConfig} that ++ * the user defines. ++ * ++ * Sample request body JSON: ++ * { ++ * "instances": [ ++ * { ++ * "prompt": "Portrait of a golden retriever on a beach." ++ * } ++ * ], ++ * "parameters": { ++ * "mimeType": "image/png", ++ * "safetyFilterLevel": "block_low_and_above", ++ * "personGeneration": "allow_all", ++ * "sampleCount": 2, ++ * "includeRaiReason": true, ++ * "aspectRatio": "9:16" ++ * } ++ * } ++ * ++ * @internal ++ */ ++export interface PredictRequestBody { ++ instances: [ ++ { ++ prompt: string; ++ storageUri?: string; ++ } ++ ]; ++ parameters: { ++ sampleCount: number; // maps to numberOfImages ++ aspectRatio: string; ++ negativePrompt?: string; ++ storageUri?: string; ++ addWatermark?: boolean; ++ safetyFilterLevel?: string; ++ personGeneration?: string; ++ includeRaiReason: boolean; ++ }; ++} +diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts +new file mode 100644 +index 000000000..9dcc6a144 +--- /dev/null ++++ b/packages/vertexai/src/types/imagen/responses.ts +@@ -0,0 +1,73 @@ ++/** ++ * @license ++ * Copyright 2024 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++/** ++ * Base class for types of images that the Imagen Model can return. ++ * ++ * @internal ++ */ ++export interface ImagenImage { ++ mimeType: string; ++} ++ ++/** ++ * Image generated by Imagen to inline bytes. ++ * ++ * @public ++ */ ++export interface ImagenInlineImage extends ImagenImage { ++ bytesBase64Encoded: string; ++} ++ ++/** ++ * Image generated by Imagen, stored in Google Cloud Storage (GCS). ++ * ++ * @public ++ */ ++export interface ImagenGCSImage extends ImagenImage { ++ /** ++ * The Google Cloud Storage (GCS) URI at which the generated image is stored. ++ */ ++ gcsURI: string; ++} ++ ++/** ++ * Imagen image response. ++ * ++ * @public ++ */ ++export interface ImagenInlineImageResponse { ++ /** ++ * The images generated by Imagen. If all images were filtered, this will be empty. ++ */ ++ images: ImagenInlineImage[]; ++ /** ++ * The reason the missing images were filtered. ++ * For the mappings of error codes to reasons, see {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. ++ */ ++ filteredReason?: string; ++} ++ ++export interface ImagenGCSImageResponse { ++ images: ImagenGCSImage[]; ++ filteredReason?: string; ++} ++ ++export interface ImagenImageReponse { ++ images: ImagenImage[]; ++ filteredReason?: string; ++} +diff --git a/packages/vertexai/src/types/index.ts b/packages/vertexai/src/types/index.ts +index 85133aa07..f575c5ba8 100644 +--- a/packages/vertexai/src/types/index.ts ++++ b/packages/vertexai/src/types/index.ts +@@ -21,3 +21,4 @@ export * from './requests'; + export * from './responses'; + export * from './error'; + export * from './schema'; ++export * from './imagen'; +diff --git a/packages/vertexai/src/types/internal.ts b/packages/vertexai/src/types/internal.ts +index 8271175fe..87c28a02a 100644 +--- a/packages/vertexai/src/types/internal.ts ++++ b/packages/vertexai/src/types/internal.ts +@@ -18,6 +18,8 @@ + import { AppCheckTokenResult } from '@firebase/app-check-interop-types'; + import { FirebaseAuthTokenData } from '@firebase/auth-interop-types'; + ++export * from './imagen/internal'; ++ + export interface ApiSettings { + apiKey: string; + project: string; +diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts +index 83cd4366f..e2a442821 100644 +--- a/packages/vertexai/src/types/responses.ts ++++ b/packages/vertexai/src/types/responses.ts +@@ -46,6 +46,11 @@ export interface GenerateContentStreamResult { + response: Promise; + } + ++export interface SafetyAttributes { ++ categories: string[]; ++ scores: number[]; ++} ++ + /** + * Response object wrapped with helper methods. + * diff --git a/packages/vertexai/src/api.test.ts b/packages/vertexai/src/api.test.ts index b6c96923856..c9432d2a7ea 100644 --- a/packages/vertexai/src/api.test.ts +++ b/packages/vertexai/src/api.test.ts @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ModelParams, VertexAIErrorCode } from './types'; +import { ImagenModelParams, ModelParams, VertexAIErrorCode } from './types'; import { VertexAIError } from './errors'; -import { getGenerativeModel } from './api'; +import { ImagenModel, getGenerativeModel, getImagenModel } from './api'; import { expect } from 'chai'; import { VertexAI } from './public-types'; import { GenerativeModel } from './models/generative-model'; @@ -84,4 +84,54 @@ describe('Top level API', () => { expect(genModel).to.be.an.instanceOf(GenerativeModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); }); + it('getImagenModel throws if no model is provided', () => { + try { + getImagenModel(fakeVertexAI, {} as ImagenModelParams); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_MODEL); + expect((e as VertexAIError).message).includes( + `VertexAI: Must provide a model name. Example: ` + + `getImagenModel({ model: 'my-model-name' }) (vertexAI/${VertexAIErrorCode.NO_MODEL})` + ); + } + }); + it('getImagenModel throws if no apiKey is provided', () => { + const fakeVertexNoApiKey = { + ...fakeVertexAI, + app: { options: { projectId: 'my-project' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoApiKey, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes(VertexAIErrorCode.NO_API_KEY); + expect((e as VertexAIError).message).equals( + `VertexAI: The "apiKey" field is empty in the local ` + + `Firebase config. Firebase VertexAI requires this field to` + + ` contain a valid API key. (vertexAI/${VertexAIErrorCode.NO_API_KEY})` + ); + } + }); + it('getImagenModel throws if no projectId is provided', () => { + const fakeVertexNoProject = { + ...fakeVertexAI, + app: { options: { apiKey: 'my-key' } } + } as VertexAI; + try { + getImagenModel(fakeVertexNoProject, { model: 'my-model' }); + } catch (e) { + expect((e as VertexAIError).code).includes( + VertexAIErrorCode.NO_PROJECT_ID + ); + expect((e as VertexAIError).message).equals( + `VertexAI: The "projectId" field is empty in the local` + + ` Firebase config. Firebase VertexAI requires this field ` + + `to contain a valid project ID. (vertexAI/${VertexAIErrorCode.NO_PROJECT_ID})` + ); + } + }); + it('getGenerativeModel gets an ImagenModel', () => { + const genModel = getImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(genModel).to.be.an.instanceOf(ImagenModel); + expect(genModel.model).to.equal('publishers/google/models/my-model'); + }); }); diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 42f33ed2a85..07154356435 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -21,14 +21,22 @@ import { getModularInstance } from '@firebase/util'; import { DEFAULT_LOCATION, VERTEX_TYPE } from './constants'; import { VertexAIService } from './service'; import { VertexAI, VertexAIOptions } from './public-types'; -import { ModelParams, RequestOptions, VertexAIErrorCode } from './types'; +import { + ImagenModelParams, + ModelParams, + RequestOptions, + VertexAIErrorCode +} from './types'; import { VertexAIError } from './errors'; import { GenerativeModel } from './models/generative-model'; +import { ImagenModel, jpeg, png } from './models/imagen-model'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; -export { GenerativeModel }; +export { jpeg, png }; + +export { GenerativeModel, ImagenModel }; export { VertexAIError }; @@ -77,3 +85,22 @@ export function getGenerativeModel( } return new GenerativeModel(vertexAI, modelParams, requestOptions); } + +/** + * Returns a {@link ImagenModel} class with methods for using Imagen. + * + * @public + */ +export function getImagenModel( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + requestOptions?: RequestOptions +): ImagenModel { + if (!modelParams.model) { + throw new VertexAIError( + VertexAIErrorCode.NO_MODEL, + `Must provide a model name. Example: getImagenModel({ model: 'my-model-name' })` + ); + } + return new ImagenModel(vertexAI, modelParams, requestOptions); +} diff --git a/packages/vertexai/src/models/imagen-model.test.ts b/packages/vertexai/src/models/imagen-model.test.ts new file mode 100644 index 00000000000..909bff3dea2 --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.test.ts @@ -0,0 +1,246 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { use, expect } from 'chai'; +import { ImagenModel } from './imagen-model'; +import { + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel, + VertexAI, + VertexAIErrorCode +} from '../public-types'; +import * as request from '../requests/request'; +import sinonChai from 'sinon-chai'; +import { VertexAIError } from '../errors'; +import { getMockResponse } from '../../test-utils/mock-response'; +import { match, restore, stub } from 'sinon'; + +use(sinonChai); + +const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key', + projectId: 'my-project' + } + }, + location: 'us-central1' +}; + +describe('ImagenModel', () => { + it('handles plain model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { model: 'my-model' }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles models/ prefixed model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'models/my-model' + }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles full model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'publishers/google/models/my-model' + }); + expect(imagenModel.model).to.equal('publishers/google/models/my-model'); + }); + it('handles prefixed tuned model name', () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'tunedModels/my-model' + }); + expect(imagenModel.model).to.equal('tunedModels/my-model'); + }); + it('throws if not passed an api key', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + projectId: 'my-project' + } + }, + location: 'us-central1' + }; + try { + new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.NO_API_KEY); + } + }); + it('throws if not passed a project ID', () => { + const fakeVertexAI: VertexAI = { + app: { + name: 'DEFAULT', + automaticDataCollectionEnabled: true, + options: { + apiKey: 'key' + } + }, + location: 'us-central1' + }; + try { + new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + } catch (e) { + expect((e as VertexAIError).code).to.equal( + VertexAIErrorCode.NO_PROJECT_ID + ); + } + }); + it('generateImages makes a request to predict with default parameters', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('sampleCount') && + value.includes('includeRaiReason') && + value.includes('aspectRatio') && + value.includes('mimeType') + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with model-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model', + addWatermark: true, + imageFormat: { mimeType: 'image/jpeg', compressionQuality: 75 }, + safetySettings: { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_ONLY_HIGH, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + } + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { numberOfImages: 1 }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('safetyFilterLevel') && + value.includes('block_only_high') && + value.includes('personFilterLevel') && + value.includes('allow_adult') && + value.includes('mimeType') && + value.includes('image/jpeg') && + value.includes('addWatermark') && + value.includes('true') + ); + }), + undefined + ); + restore(); + }); + it('generateImages makes a request to predict with request-level image configs', async () => { + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const prompt = 'A photorealistic image of a toy boat at sea.'; + await imagenModel.generateImages(prompt, { + numberOfImages: 4, + aspectRatio: ImagenAspectRatio.WIDESCREEN, + negativePrompt: 'do not hallucinate' + }); + expect(makeRequestStub).to.be.calledWith( + 'publishers/google/models/my-model', + request.Task.PREDICT, + match.any, + false, + match((value: string) => { + return ( + value.includes('sampleCount') && + value.includes('4') && + value.includes('aspectRatio') && + value.includes('16:9') && + value.includes('negativePrompt') && + value.includes('do not hallucinate') + ); + }), + undefined + ); + + restore(); + }); + it('throws if prompt blocked', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-prompt-blocked.json' + ); + + stub(globalThis, 'fetch').resolves({ + ok: false, + status: 400, // TODO (dlarocque): Get this from the mock response + statusText: 'Bad Request', + json: mockResponse.json + } as Response); + + const imagenModel = new ImagenModel(fakeVertexAI, { + model: 'my-model' + }); + try { + await imagenModel.generateImages( + 'A photorealistic image of a toy boat at sea.', + { numberOfImages: 1 } + ); + } catch (e) { + expect((e as VertexAIError).code).to.equal(VertexAIErrorCode.FETCH_ERROR); + expect((e as VertexAIError).message).to.include('400'); + expect((e as VertexAIError).message).to.include( + "Image generation failed with the following error: The prompt could not be submitted. This prompt contains sensitive words that violate Google's Responsible AI practices. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 42876398" + ); + } finally { + restore(); + } + }); +}); diff --git a/packages/vertexai/src/models/imagen-model.ts b/packages/vertexai/src/models/imagen-model.ts new file mode 100644 index 00000000000..17d1e1cfb2c --- /dev/null +++ b/packages/vertexai/src/models/imagen-model.ts @@ -0,0 +1,199 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { VertexAIError } from '../errors'; +import { VertexAI } from '../public-types'; +import { Task, makeRequest } from '../requests/request'; +import { createPredictRequestBody } from '../requests/request-helpers'; +import { handlePredictResponse } from '../requests/response-helpers'; +import { VertexAIService } from '../service'; +import { + ImagenGCSImage, + ImagenGCSImageResponse, + ImagenImageFormat, + ImagenGenerationConfig, + ImagenInlineImage, + RequestOptions, + VertexAIErrorCode, + ImagenModelParams, + ImagenInlineImageResponse, + ImagenModelConfig +} from '../types'; +import { ApiSettings } from '../types/internal'; + +/** + * Class for Imagen model APIs. + * @public + */ +export class ImagenModel { + model: string; + private _apiSettings: ApiSettings; + private modelConfig: ImagenModelConfig; + + /** + * + * @param vertexAI + * @param modelParams + * @param requestOptions + */ + constructor( + vertexAI: VertexAI, + modelParams: ImagenModelParams, + private requestOptions?: RequestOptions + ) { + const { model, ...modelConfig } = modelParams; + this.modelConfig = modelConfig; + if (model.includes('/')) { + if (model.startsWith('models/')) { + // Add "publishers/google" if the user is only passing in 'models/model-name'. + this.model = `publishers/google/${model}`; + } else { + // Any other custom format (e.g. tuned models) must be passed in correctly. + this.model = model; + } + } else { + // If path is not included, assume it's a non-tuned model. + this.model = `publishers/google/models/${model}`; + } + + if (!vertexAI.app?.options?.apiKey) { + throw new VertexAIError( + VertexAIErrorCode.NO_API_KEY, + `The "apiKey" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid API key.` + ); + } else if (!vertexAI.app?.options?.projectId) { + throw new VertexAIError( + VertexAIErrorCode.NO_PROJECT_ID, + `The "projectId" field is empty in the local Firebase config. Firebase VertexAI requires this field to contain a valid project ID.` + ); + } else { + this._apiSettings = { + apiKey: vertexAI.app.options.apiKey, + project: vertexAI.app.options.projectId, + location: vertexAI.location + }; + if ((vertexAI as VertexAIService).appCheck) { + this._apiSettings.getAppCheckToken = () => + (vertexAI as VertexAIService).appCheck!.getToken(); + } + + if ((vertexAI as VertexAIService).auth) { + this._apiSettings.getAuthToken = () => + (vertexAI as VertexAIService).auth!.getToken(); + } + } + } + + /** + * Generates images using the Imagen model and returns them as base64-encoded strings. + * + * @param prompt The text prompt used to generate the images. + * @param imagenRequestOptions Configuration options for the Imagen generation request. + * See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenInlineImageResponse} object containing the generated images. + * + * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * + * @remarks + * If one or more images are filtered, the returned object will have a defined `filteredReason` property. + * If all images are filtered, the `images` array will be empty, and no error will be thrown. + */ + async generateImages( + prompt: string, + imagenRequestOptions?: ImagenGenerationConfig + ): Promise { + const body = createPredictRequestBody({ + prompt, + ...imagenRequestOptions, + ...this.modelConfig + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } + + /** + * Generates images using the Imagen model and returns them as base64-encoded strings. + * + * @param prompt The text prompt used to generate the images. + * @param gcsURI The GCS URI where the images should be stored. + * @param imagenRequestOptions Configuration options for the Imagen generation request. + * See {@link ImagenGenerationConfig}. + * @returns A promise that resolves to an {@link ImagenGCSImageResponse} object containing the generated images. + * + * @throws If the request fails or if the prompt is blocked, throws a {@link VertexAIError}. + * + * @remarks + * If one or more images are filtered, the returned object will have a defined `filteredReason` property. + * If all images are filtered, the `images` array will be empty, and no error will be thrown. + */ + async generateImagesGCS( + prompt: string, + gcsURI: string, + imagenRequestOptions?: ImagenGenerationConfig + ): Promise { + const body = createPredictRequestBody({ + prompt, + gcsURI, + ...imagenRequestOptions, + ...this.modelConfig + }); + const response = await makeRequest( + this.model, + Task.PREDICT, + this._apiSettings, + /* stream */ false, + JSON.stringify(body), + this.requestOptions + ); + return handlePredictResponse(response); + } +} + +/** + * Creates an {@link ImagenImageFormat} for a JPEG image, to be included in an {@link ImagenModelParams}. + * + * @param compressionQuality The level of compression. + * @returns {@link ImagenImageFormat} + * + * @public + */ +export function jpeg(compressionQuality: number): ImagenImageFormat { + return { + mimeType: 'image/jpeg', + compressionQuality + }; +} + +/** + * Creates an {@link ImageImageFormat} for a PNG image, to be included in a {@link ImagenModelParams}. + * + * @returns {@link ImageImageFormat} + * + * @public + */ +export function png(): ImagenImageFormat { + return { + mimeType: 'image/png' + }; +} diff --git a/packages/vertexai/src/requests/request-helpers.test.ts b/packages/vertexai/src/requests/request-helpers.test.ts index 76b2f0ca1bf..9582f384a2c 100644 --- a/packages/vertexai/src/requests/request-helpers.test.ts +++ b/packages/vertexai/src/requests/request-helpers.test.ts @@ -17,8 +17,16 @@ import { expect, use } from 'chai'; import sinonChai from 'sinon-chai'; -import { Content } from '../types'; -import { formatGenerateContentInput } from './request-helpers'; +import { + Content, + ImagenAspectRatio, + ImagenPersonFilterLevel, + ImagenSafetyFilterLevel +} from '../types'; +import { + createPredictRequestBody, + formatGenerateContentInput +} from './request-helpers'; use(sinonChai); @@ -199,4 +207,71 @@ describe('request formatting methods', () => { }); }); }); + describe('createPredictRequestBody', () => { + it('creates body with default request parameters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const body = createPredictRequestBody({ + prompt + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.sampleCount).to.equal(1); + expect(body.parameters.mimeType).to.equal('image/png'); + expect(body.parameters.includeRaiReason).to.be.true; + expect(body.parameters.aspectRatio).to.equal('1:1'); + + // Parameters without default values should be undefined + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.compressionQuality).to.be.undefined; + expect(body.parameters.negativePrompt).to.be.undefined; + expect(body.parameters.storageUri).to.be.undefined; + expect(body.parameters.addWatermark).to.be.undefined; + expect(body.parameters.safetyFilterLevel).to.be.undefined; + expect(body.parameters.personGeneration).to.be.undefined; + }); + }); + it('creates body with non-default request paramaters', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const imageFormat = { mimeType: 'image/jpeg', compressionQuality: 75 }; + const safetySettings = { + safetyFilterLevel: ImagenSafetyFilterLevel.BLOCK_LOW_AND_ABOVE, + personFilterLevel: ImagenPersonFilterLevel.ALLOW_ADULT + }; + const addWatermark = true; + const numberOfImages = 4; + const negativePrompt = 'do not hallucinate'; + const aspectRatio = ImagenAspectRatio.WIDESCREEN; + const body = createPredictRequestBody({ + prompt, + numberOfImages, + imageFormat, + safetySettings, + addWatermark, + negativePrompt, + aspectRatio + }); + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters).deep.equal({ + sampleCount: numberOfImages, + mimeType: imageFormat.mimeType, + compressionQuality: imageFormat.compressionQuality, + addWatermark, + negativePrompt, + safetyFilterLevel: safetySettings.safetyFilterLevel, + personFilterLevel: safetySettings.personFilterLevel, + aspectRatio, + includeRaiReason: true, + storageUri: undefined + }); + }); + it('creates body with GCS URI', () => { + const prompt = 'A photorealistic image of a toy boat at sea.'; + const gcsURI = 'gcs-uri'; + const body = createPredictRequestBody({ + prompt, + gcsURI + }); + + expect(body.instances[0].prompt).to.equal(prompt); + expect(body.parameters.storageUri).to.equal(gcsURI); + }); }); diff --git a/packages/vertexai/src/requests/request-helpers.ts b/packages/vertexai/src/requests/request-helpers.ts index 9e525b2a875..fa4bdf402d7 100644 --- a/packages/vertexai/src/requests/request-helpers.ts +++ b/packages/vertexai/src/requests/request-helpers.ts @@ -18,8 +18,11 @@ import { Content, GenerateContentRequest, + PredictRequestBody, Part, - VertexAIErrorCode + VertexAIErrorCode, + ImagenAspectRatio, + ImagenRequestConfig } from '../types'; import { VertexAIError } from '../errors'; @@ -49,11 +52,13 @@ export function formatNewContent( if (typeof request === 'string') { newParts = [{ text: request }]; } else { - for (const partOrString of request) { - if (typeof partOrString === 'string') { - newParts.push({ text: partOrString }); + for (const elem of request) { + // This throws an error if request is not iterable + if (typeof elem === 'string') { + newParts.push({ text: elem }); } else { - newParts.push(partOrString); + // We assume this is a Part, but it could be anything. + newParts.push(elem); // This could be } } } @@ -114,6 +119,7 @@ export function formatGenerateContentInput( formattedRequest = params as GenerateContentRequest; } else { // Array or string + // ... or something else const content = formatNewContent(params as string | Array); formattedRequest = { contents: [content] }; } @@ -124,3 +130,40 @@ export function formatGenerateContentInput( } return formattedRequest; } + +/** + * Convert the user-defined parameters in {@link ImagenRequestConfig} to the format + * that is expected from the REST API. + * + * @internal + */ +export function createPredictRequestBody({ + prompt, + gcsURI, + imageFormat = { mimeType: 'image/png' }, + addWatermark, + safetySettings, + numberOfImages = 1, + negativePrompt, + aspectRatio = ImagenAspectRatio.SQUARE +}: ImagenRequestConfig): PredictRequestBody { + // Properties that are undefined will be omitted from the JSON string. + const body: PredictRequestBody = { + instances: [ + { + prompt + } + ], + parameters: { + storageUri: gcsURI, + ...imageFormat, + addWatermark, + ...safetySettings, + sampleCount: numberOfImages, + includeRaiReason: true, + negativePrompt, + aspectRatio + } + }; + return body; +} diff --git a/packages/vertexai/src/requests/request.ts b/packages/vertexai/src/requests/request.ts index f81b40635e3..9b9465db776 100644 --- a/packages/vertexai/src/requests/request.ts +++ b/packages/vertexai/src/requests/request.ts @@ -30,7 +30,8 @@ import { logger } from '../logger'; export enum Task { GENERATE_CONTENT = 'generateContent', STREAM_GENERATE_CONTENT = 'streamGenerateContent', - COUNT_TOKENS = 'countTokens' + COUNT_TOKENS = 'countTokens', + PREDICT = 'predict' } export class RequestUrl { diff --git a/packages/vertexai/src/requests/response-helpers.test.ts b/packages/vertexai/src/requests/response-helpers.test.ts index 91a60d2cfce..4cab8cde047 100644 --- a/packages/vertexai/src/requests/response-helpers.test.ts +++ b/packages/vertexai/src/requests/response-helpers.test.ts @@ -15,7 +15,11 @@ * limitations under the License. */ -import { addHelpers, formatBlockErrorMessage } from './response-helpers'; +import { + addHelpers, + formatBlockErrorMessage, + handlePredictResponse +} from './response-helpers'; import { expect, use } from 'chai'; import { restore } from 'sinon'; import sinonChai from 'sinon-chai'; @@ -23,8 +27,11 @@ import { BlockReason, Content, FinishReason, - GenerateContentResponse + GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage } from '../types'; +import { getMockResponse } from '../../test-utils/mock-response'; use(sinonChai); @@ -246,4 +253,71 @@ describe('response-helpers methods', () => { ); }); }); + + describe('handlePredictResponse', () => { + it('returns base64 images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-base64.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded.length).to.be.greaterThan(0); + }); + }); + }); + it('returns GCS images', async () => { + const mockResponse = getMockResponse( + 'unary-success-generate-images-gcs.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.be.undefined; + expect(res.images.length).to.equal(4); + res.images.forEach((image, i) => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.equal( + `gs://test-project-id-1234.firebasestorage.app/images/1234567890123/sample_${i}.jpg` + ); + }); + }); + it('has filtered reason and no images if all images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-all-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback. Support codes: 39322892, 29310472" + ); + expect(res.images.length).to.equal(0); + }); + it('has filtered reason and no images if all base64 images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-base64-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/png'); + expect(image.bytesBase64Encoded).to.have.length.greaterThan(0); + }); + }); + it('has filtered reason and no images if all GCS images were filtered', async () => { + const mockResponse = getMockResponse( + 'unary-failure-generate-images-gcs-some-filtered.json' + ) as Response; + const res = await handlePredictResponse(mockResponse); + expect(res.filteredReason).to.equal( + 'Your current safety filter threshold filtered out 2 generated images. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback.' + ); + expect(res.images.length).to.equal(2); + res.images.forEach(image => { + expect(image.mimeType).to.equal('image/jpeg'); + expect(image.gcsURI).to.have.length.greaterThan(0); + }); + }); }); diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 27347d10f0d..6b82463dfa1 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -21,10 +21,13 @@ import { FunctionCall, GenerateContentCandidate, GenerateContentResponse, + ImagenGCSImage, + ImagenInlineImage, VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; import { logger } from '../logger'; +import { ImagenResponseInternal } from '../types/internal'; /** * Creates an EnhancedGenerateContentResponse object that has helper functions and @@ -196,3 +199,52 @@ export function formatBlockErrorMessage( } return message; } + +/** + * Convert a generic successful fetch {@link Response} body to an Imagen response object + * that can be returned to the user. This converts the REST APIs response format to our + * representation of a response. + * + * @internal + */ + +export async function handlePredictResponse< + T extends ImagenInlineImage | ImagenGCSImage +>(response: Response): Promise<{ images: T[]; filteredReason?: string }> { + const responseJson: ImagenResponseInternal = await response.json(); + + const images: T[] = []; + let filteredReason: string | undefined = undefined; + + if (!responseJson.predictions || responseJson.predictions?.length === 0) { + throw new VertexAIError( + VertexAIErrorCode.ERROR, + "Predictions array is undefined or empty in response. Was 'includeRaiReason' enabled in the request?" + ); + } + + for (const prediction of responseJson.predictions) { + if (prediction.raiFilteredReason) { + filteredReason = prediction.raiFilteredReason; + } else if (prediction.mimeType && prediction.bytesBase64Encoded) { + images.push({ + mimeType: prediction.mimeType, + bytesBase64Encoded: prediction.bytesBase64Encoded + } as T); + } else if (prediction.mimeType && prediction.gcsUri) { + images.push({ + mimeType: prediction.mimeType, + gcsURI: prediction.gcsUri + } as T); + } else { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + `Predictions array in response has missing properties. Response: ${JSON.stringify( + responseJson + )}` + ); + } + } + + return { images, filteredReason }; +} diff --git a/packages/vertexai/src/types/imagen/index.ts b/packages/vertexai/src/types/imagen/index.ts new file mode 100644 index 00000000000..0037b185e50 --- /dev/null +++ b/packages/vertexai/src/types/imagen/index.ts @@ -0,0 +1,19 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './requests'; +export * from './responses'; diff --git a/packages/vertexai/src/types/imagen/internal.ts b/packages/vertexai/src/types/imagen/internal.ts new file mode 100644 index 00000000000..1171df81278 --- /dev/null +++ b/packages/vertexai/src/types/imagen/internal.ts @@ -0,0 +1,54 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Internal Imagen types + +/** + * A response from the REST API is expected to look like this in the success case: + * { + * "predictions": [ + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "iVBORw0KG..." + * }, + * { + * "mimeType": "image/png", + * "bytesBase64Encoded": "i4BOtw0KG..." + * } + * ] + * } + * + * And like this in the failure case: + * { + * "predictions": [ + * { + * "raiFilteredReason": "..." + * } + * ] + * } + */ +export interface ImagenResponseInternal { + predictions?: Array<{ + // Defined if the prediction was not filtered + mimeType?: string; + bytesBase64Encoded?: string; + gcsUri?: string; + + // Defined if the prediction was filtered, and there is no image + raiFilteredReason?: string; + }>; +} diff --git a/packages/vertexai/src/types/imagen/requests.ts b/packages/vertexai/src/types/imagen/requests.ts new file mode 100644 index 00000000000..d87723a31c7 --- /dev/null +++ b/packages/vertexai/src/types/imagen/requests.ts @@ -0,0 +1,142 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface ImagenModelParams extends ImagenModelConfig { + model: string; +} + +export interface ImagenModelConfig { + imageFormat?: ImagenImageFormat; + addWatermark?: boolean; + safetySettings?: ImagenSafetySettings; +} + +export interface ImagenGenerationConfig { + numberOfImages?: number; // Default to 1. Possible values are [1...4] + negativePrompt?: string; // Default to null + aspectRatio?: ImagenAspectRatio; // Default to "1:1" +} + +/** + * Contains all possible REST API paramaters. + * This is the intersection of the model-level (`ImagenModelParams`), + * request-level (`ImagenGenerationConfig`) configurations, along with + * the other required parameters prompt and gcsURI (for GCS generation only). + * + * @internal + */ +export interface ImagenRequestConfig + extends ImagenModelConfig, + ImagenGenerationConfig { + prompt: string; + gcsURI?: string; +} + +export interface ImagenImageFormat { + mimeType: string; // image/png, or image/jpeg, default image/png + compressionQuality?: number; // 0-100, default 75. Only for image/jpeg +} + +/** + * @public + */ +export enum ImagenSafetyFilterLevel { + BLOCK_LOW_AND_ABOVE = 'block_low_and_above', + BLOCK_MEDIUM_AND_ABOVE = 'block_medium_and_above', + BLOCK_ONLY_HIGH = 'block_only_high', + BLOCK_NONE = 'block_none' +} + +/** + * @public + */ +export enum ImagenPersonFilterLevel { + BLOCK_ALL = 'dont_allow', + ALLOW_ADULT = 'allow_adult', + ALLOW_ALL = 'allow_all' +} + +/** + * @public + */ +export interface ImagenSafetySettings { + /** + * Safety filter level + */ + safetyFilterLevel?: ImagenSafetyFilterLevel; + /** + * Generate people. + */ + personFilterLevel?: ImagenPersonFilterLevel; +} + +export enum ImagenAspectRatio { + SQUARE = '1:1', + CLASSIC_PORTRAIT = '3:4', + CLASSIC_LANDSCAPE = '4:3', + WIDESCREEN = '16:9', + PORTRAIT = '9:16' +} + +/** + * The parameters to be sent in the request body of the HTTP call + * to the Vertex AI backend. + * + * We need a seperate internal-only interface for this because the REST + * API expects different parameter names than what we show to our users. + * + * This interface should be populated from the {@link ImagenGenerationConfig} that + * the user defines. + * + * Sample request body JSON: + * { + * "instances": [ + * { + * "prompt": "Portrait of a golden retriever on a beach." + * } + * ], + * "parameters": { + * "mimeType": "image/png", + * "safetyFilterLevel": "block_low_and_above", + * "personGeneration": "allow_all", + * "sampleCount": 2, + * "includeRaiReason": true, + * "aspectRatio": "9:16" + * } + * } + * + * @internal + */ +export interface PredictRequestBody { + instances: [ + { + prompt: string; + } + ]; + parameters: { + sampleCount: number; // maps to numberOfImages + aspectRatio: string; + mimeType: string; + compressionQuality?: number; + negativePrompt?: string; + storageUri?: string; + addWatermark?: boolean; + safetyFilterLevel?: string; + personGeneration?: string; + includeRaiReason: boolean; + }; +} diff --git a/packages/vertexai/src/types/imagen/responses.ts b/packages/vertexai/src/types/imagen/responses.ts new file mode 100644 index 00000000000..620f49be4b9 --- /dev/null +++ b/packages/vertexai/src/types/imagen/responses.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Base class for types of images that the Imagen Model can return. + * + * @internal + */ +export interface ImagenImage { + mimeType: string; +} + +/** + * Image generated by Imagen to inline bytes. + * + * @public + */ +export interface ImagenInlineImage extends ImagenImage { + bytesBase64Encoded: string; +} + +/** + * Image generated by Imagen, stored in Google Cloud Storage (GCS). + * + * @public + */ +export interface ImagenGCSImage extends ImagenImage { + /** + * The Google Cloud Storage (GCS) URI at which the generated image is stored. + */ + gcsURI: string; +} + +/** + * Imagen image response. + * + * @public + */ +export interface ImagenInlineImageResponse { + /** + * The images generated by Imagen. If all images were filtered, this will be empty. + */ + images: ImagenInlineImage[]; + /** + * The reason the missing images were filtered. + * For the mappings of error codes to reasons, see {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-categories}. + */ + filteredReason?: string; +} + +export interface ImagenGCSImageResponse { + images: ImagenGCSImage[]; + filteredReason?: string; +} + +export interface ImagenImageReponse { + images: ImagenImage[]; + filteredReason?: string; +} diff --git a/packages/vertexai/src/types/index.ts b/packages/vertexai/src/types/index.ts index 85133aa07c5..f575c5ba8e9 100644 --- a/packages/vertexai/src/types/index.ts +++ b/packages/vertexai/src/types/index.ts @@ -21,3 +21,4 @@ export * from './requests'; export * from './responses'; export * from './error'; export * from './schema'; +export * from './imagen'; diff --git a/packages/vertexai/src/types/internal.ts b/packages/vertexai/src/types/internal.ts index 8271175feff..87c28a02ab2 100644 --- a/packages/vertexai/src/types/internal.ts +++ b/packages/vertexai/src/types/internal.ts @@ -18,6 +18,8 @@ import { AppCheckTokenResult } from '@firebase/app-check-interop-types'; import { FirebaseAuthTokenData } from '@firebase/auth-interop-types'; +export * from './imagen/internal'; + export interface ApiSettings { apiKey: string; project: string; diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 83cd4366f12..e2a442821da 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -46,6 +46,11 @@ export interface GenerateContentStreamResult { response: Promise; } +export interface SafetyAttributes { + categories: string[]; + scores: number[]; +} + /** * Response object wrapped with helper methods. *