diff --git a/docs/plugins/vertex-ai.md b/docs/plugins/vertex-ai.md index 44c09e471..f82d845dc 100644 --- a/docs/plugins/vertex-ai.md +++ b/docs/plugins/vertex-ai.md @@ -6,6 +6,7 @@ The Vertex AI plugin provides interfaces to several AI services: * Gemini text generation * Imagen2 and Imagen3 image generation * Text embedding generation + * Multimodal embedding generation * A subset of evaluation metrics through the Vertex AI [Rapid Evaluation API](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation): * [BLEU](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#bleuinput) * [ROUGE](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#rougeinput) @@ -130,7 +131,7 @@ const ai = genkit({ }); ``` -Or you can generate an embedding directly: +Or you can generate embeddings directly: ```ts const ai = genkit({ @@ -146,6 +147,8 @@ const embeddings = await ai.embed({ This plugin can also handle multimodal embeddings: ```ts +import { multimodalEmbedding001, vertexAI } from '@genkit-ai/vertexai'; + const ai = genkit({ plugins: [vertextAI({location: 'us-central1' })], }); @@ -218,7 +221,7 @@ Refer to [Imagen model documentation](https://cloud.google.com/vertex-ai/generat If you have access to Claude 3 models ([haiku](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-haiku), [sonnet](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-sonnet) or [opus](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-opus)) in Vertex AI Model Garden you can use them with Genkit. -Here's sample configuration for enabling Vertex AI Model Garden models: +Here's a sample configuration for enabling Vertex AI Model Garden models: ```ts import { genkit } from 'genkit'; @@ -268,7 +271,7 @@ const ai = genkit({ }); ``` -Then use it as regular models: +Then use it as a regular model: ```ts const llmResponse = await ai.generate({ @@ -281,7 +284,7 @@ const llmResponse = await ai.generate({ If you have access to Mistral models ([Mistral Large](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/mistral-large), [Mistral Nemo](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/mistral-nemo), or [Codestral](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/codestral)) in Vertex AI Model Garden, you can use them with Genkit. -Here's sample configuration for enabling Vertex AI Model Garden models: +Here's a sample configuration for enabling Vertex AI Model Garden models: ```ts import { genkit } from 'genkit'; @@ -388,9 +391,9 @@ Important: Pricing for Vector Search consists of both a charge for every gigabyt To use Vertex AI Vector Search: -1. Choose an embedding model. This model is responsible for creating vector embeddings from text. Advanced users might use an embedding model optimized for their particular data sets, but for most users, Vertex AI's `text-embedding-004` model is a good choice for English text and the `text-multilingual-embedding-002` model is good for multilingual text. +1. Choose an embedding model. This model is responsible for creating vector embeddings from text or media. Advanced users might use an embedding model optimized for their particular data sets, but for most users, Vertex AI's `text-embedding-004` model is a good choice for English text, the `text-multilingual-embedding-002` model is good for multilingual text, and the `multimodalEmbedding001` model is good for mixed text, images, and video. 2. In the [Vector Search](https://console.cloud.google.com/vertex-ai/matching-engine/indexes) section of the Google Cloud console, create a new index. The most important settings are: - * **Dimensions:** Specify the dimensionality of the vectors produced by your chosen embedding model. The `text-embedding-004` and `text-multilingual-embedding-002` models produce vectors of 768 dimensions. + * **Dimensions:** Specify the dimensionality of the vectors produced by your chosen embedding model. The `text-embedding-004` and `text-multilingual-embedding-002` models produce vectors of 768 dimensions. The `multimodalEmbedding001` model can produce vectors of 128, 256, 512, or 1408 dimensions for text and image, and will produce vectors of 1408 dimensions for video. * **Update method:** Select streaming updates. After you create the index, deploy it to a standard (public) endpoint. diff --git a/js/testapps/custom-evaluators/package.json b/js/testapps/custom-evaluators/package.json index 7ecd18037..a5c258b18 100644 --- a/js/testapps/custom-evaluators/package.json +++ b/js/testapps/custom-evaluators/package.json @@ -10,7 +10,7 @@ "build:clean": "rimraf ./lib", "build:watch": "tsc --watch", "dev": "tsx --watch src/index.ts", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev" + "genkit:dev": "genkit start -- tsx --watch src/index.ts" }, "keywords": [], "author": "", diff --git a/js/testapps/dev-ui-gallery/package.json b/js/testapps/dev-ui-gallery/package.json index 1da156f87..a9267e3fa 100644 --- a/js/testapps/dev-ui-gallery/package.json +++ b/js/testapps/dev-ui-gallery/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "node lib/index.js", "dev": "tsx --watch src/index.ts", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev", + "genkit:dev": "genkit start -- tsx --watch src/index.ts", "compile": "tsc", "build": "pnpm build:clean && pnpm compile", "build:clean": "rimraf ./lib", diff --git a/js/testapps/esm/package.json b/js/testapps/esm/package.json index 26c62e330..10ccab144 100644 --- a/js/testapps/esm/package.json +++ b/js/testapps/esm/package.json @@ -7,7 +7,7 @@ "scripts": { "start": "node lib/index.js", "dev": "tsx --watch src/index.ts", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev", + "genkit:dev": "genkit start -- tsx --watch src/index.ts", "compile": "tsc", "build": "pnpm build:clean && pnpm compile", "build:clean": "rimraf ./lib", diff --git a/js/testapps/evals/package.json b/js/testapps/evals/package.json index 307334903..2a66e1f19 100644 --- a/js/testapps/evals/package.json +++ b/js/testapps/evals/package.json @@ -11,7 +11,7 @@ "build:watch": "tsc --watch", "build-and-run": "pnpm build && node lib/index.js", "dev": "tsx --watch src/index.ts", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev" + "genkit:dev": "genkit start -- tsx --watch src/index.ts" }, "keywords": [], "author": "", diff --git a/js/testapps/multimodal/package.json b/js/testapps/multimodal/package.json index b0e883538..5308d3ba4 100644 --- a/js/testapps/multimodal/package.json +++ b/js/testapps/multimodal/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "node lib/index.js", "dev": "tsx --watch src/index.ts", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev", + "genkit:dev": "genkit start -- tsx --watch src/index.ts", "compile": "tsc", "build": "pnpm build:clean && pnpm compile", "build:clean": "rimraf ./lib", diff --git a/js/testapps/ollama/package.json b/js/testapps/ollama/package.json index 549fd7b2d..18880c75b 100644 --- a/js/testapps/ollama/package.json +++ b/js/testapps/ollama/package.json @@ -8,7 +8,7 @@ "start": "node lib/index.js", "build": "tsc", "build:watch": "tsc --watch", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev", + "genkit:dev": "genkit start -- tsx --watch src/index.ts", "dev": "tsx --watch src/index.ts" }, "keywords": [], diff --git a/js/testapps/rag/package.json b/js/testapps/rag/package.json index 6a610fc0c..96e669569 100644 --- a/js/testapps/rag/package.json +++ b/js/testapps/rag/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "node lib/index.js", "dev": "tsx --watch src/index.ts", - "genkit:dev": "cross-env GENKIT_ENV=dev pnpm dev", + "genkit:dev": "genkit start -- tsx --watch src/index.ts", "compile": "tsc", "build": "pnpm build:clean && pnpm compile", "build:clean": "rimraf ./lib", diff --git a/js/testapps/vertexai-vector-search-firestore/README.md b/js/testapps/vertexai-vector-search-firestore/README.md index 94f69173a..873de5c6e 100644 --- a/js/testapps/vertexai-vector-search-firestore/README.md +++ b/js/testapps/vertexai-vector-search-firestore/README.md @@ -21,7 +21,7 @@ the core packages. This sample uses `workspace:*` dependencies, so they will nee Then ```bash -cd js/testapps/vertex-vector-search-bigquery && pnpm i +cd js/testapps/vertexai-vector-search-firestore && pnpm i ``` ### Step 3: Set Up Environment Variables diff --git a/js/testapps/vertexai-vector-search-firestore/package.json b/js/testapps/vertexai-vector-search-firestore/package.json index cf0a1f390..3b0d7c223 100644 --- a/js/testapps/vertexai-vector-search-firestore/package.json +++ b/js/testapps/vertexai-vector-search-firestore/package.json @@ -5,6 +5,8 @@ "main": "lib/index.js", "scripts": { "start": "node lib/index.js", + "dev": "tsx --watch src/index.ts", + "genkit:dev": "genkit start -- tsx --watch src/index.ts", "compile": "tsc", "build": "pnpm build:clean && pnpm compile", "build:clean": "rimraf ./lib",