From 6febb22e66f6ff4655d5de7d2d99b11b711efb3c Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 14 Nov 2025 19:46:34 -0500 Subject: [PATCH 01/21] refactor(js): moved parts into a separate file to share between model and docs --- genkit-tools/common/src/types/document.ts | 149 +---- genkit-tools/common/src/types/model.ts | 27 +- genkit-tools/common/src/types/parts.ts | 192 ++++++ genkit-tools/genkit-schema.json | 716 +++++++++++----------- js/ai/src/document.ts | 151 +---- js/ai/src/index.ts | 9 +- js/ai/src/model-types.ts | 4 +- js/ai/src/model.ts | 28 +- js/ai/src/parts.ts | 177 ++++++ js/ai/src/reranker.ts | 2 +- js/ai/src/retriever.ts | 10 +- 11 files changed, 759 insertions(+), 706 deletions(-) create mode 100644 genkit-tools/common/src/types/parts.ts create mode 100644 js/ai/src/parts.ts diff --git a/genkit-tools/common/src/types/document.ts b/genkit-tools/common/src/types/document.ts index f84b802def..0778c9bc23 100644 --- a/genkit-tools/common/src/types/document.ts +++ b/genkit-tools/common/src/types/document.ts @@ -14,159 +14,12 @@ * limitations under the License. */ import z from 'zod'; +import { MediaPartSchema, TextPartSchema } from './parts.js'; // // IMPORTANT: Keep this file in sync with genkit/ai/src/document.ts! // -const EmptyPartSchema = z.object({ - text: z.never().optional(), - media: z.never().optional(), - toolRequest: z.never().optional(), - toolResponse: z.never().optional(), - data: z.unknown().optional(), - metadata: z.record(z.unknown()).optional(), - custom: z.record(z.unknown()).optional(), - reasoning: z.never().optional(), - resource: z.never().optional(), -}); - -/** - * Zod schema for a text part. - */ -export const TextPartSchema = EmptyPartSchema.extend({ - /** The text of the document. */ - text: z.string(), -}); - -/** - * Text part. - */ -export type TextPart = z.infer; - -/** - * Zod schema for a reasoning part. - */ -export const ReasoningPartSchema = EmptyPartSchema.extend({ - /** The reasoning text of the message. */ - reasoning: z.string(), -}); - -/** - * Reasoning part. - */ -export type ReasoningPart = z.infer; - -/** - * Zod schema of media. - */ -export const MediaSchema = z.object({ - /** The media content type. Inferred from data uri if not provided. */ - contentType: z.string().optional(), - /** A `data:` or `https:` uri containing the media content. */ - url: z.string(), -}); - -/** - * Zod schema of a media part. - */ -export const MediaPartSchema = EmptyPartSchema.extend({ - media: MediaSchema, -}); - -/** - * Media part. - */ -export type MediaPart = z.infer; - -/** - * Zod schema of a tool request. - */ -export const ToolRequestSchema = z.object({ - /** The call id or reference for a specific request. */ - ref: z.string().optional(), - /** The name of the tool to call. */ - name: z.string(), - /** The input parameters for the tool, usually a JSON object. */ - input: z.unknown().optional(), -}); - -/** - * Zod schema of a tool request part. - */ -export const ToolRequestPartSchema = EmptyPartSchema.extend({ - /** A request for a tool to be executed, usually provided by a model. */ - toolRequest: ToolRequestSchema, -}); - -/** - * Tool part. - */ -export type ToolRequestPart = z.infer; - -/** - * Zod schema of a tool response. - */ -export const ToolResponseSchema = z.object({ - /** The call id or reference for a specific request. */ - ref: z.string().optional(), - /** The name of the tool. */ - name: z.string(), - /** The output data returned from the tool, usually a JSON object. */ - output: z.unknown().optional(), -}); - -/** - * Zod schema of a tool response part. - */ -export const ToolResponsePartSchema = EmptyPartSchema.extend({ - /** A provided response to a tool call. */ - toolResponse: ToolResponseSchema, -}); - -/** - * Tool response part. - */ -export type ToolResponsePart = z.infer; - -/** - * Zod schema of a data part. - */ -export const DataPartSchema = EmptyPartSchema.extend({ - data: z.unknown(), -}); - -/** - * Data part. - */ -export type DataPart = z.infer; - -/** - * Zod schema of a custom part. - */ -export const CustomPartSchema = EmptyPartSchema.extend({ - custom: z.record(z.any()), -}); - -/** - * Custom part. - */ -export type CustomPart = z.infer; - -/** - * Zod schema of a resource part. - */ -export const ResourcePartSchema = EmptyPartSchema.extend({ - resource: z.object({ - uri: z.string(), - }), -}); - -/** - * Resource part. - */ -export type ResourcePart = z.infer; - // Disclaimer: genkit/js/ai/document.ts defines the following schema, type pair // as PartSchema and Part, respectively. genkit-tools cannot retain those names // due to it clashing with similar schema in model.ts, and genkit-tools diff --git a/genkit-tools/common/src/types/model.ts b/genkit-tools/common/src/types/model.ts index 617d914383..fcc426d633 100644 --- a/genkit-tools/common/src/types/model.ts +++ b/genkit-tools/common/src/types/model.ts @@ -14,11 +14,12 @@ * limitations under the License. */ import { z } from 'zod'; +import { DocumentDataSchema } from './document.js'; import { CustomPartSchema, DataPartSchema, - DocumentDataSchema, MediaPartSchema, + PartSchema, ReasoningPartSchema, ResourcePartSchema, TextPartSchema, @@ -27,16 +28,18 @@ import { type CustomPart, type DataPart, type MediaPart, + type Part, type ReasoningPart, type ResourcePart, type TextPart, type ToolRequestPart, type ToolResponsePart, -} from './document'; +} from './parts.js'; export { CustomPartSchema, DataPartSchema, MediaPartSchema, + PartSchema, ReasoningPartSchema, ResourcePartSchema, TextPartSchema, @@ -45,6 +48,7 @@ export { type CustomPart, type DataPart, type MediaPart, + type Part, type ReasoningPart, type ResourcePart, type TextPart, @@ -73,25 +77,6 @@ export const OperationSchema = z.object({ */ export type OperationData = z.infer; -/** - * Zod schema of message part. - */ -export const PartSchema = z.union([ - TextPartSchema, - MediaPartSchema, - ToolRequestPartSchema, - ToolResponsePartSchema, - DataPartSchema, - CustomPartSchema, - ReasoningPartSchema, - ResourcePartSchema, -]); - -/** - * Message part. - */ -export type Part = z.infer; - /** * Zod schema of a message role. */ diff --git a/genkit-tools/common/src/types/parts.ts b/genkit-tools/common/src/types/parts.ts new file mode 100644 index 0000000000..36f2bca250 --- /dev/null +++ b/genkit-tools/common/src/types/parts.ts @@ -0,0 +1,192 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'zod'; + +const EmptyPartSchema = z.object({ + text: z.never().optional(), + media: z.never().optional(), + toolRequest: z.never().optional(), + toolResponse: z.never().optional(), + data: z.unknown().optional(), + metadata: z.record(z.unknown()).optional(), + custom: z.record(z.unknown()).optional(), + reasoning: z.never().optional(), + resource: z.never().optional(), +}); + +/** + * Zod schema for a text part. + */ +export const TextPartSchema = EmptyPartSchema.extend({ + /** The text of the document. */ + text: z.string(), +}); + +/** + * Text part. + */ +export type TextPart = z.infer; + +/** + * Zod schema for a reasoning part. + */ +export const ReasoningPartSchema = EmptyPartSchema.extend({ + /** The reasoning text of the message. */ + reasoning: z.string(), +}); + +/** + * Reasoning part. + */ +export type ReasoningPart = z.infer; + +/** + * Zod schema of media. + */ +export const MediaSchema = z.object({ + /** The media content type. Inferred from data uri if not provided. */ + contentType: z.string().optional(), + /** A `data:` or `https:` uri containing the media content. */ + url: z.string(), +}); + +/** + * Zod schema of a media part. + */ +export const MediaPartSchema = EmptyPartSchema.extend({ + media: MediaSchema, +}); + +/** + * Media part. + */ +export type MediaPart = z.infer; + +/** + * Zod schema of a tool request. + */ +export const ToolRequestSchema = z.object({ + /** The call id or reference for a specific request. */ + ref: z.string().optional(), + /** The name of the tool to call. */ + name: z.string(), + /** The input parameters for the tool, usually a JSON object. */ + input: z.unknown().optional(), +}); + +/** + * Zod schema of a tool request part. + */ +export const ToolRequestPartSchema = EmptyPartSchema.extend({ + /** A request for a tool to be executed, usually provided by a model. */ + toolRequest: ToolRequestSchema, +}); + +/** + * Tool part. + */ +export type ToolRequestPart = z.infer; + +/** + * Zod schema of a tool response. + */ +export const ToolResponseSchemaBase = z.object({ + /** The call id or reference for a specific request. */ + ref: z.string().optional(), + /** The name of the tool. */ + name: z.string(), + /** The output data returned from the tool, usually a JSON object. */ + output: z.unknown().optional(), + payloadStrategy: z.enum(['both', 'fallback']).optional(), // default: both +}); + +/** + * Tool response part. + */ +export type ToolResponsePart = z.infer & { + content?: Part[]; +}; + +export const ToolResponseSchema: z.ZodType = + ToolResponseSchemaBase.extend({ + content: z.array(z.lazy(() => PartSchema)).optional(), + }); + +/** + * Zod schema of a tool response part. + */ +export const ToolResponsePartSchema = EmptyPartSchema.extend({ + /** A provided response to a tool call. */ + toolResponse: ToolResponseSchema, +}); + +/** + * Zod schema of a data part. + */ +export const DataPartSchema = EmptyPartSchema.extend({ + data: z.unknown(), +}); + +/** + * Data part. + */ +export type DataPart = z.infer; + +/** + * Zod schema of a custom part. + */ +export const CustomPartSchema = EmptyPartSchema.extend({ + custom: z.record(z.any()), +}); + +/** + * Custom part. + */ +export type CustomPart = z.infer; + +/** + * Zod schema of a resource part. + */ +export const ResourcePartSchema = EmptyPartSchema.extend({ + resource: z.object({ + uri: z.string(), + }), +}); + +/** + * Resource part. + */ +export type ResourcePart = z.infer; + +/** + * Zod schema of message part. + */ +export const PartSchema = z.union([ + TextPartSchema, + MediaPartSchema, + ToolRequestPartSchema, + ToolResponsePartSchema, + DataPartSchema, + CustomPartSchema, + ReasoningPartSchema, + ResourcePartSchema, +]); + +/** + * Message part. + */ +export type Part = z.infer; diff --git a/genkit-tools/genkit-schema.json b/genkit-tools/genkit-schema.json index 8a937f7c41..c7f9aa34a5 100644 --- a/genkit-tools/genkit-schema.json +++ b/genkit-tools/genkit-schema.json @@ -1,74 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "$defs": { - "CustomPart": { - "type": "object", - "properties": { - "text": { - "not": {} - }, - "media": { - "not": {} - }, - "toolRequest": { - "not": {} - }, - "toolResponse": { - "not": {} - }, - "data": {}, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "custom": { - "type": "object", - "additionalProperties": {} - }, - "reasoning": { - "not": {} - }, - "resource": { - "not": {} - } - }, - "required": [ - "custom" - ], - "additionalProperties": false - }, - "DataPart": { - "type": "object", - "properties": { - "text": { - "$ref": "#/$defs/CustomPart/properties/text" - }, - "media": { - "$ref": "#/$defs/CustomPart/properties/media" - }, - "toolRequest": { - "$ref": "#/$defs/CustomPart/properties/toolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/CustomPart/properties/toolResponse" - }, - "data": {}, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "type": "object", - "additionalProperties": {} - }, - "reasoning": { - "$ref": "#/$defs/CustomPart/properties/reasoning" - }, - "resource": { - "$ref": "#/$defs/CustomPart/properties/resource" - } - }, - "additionalProperties": false - }, "DocumentData": { "type": "object", "properties": { @@ -98,278 +30,6 @@ } ] }, - "MediaPart": { - "type": "object", - "properties": { - "text": { - "$ref": "#/$defs/CustomPart/properties/text" - }, - "media": { - "$ref": "#/$defs/Media" - }, - "toolRequest": { - "$ref": "#/$defs/CustomPart/properties/toolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/CustomPart/properties/toolResponse" - }, - "data": { - "$ref": "#/$defs/CustomPart/properties/data" - }, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "$ref": "#/$defs/DataPart/properties/custom" - }, - "reasoning": { - "$ref": "#/$defs/CustomPart/properties/reasoning" - }, - "resource": { - "$ref": "#/$defs/CustomPart/properties/resource" - } - }, - "required": [ - "media" - ], - "additionalProperties": false - }, - "Media": { - "type": "object", - "properties": { - "contentType": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "required": [ - "url" - ], - "additionalProperties": false - }, - "ReasoningPart": { - "type": "object", - "properties": { - "text": { - "$ref": "#/$defs/CustomPart/properties/text" - }, - "media": { - "$ref": "#/$defs/CustomPart/properties/media" - }, - "toolRequest": { - "$ref": "#/$defs/CustomPart/properties/toolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/CustomPart/properties/toolResponse" - }, - "data": { - "$ref": "#/$defs/CustomPart/properties/data" - }, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "$ref": "#/$defs/DataPart/properties/custom" - }, - "reasoning": { - "type": "string" - }, - "resource": { - "$ref": "#/$defs/CustomPart/properties/resource" - } - }, - "required": [ - "reasoning" - ], - "additionalProperties": false - }, - "ResourcePart": { - "type": "object", - "properties": { - "text": { - "$ref": "#/$defs/CustomPart/properties/text" - }, - "media": { - "$ref": "#/$defs/CustomPart/properties/media" - }, - "toolRequest": { - "$ref": "#/$defs/CustomPart/properties/toolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/CustomPart/properties/toolResponse" - }, - "data": { - "$ref": "#/$defs/CustomPart/properties/data" - }, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "$ref": "#/$defs/DataPart/properties/custom" - }, - "reasoning": { - "$ref": "#/$defs/CustomPart/properties/reasoning" - }, - "resource": { - "type": "object", - "properties": { - "uri": { - "type": "string" - } - }, - "required": [ - "uri" - ], - "additionalProperties": false - } - }, - "required": [ - "resource" - ], - "additionalProperties": false - }, - "TextPart": { - "type": "object", - "properties": { - "text": { - "type": "string" - }, - "media": { - "$ref": "#/$defs/CustomPart/properties/media" - }, - "toolRequest": { - "$ref": "#/$defs/CustomPart/properties/toolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/CustomPart/properties/toolResponse" - }, - "data": { - "$ref": "#/$defs/CustomPart/properties/data" - }, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "$ref": "#/$defs/DataPart/properties/custom" - }, - "reasoning": { - "$ref": "#/$defs/CustomPart/properties/reasoning" - }, - "resource": { - "$ref": "#/$defs/CustomPart/properties/resource" - } - }, - "required": [ - "text" - ], - "additionalProperties": false - }, - "ToolRequestPart": { - "type": "object", - "properties": { - "text": { - "$ref": "#/$defs/CustomPart/properties/text" - }, - "media": { - "$ref": "#/$defs/CustomPart/properties/media" - }, - "toolRequest": { - "$ref": "#/$defs/ToolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/CustomPart/properties/toolResponse" - }, - "data": { - "$ref": "#/$defs/CustomPart/properties/data" - }, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "$ref": "#/$defs/DataPart/properties/custom" - }, - "reasoning": { - "$ref": "#/$defs/CustomPart/properties/reasoning" - }, - "resource": { - "$ref": "#/$defs/CustomPart/properties/resource" - } - }, - "required": [ - "toolRequest" - ], - "additionalProperties": false - }, - "ToolRequest": { - "type": "object", - "properties": { - "ref": { - "type": "string" - }, - "name": { - "type": "string" - }, - "input": {} - }, - "required": [ - "name" - ], - "additionalProperties": false - }, - "ToolResponsePart": { - "type": "object", - "properties": { - "text": { - "$ref": "#/$defs/CustomPart/properties/text" - }, - "media": { - "$ref": "#/$defs/CustomPart/properties/media" - }, - "toolRequest": { - "$ref": "#/$defs/CustomPart/properties/toolRequest" - }, - "toolResponse": { - "$ref": "#/$defs/ToolResponse" - }, - "data": { - "$ref": "#/$defs/CustomPart/properties/data" - }, - "metadata": { - "$ref": "#/$defs/CustomPart/properties/metadata" - }, - "custom": { - "$ref": "#/$defs/DataPart/properties/custom" - }, - "reasoning": { - "$ref": "#/$defs/CustomPart/properties/reasoning" - }, - "resource": { - "$ref": "#/$defs/CustomPart/properties/resource" - } - }, - "required": [ - "toolResponse" - ], - "additionalProperties": false - }, - "ToolResponse": { - "type": "object", - "properties": { - "ref": { - "type": "string" - }, - "name": { - "type": "string" - }, - "output": {} - }, - "required": [ - "name" - ], - "additionalProperties": false - }, "EmbedRequest": { "type": "object", "properties": { @@ -661,6 +321,74 @@ ], "additionalProperties": false }, + "CustomPart": { + "type": "object", + "properties": { + "text": { + "not": {} + }, + "media": { + "not": {} + }, + "toolRequest": { + "not": {} + }, + "toolResponse": { + "not": {} + }, + "data": {}, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "custom": { + "type": "object", + "additionalProperties": {} + }, + "reasoning": { + "not": {} + }, + "resource": { + "not": {} + } + }, + "required": [ + "custom" + ], + "additionalProperties": false + }, + "DataPart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" + }, + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" + }, + "data": {}, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "type": "object", + "additionalProperties": {} + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" + }, + "resource": { + "$ref": "#/$defs/CustomPart/properties/resource" + } + }, + "additionalProperties": false + }, "FinishReason": { "type": "string", "enum": [ @@ -921,31 +649,79 @@ "outputImages": { "type": "number" }, - "inputVideos": { - "type": "number" + "inputVideos": { + "type": "number" + }, + "outputVideos": { + "type": "number" + }, + "inputAudioFiles": { + "type": "number" + }, + "outputAudioFiles": { + "type": "number" + }, + "custom": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "thoughtsTokens": { + "type": "number" + }, + "cachedContentTokens": { + "type": "number" + } + }, + "additionalProperties": false + }, + "MediaPart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "type": "object", + "properties": { + "contentType": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" }, - "outputVideos": { - "type": "number" + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" }, - "inputAudioFiles": { - "type": "number" + "data": { + "$ref": "#/$defs/CustomPart/properties/data" }, - "outputAudioFiles": { - "type": "number" + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" }, "custom": { - "type": "object", - "additionalProperties": { - "type": "number" - } + "$ref": "#/$defs/DataPart/properties/custom" }, - "thoughtsTokens": { - "type": "number" + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" }, - "cachedContentTokens": { - "type": "number" + "resource": { + "$ref": "#/$defs/CustomPart/properties/resource" } }, + "required": [ + "media" + ], "additionalProperties": false }, "Message": { @@ -1213,6 +989,87 @@ } ] }, + "ReasoningPart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" + }, + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" + }, + "data": { + "$ref": "#/$defs/CustomPart/properties/data" + }, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "type": "string" + }, + "resource": { + "$ref": "#/$defs/CustomPart/properties/resource" + } + }, + "required": [ + "reasoning" + ], + "additionalProperties": false + }, + "ResourcePart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" + }, + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" + }, + "data": { + "$ref": "#/$defs/CustomPart/properties/data" + }, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" + }, + "resource": { + "type": "object", + "properties": { + "uri": { + "type": "string" + } + }, + "required": [ + "uri" + ], + "additionalProperties": false + } + }, + "required": [ + "resource" + ], + "additionalProperties": false + }, "Role": { "type": "string", "enum": [ @@ -1222,6 +1079,42 @@ "tool" ] }, + "TextPart": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" + }, + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" + }, + "data": { + "$ref": "#/$defs/CustomPart/properties/data" + }, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" + }, + "resource": { + "$ref": "#/$defs/CustomPart/properties/resource" + } + }, + "required": [ + "text" + ], + "additionalProperties": false + }, "ToolDefinition": { "type": "object", "properties": { @@ -1269,6 +1162,117 @@ ], "additionalProperties": false }, + "ToolRequestPart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "name": { + "type": "string" + }, + "input": {} + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" + }, + "data": { + "$ref": "#/$defs/CustomPart/properties/data" + }, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" + }, + "resource": { + "$ref": "#/$defs/CustomPart/properties/resource" + } + }, + "required": [ + "toolRequest" + ], + "additionalProperties": false + }, + "ToolResponsePart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" + }, + "toolResponse": { + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "name": { + "type": "string" + }, + "output": {}, + "payloadStrategy": { + "type": "string", + "enum": [ + "both", + "fallback" + ] + }, + "content": { + "type": "array", + "items": { + "$ref": "#/$defs/Part" + } + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "data": { + "$ref": "#/$defs/CustomPart/properties/data" + }, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" + }, + "resource": { + "$ref": "#/$defs/CustomPart/properties/resource" + } + }, + "required": [ + "toolResponse" + ], + "additionalProperties": false + }, "CommonRerankerOptions": { "type": "object", "properties": { diff --git a/js/ai/src/document.ts b/js/ai/src/document.ts index c539f83fc3..67a03fd3ca 100644 --- a/js/ai/src/document.ts +++ b/js/ai/src/document.ts @@ -15,155 +15,8 @@ */ import { z } from '@genkit-ai/core'; -import type { Embedding } from './embedder'; - -const EmptyPartSchema = z.object({ - text: z.never().optional(), - media: z.never().optional(), - toolRequest: z.never().optional(), - toolResponse: z.never().optional(), - data: z.unknown().optional(), - metadata: z.record(z.unknown()).optional(), - custom: z.record(z.unknown()).optional(), - reasoning: z.never().optional(), - resource: z.never().optional(), -}); - -/** - * Zod schema for a text part. - */ -export const TextPartSchema = EmptyPartSchema.extend({ - /** The text of the message. */ - text: z.string(), -}); - -/** - * Zod schema for a reasoning part. - */ -export const ReasoningPartSchema = EmptyPartSchema.extend({ - /** The reasoning text of the message. */ - reasoning: z.string(), -}); - -/** - * Text part. - */ -export type TextPart = z.infer; - -/** - * Zod schema of media. - */ -export const MediaSchema = z.object({ - /** The media content type. Inferred from data uri if not provided. */ - contentType: z.string().optional(), - /** A `data:` or `https:` uri containing the media content. */ - url: z.string(), -}); - -/** - * Zod schema of a media part. - */ -export const MediaPartSchema = EmptyPartSchema.extend({ - media: MediaSchema, -}); - -/** - * Media part. - */ -export type MediaPart = z.infer; - -/** - * Zod schema of a tool request. - */ -export const ToolRequestSchema = z.object({ - /** The call id or reference for a specific request. */ - ref: z.string().optional(), - /** The name of the tool to call. */ - name: z.string(), - /** The input parameters for the tool, usually a JSON object. */ - input: z.unknown().optional(), -}); -export type ToolRequest = z.infer; - -/** - * Zod schema of a tool request part. - */ -export const ToolRequestPartSchema = EmptyPartSchema.extend({ - /** A request for a tool to be executed, usually provided by a model. */ - toolRequest: ToolRequestSchema, -}); - -/** - * Tool part. - */ -export type ToolRequestPart = z.infer; - -/** - * Zod schema of a tool response. - */ -export const ToolResponseSchema = z.object({ - /** The call id or reference for a specific request. */ - ref: z.string().optional(), - /** The name of the tool. */ - name: z.string(), - /** The output data returned from the tool, usually a JSON object. */ - output: z.unknown().optional(), -}); -export type ToolResponse = z.infer; - -/** - * Zod schema of a tool response part. - */ -export const ToolResponsePartSchema = EmptyPartSchema.extend({ - /** A provided response to a tool call. */ - toolResponse: ToolResponseSchema, -}); - -/** - * Tool response part. - */ -export type ToolResponsePart = z.infer; - -/** - * Zod schema of a data part. - */ -export const DataPartSchema = EmptyPartSchema.extend({ - data: z.unknown(), -}); - -/** - * Data part. - */ -export type DataPart = z.infer; - -/** - * Zod schema of a custom part. - */ -export const CustomPartSchema = EmptyPartSchema.extend({ - custom: z.record(z.any()), -}); - -/** - * Custom part. - */ -export type CustomPart = z.infer; - -/** - * Zod schema of a resource part. - */ -export const ResourcePartSchema = EmptyPartSchema.extend({ - resource: z.object({ - uri: z.string(), - }), -}); - -/** - * Resource part. - */ -export type ResourcePart = z.infer; - -export const PartSchema = z.union([TextPartSchema, MediaPartSchema]); -export type Part = z.infer; +import type { Embedding } from './embedder.js'; +import { PartSchema, type Part } from './parts.js'; // We need both metadata and embedMetadata because they can // contain the same fields (e.g. video start/stop) with different values. diff --git a/js/ai/src/index.ts b/js/ai/src/index.ts index 3f80d093d4..7baccf7ee6 100644 --- a/js/ai/src/index.ts +++ b/js/ai/src/index.ts @@ -15,13 +15,7 @@ */ export { checkOperation } from './check-operation.js'; -export { - Document, - DocumentDataSchema, - type DocumentData, - type ToolRequest, - type ToolResponse, -} from './document.js'; +export { Document, DocumentDataSchema, type DocumentData } from './document.js'; export { embed, embedderActionMetadata, @@ -88,6 +82,7 @@ export { type ToolRequestPart, type ToolResponsePart, } from './model.js'; +export { type ToolRequest, type ToolResponse } from './parts.js'; export { defineHelper, definePartial, diff --git a/js/ai/src/model-types.ts b/js/ai/src/model-types.ts index e64381345f..be2a18fb7c 100644 --- a/js/ai/src/model-types.ts +++ b/js/ai/src/model-types.ts @@ -15,17 +15,17 @@ */ import { OperationSchema, z } from '@genkit-ai/core'; +import { DocumentDataSchema } from './document.js'; import { CustomPartSchema, DataPartSchema, - DocumentDataSchema, MediaPartSchema, ReasoningPartSchema, ResourcePartSchema, TextPartSchema, ToolRequestPartSchema, ToolResponsePartSchema, -} from './document.js'; +} from './parts.js'; // // IMPORTANT: Please keep type definitions in sync with diff --git a/js/ai/src/model.ts b/js/ai/src/model.ts index 883397e301..7877133ba3 100644 --- a/js/ai/src/model.ts +++ b/js/ai/src/model.ts @@ -35,20 +35,6 @@ import { logger } from '@genkit-ai/core/logging'; import type { Registry } from '@genkit-ai/core/registry'; import { toJsonSchema } from '@genkit-ai/core/schema'; import { performance } from 'node:perf_hooks'; -import { - CustomPartSchema, - DataPartSchema, - MediaPartSchema, - TextPartSchema, - ToolRequestPartSchema, - ToolResponsePartSchema, - type CustomPart, - type DataPart, - type MediaPart, - type TextPart, - type ToolRequestPart, - type ToolResponsePart, -} from './document.js'; import { CandidateData, GenerateRequest, @@ -66,6 +52,20 @@ import { augmentWithContext, simulateConstrainedGeneration, } from './model/middleware.js'; +import { + CustomPartSchema, + DataPartSchema, + MediaPartSchema, + TextPartSchema, + ToolRequestPartSchema, + ToolResponsePartSchema, + type CustomPart, + type DataPart, + type MediaPart, + type TextPart, + type ToolRequestPart, + type ToolResponsePart, +} from './parts.js'; export { defineGenerateAction } from './generate/action.js'; export * from './model-types.js'; export { diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts new file mode 100644 index 0000000000..3011ec9992 --- /dev/null +++ b/js/ai/src/parts.ts @@ -0,0 +1,177 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from '@genkit-ai/core'; + +const EmptyPartSchema = z.object({ + text: z.never().optional(), + media: z.never().optional(), + toolRequest: z.never().optional(), + toolResponse: z.never().optional(), + data: z.unknown().optional(), + metadata: z.record(z.unknown()).optional(), + custom: z.record(z.unknown()).optional(), + reasoning: z.never().optional(), + resource: z.never().optional(), +}); + +/** + * Zod schema for a text part. + */ +export const TextPartSchema = EmptyPartSchema.extend({ + /** The text of the message. */ + text: z.string(), +}); + +/** + * Zod schema for a reasoning part. + */ +export const ReasoningPartSchema = EmptyPartSchema.extend({ + /** The reasoning text of the message. */ + reasoning: z.string(), +}); + +/** + * Text part. + */ +export type TextPart = z.infer; + +/** + * Zod schema of media. + */ +export const MediaSchema = z.object({ + /** The media content type. Inferred from data uri if not provided. */ + contentType: z.string().optional(), + /** A `data:` or `https:` uri containing the media content. */ + url: z.string(), +}); + +/** + * Zod schema of a media part. + */ +export const MediaPartSchema = EmptyPartSchema.extend({ + media: MediaSchema, +}); + +/** + * Media part. + */ +export type MediaPart = z.infer; + +/** + * Zod schema of a tool request. + */ +export const ToolRequestSchema = z.object({ + /** The call id or reference for a specific request. */ + ref: z.string().optional(), + /** The name of the tool to call. */ + name: z.string(), + /** The input parameters for the tool, usually a JSON object. */ + input: z.unknown().optional(), +}); +export type ToolRequest = z.infer; + +/** + * Zod schema of a tool request part. + */ +export const ToolRequestPartSchema = EmptyPartSchema.extend({ + /** A request for a tool to be executed, usually provided by a model. */ + toolRequest: ToolRequestSchema, +}); + +/** + * Tool part. + */ +export type ToolRequestPart = z.infer; + +/** + * Zod schema of a tool response. + */ +export const ToolResponseSchemaBase = z.object({ + /** The call id or reference for a specific request. */ + ref: z.string().optional(), + /** The name of the tool. */ + name: z.string(), + /** The output data returned from the tool, usually a JSON object. */ + output: z.unknown().optional(), + payloadStrategy: z.enum(['both', 'fallback']).optional(), // default: both +}); + +/** + * Tool response part. + */ +export type ToolResponse = z.infer & { + content?: Part[]; +}; + +export const ToolResponseSchema: z.ZodType = + ToolResponseSchemaBase.extend({ + content: z.array(z.lazy(() => PartSchema)).optional(), + }); + +/** + * Zod schema of a tool response part. + */ +export const ToolResponsePartSchema = EmptyPartSchema.extend({ + /** A provided response to a tool call. */ + toolResponse: ToolResponseSchema, +}); + +/** + * Tool response part. + */ +export type ToolResponsePart = z.infer; + +/** + * Zod schema of a data part. + */ +export const DataPartSchema = EmptyPartSchema.extend({ + data: z.unknown(), +}); + +/** + * Data part. + */ +export type DataPart = z.infer; + +/** + * Zod schema of a custom part. + */ +export const CustomPartSchema = EmptyPartSchema.extend({ + custom: z.record(z.any()), +}); + +/** + * Custom part. + */ +export type CustomPart = z.infer; + +/** + * Zod schema of a resource part. + */ +export const ResourcePartSchema = EmptyPartSchema.extend({ + resource: z.object({ + uri: z.string(), + }), +}); + +/** + * Resource part. + */ +export type ResourcePart = z.infer; + +export const PartSchema = z.union([TextPartSchema, MediaPartSchema]); +export type Part = z.infer; diff --git a/js/ai/src/reranker.ts b/js/ai/src/reranker.ts index 45cf7deba5..402777c71f 100644 --- a/js/ai/src/reranker.ts +++ b/js/ai/src/reranker.ts @@ -17,7 +17,7 @@ import { action, z, type Action } from '@genkit-ai/core'; import type { Registry } from '@genkit-ai/core/registry'; import { toJsonSchema } from '@genkit-ai/core/schema'; -import { PartSchema, type Part } from './document.js'; +import { PartSchema, type Part } from './parts.js'; import { Document, DocumentDataSchema, diff --git a/js/ai/src/retriever.ts b/js/ai/src/retriever.ts index 40bdf7e535..01702acb4c 100644 --- a/js/ai/src/retriever.ts +++ b/js/ai/src/retriever.ts @@ -20,14 +20,8 @@ import { toJsonSchema } from '@genkit-ai/core/schema'; import { Document, DocumentDataSchema, type DocumentData } from './document.js'; import type { EmbedderInfo } from './embedder.js'; -export { - Document, - DocumentDataSchema, - type DocumentData, - type MediaPart, - type Part, - type TextPart, -} from './document.js'; +export { Document, DocumentDataSchema, type DocumentData } from './document.js'; +export { type MediaPart, type Part, type TextPart } from './parts.js'; /** * Retriever implementation function signature. From b9438ad773f1b0a25af25edce8bbed896f25dedf Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 14 Nov 2025 19:47:07 -0500 Subject: [PATCH 02/21] feat(js/ai): added support for tools that can return multipart responses --- js/ai/src/generate.ts | 3 + js/ai/src/generate/resolve-tool-requests.ts | 38 +++- js/ai/src/tool.ts | 153 ++++++++++++---- js/ai/tests/generate/generate_test.ts | 188 ++++++++++++++++++++ js/ai/tests/tool_test.ts | 40 +++++ js/core/src/registry.ts | 1 + js/genkit/src/genkit.ts | 26 ++- js/genkit/tests/generate_test.ts | 52 ++++++ 8 files changed, 451 insertions(+), 50 deletions(-) diff --git a/js/ai/src/generate.ts b/js/ai/src/generate.ts index 3d55762992..5b2be557f9 100755 --- a/js/ai/src/generate.ts +++ b/js/ai/src/generate.ts @@ -588,6 +588,9 @@ async function resolveFullToolNames( if (await registry.lookupAction(`/tool/${name}`)) { return [`/tool/${name}`]; } + if (await registry.lookupAction(`/multipart-tool/${name}`)) { + return [`/multipart-tool/${name}`]; + } if (await registry.lookupAction(`/prompt/${name}`)) { return [`/prompt/${name}`]; } diff --git a/js/ai/src/generate/resolve-tool-requests.ts b/js/ai/src/generate/resolve-tool-requests.ts index 7faf3a8c21..32dabcce0d 100644 --- a/js/ai/src/generate/resolve-tool-requests.ts +++ b/js/ai/src/generate/resolve-tool-requests.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { GenkitError, stripUndefinedProps } from '@genkit-ai/core'; +import { GenkitError, stripUndefinedProps, z } from '@genkit-ai/core'; import { logger } from '@genkit-ai/core/logging'; import type { Registry } from '@genkit-ai/core/registry'; import type { @@ -25,8 +25,10 @@ import type { ToolRequestPart, ToolResponsePart, } from '../model.js'; +import { ToolResponse } from '../parts.js'; import { isPromptAction } from '../prompt.js'; import { + MultipartToolResponseSchema, ToolInterruptError, isToolRequest, resolveTools, @@ -120,15 +122,33 @@ export async function resolveToolRequest( // otherwise, execute the tool and catch interrupts try { const output = await tool(part.toolRequest.input, toRunOptions(part)); - const response = stripUndefinedProps({ - toolResponse: { - name: part.toolRequest.name, - ref: part.toolRequest.ref, - output, - }, - }); + if (tool.__action.actionType === 'multipart-tool') { + const multipartResponse = output as z.infer< + typeof MultipartToolResponseSchema + >; + const strategy = multipartResponse.fallbackOutput ? 'fallback' : 'both'; + const response = stripUndefinedProps({ + toolResponse: { + name: part.toolRequest.name, + ref: part.toolRequest.ref, + output: multipartResponse.output || multipartResponse.fallbackOutput, + content: multipartResponse.content, + payloadStrategy: strategy, + } as ToolResponse, + }); - return { response }; + return { response }; + } else { + const response = stripUndefinedProps({ + toolResponse: { + name: part.toolRequest.name, + ref: part.toolRequest.ref, + output, + }, + }); + + return { response }; + } } catch (e) { if ( e instanceof ToolInterruptError || diff --git a/js/ai/src/tool.ts b/js/ai/src/tool.ts index 19c9665676..44e167b48b 100644 --- a/js/ai/src/tool.ts +++ b/js/ai/src/tool.ts @@ -17,7 +17,6 @@ import { action, assertUnstable, - defineAction, isAction, stripUndefinedProps, z, @@ -29,11 +28,12 @@ import { import type { Registry } from '@genkit-ai/core/registry'; import { parseSchema, toJsonSchema } from '@genkit-ai/core/schema'; import { setCustomMetadataAttributes } from '@genkit-ai/core/tracing'; -import type { - Part, - ToolDefinition, - ToolRequestPart, - ToolResponsePart, +import { + PartSchema, + type Part, + type ToolDefinition, + type ToolRequestPart, + type ToolResponsePart, } from './model.js'; import { isExecutablePrompt, type ExecutablePrompt } from './prompt.js'; @@ -100,6 +100,26 @@ export type ToolAction< }; }; +/** + * An action with a `multipart-tool` type. + */ +export type MultipartToolAction< + I extends z.ZodTypeAny = z.ZodTypeAny, + O extends z.ZodTypeAny = z.ZodTypeAny, +> = Action< + I, + typeof MultipartToolResponseSchema, + z.ZodTypeAny, + ToolRunOptions +> & + Resumable & { + __action: { + metadata: { + type: 'multipart-tool'; + }; + }; + }; + /** * A dynamic action with a `tool` type. Dynamic tools are detached actions -- not associated with any registry. */ @@ -218,6 +238,7 @@ export async function lookupToolByName( const tool = (await registry.lookupAction(name)) || (await registry.lookupAction(`/tool/${name}`)) || + (await registry.lookupAction(`/multipart-tool/${name}`)) || (await registry.lookupAction(`/prompt/${name}`)) || (await registry.lookupAction(`/dynamic-action-provider/${name}`)); if (!tool) { @@ -273,6 +294,26 @@ export type ToolFn = ( ctx: ToolFnOptions & ToolRunOptions ) => Promise>; +export type MultipartToolFn = ( + input: z.infer, + ctx: ToolFnOptions & ToolRunOptions +) => Promise<{ + output?: z.infer; + fallbackOutput?: z.infer; + content: Part[]; +}>; + +export function defineTool( + registry: Registry, + config: { multipart: true } & ToolConfig, + fn?: ToolFn +): MultipartToolAction; +export function defineTool( + registry: Registry, + config: ToolConfig, + fn?: ToolFn +): ToolAction; + /** * Defines a tool. * @@ -280,25 +321,11 @@ export type ToolFn = ( */ export function defineTool( registry: Registry, - config: ToolConfig, - fn: ToolFn -): ToolAction { - const a = defineAction( - registry, - { - ...config, - actionType: 'tool', - metadata: { ...(config.metadata || {}), type: 'tool' }, - }, - (i, runOptions) => { - return fn(i, { - ...runOptions, - context: { ...runOptions.context }, - interrupt: interruptTool(registry), - }); - } - ); - implementTool(a as ToolAction, config, registry); + config: { multipart?: true } & ToolConfig, + fn?: ToolFn | MultipartToolFn +): ToolAction | MultipartToolAction { + const a = tool(config, fn); + registry.registerAction(config.multipart ? 'multipart-tool' : 'tool', a); return a as ToolAction; } @@ -432,27 +459,30 @@ function interruptTool(registry?: Registry) { }; } -/** - * Defines a dynamic tool. Dynamic tools are just like regular tools but will not be registered in the - * Genkit registry and can be defined dynamically at runtime. - */ +export function tool( + config: { multipart: true } & ToolConfig, + fn?: ToolFn +): MultipartToolAction; export function tool( config: ToolConfig, fn?: ToolFn -): ToolAction { - return dynamicTool(config, fn); -} +): ToolAction; /** * Defines a dynamic tool. Dynamic tools are just like regular tools but will not be registered in the * Genkit registry and can be defined dynamically at runtime. - * - * @deprecated renamed to {@link tool}. */ -export function dynamicTool( +export function tool( + config: { multipart?: true } & ToolConfig, + fn?: ToolFn | MultipartToolFn +): ToolAction | MultipartToolAction { + return config.multipart ? multipartTool(config, fn) : basicTool(config, fn); +} + +function basicTool( config: ToolConfig, fn?: ToolFn -): DynamicToolAction { +): ToolAction { const a = action( { ...config, @@ -470,8 +500,55 @@ export function dynamicTool( } return interrupt(); } - ) as DynamicToolAction; + ) as ToolAction; + implementTool(a, config); + return a; +} + +export const MultipartToolResponseSchema = z.object({ + output: z.any().optional(), + fallbackOutput: z.any().optional(), + content: z.array(PartSchema), +}); + +function multipartTool( + config: ToolConfig, + fn?: MultipartToolFn +): MultipartToolAction { + const a = action( + { + ...config, + outputSchema: MultipartToolResponseSchema, + actionType: 'multipart-tool', + metadata: { ...(config.metadata || {}), type: 'multipart-tool' }, + }, + (i, runOptions) => { + const interrupt = interruptTool(runOptions.registry); + if (fn) { + return fn(i, { + ...runOptions, + context: { ...runOptions.context }, + interrupt, + }); + } + return interrupt(); + } + ) as MultipartToolAction; implementTool(a as any, config); - a.attach = (_: Registry) => a; return a; } + +/** + * Defines a dynamic tool. Dynamic tools are just like regular tools but will not be registered in the + * Genkit registry and can be defined dynamically at runtime. + * + * @deprecated renamed to {@link tool}. + */ +export function dynamicTool( + config: ToolConfig, + fn?: ToolFn +): DynamicToolAction { + const t = basicTool(config, fn) as DynamicToolAction; + t.attach = (_: Registry) => t; + return t; +} diff --git a/js/ai/tests/generate/generate_test.ts b/js/ai/tests/generate/generate_test.ts index 4abd89038e..089edf170e 100644 --- a/js/ai/tests/generate/generate_test.ts +++ b/js/ai/tests/generate/generate_test.ts @@ -602,4 +602,192 @@ describe('generate', () => { ['Testing default step name', 'Testing default step name'] ); }); + + it('handles multipart tool responses', async () => { + defineTool( + registry, + { + name: 'multiTool', + description: 'a tool with multiple parts', + multipart: true, + }, + async () => { + return { + output: 'main output', + content: [{ text: 'part 1' }], + }; + } + ); + + let requestCount = 0; + defineModel( + registry, + { name: 'multi-tool-model', supports: { tools: true } }, + async (input) => { + requestCount++; + return { + message: { + role: 'model', + content: [ + requestCount == 1 + ? { + toolRequest: { + name: 'multiTool', + input: {}, + }, + } + : { text: 'done' }, + ], + }, + finishReason: 'stop', + }; + } + ); + + const response = await generate(registry, { + model: 'multi-tool-model', + prompt: 'go', + tools: ['multiTool'], + }); + assert.deepStrictEqual(response.messages, [ + { + role: 'user', + content: [ + { + text: 'go', + }, + ], + }, + { + role: 'model', + content: [ + { + toolRequest: { + name: 'multiTool', + input: {}, + }, + }, + ], + }, + { + role: 'tool', + content: [ + { + toolResponse: { + name: 'multiTool', + output: 'main output', + content: [ + { + text: 'part 1', + }, + ], + payloadStrategy: 'both', + }, + }, + ], + }, + { + role: 'model', + content: [ + { + text: 'done', + }, + ], + }, + ]); + }); + + it('handles fallback tool responses', async () => { + defineTool( + registry, + { + name: 'fallbackTool', + description: 'a tool with fallback output', + multipart: true, + }, + async () => { + return { + fallbackOutput: 'fallback output', + content: [{ text: 'part 1' }], + }; + } + ); + + let requestCount = 0; + defineModel( + registry, + { name: 'fallback-tool-model', supports: { tools: true } }, + async (input) => { + requestCount++; + return { + message: { + role: 'model', + content: [ + requestCount == 1 + ? { + toolRequest: { + name: 'fallbackTool', + input: {}, + }, + } + : { text: 'done' }, + ], + }, + finishReason: 'stop', + }; + } + ); + + const response = await generate(registry, { + model: 'fallback-tool-model', + prompt: 'go', + tools: ['fallbackTool'], + }); + assert.deepStrictEqual(response.messages, [ + { + role: 'user', + content: [ + { + text: 'go', + }, + ], + }, + { + role: 'model', + content: [ + { + toolRequest: { + name: 'fallbackTool', + input: {}, + }, + }, + ], + }, + { + role: 'tool', + content: [ + { + toolResponse: { + name: 'fallbackTool', + output: 'fallback output', + content: [ + { + text: 'part 1', + }, + ], + payloadStrategy: 'fallback', + }, + }, + ], + }, + { + role: 'model', + content: [ + { + text: 'done', + }, + ], + }, + ]); + }); }); diff --git a/js/ai/tests/tool_test.ts b/js/ai/tests/tool_test.ts index 74a0194e91..cd0616b896 100644 --- a/js/ai/tests/tool_test.ts +++ b/js/ai/tests/tool_test.ts @@ -107,6 +107,46 @@ describe('defineInterrupt', () => { type: 'string', }); }); + + describe('multipart tools', () => { + it('should define a multipart tool', async () => { + const t = defineTool( + registry, + { name: 'test', description: 'test', multipart: true }, + async () => { + return { + output: 'main output', + content: [{ text: 'part 1' }], + }; + } + ); + assert.equal(t.__action.metadata.type, 'multipart-tool'); + assert.equal(t.__action.actionType, 'multipart-tool'); + const result = await t({}); + assert.deepStrictEqual(result, { + output: 'main output', + content: [{ text: 'part 1' }], + }); + }); + + it('should handle fallback output', async () => { + const t = defineTool( + registry, + { name: 'test', description: 'test', multipart: true }, + async () => { + return { + fallbackOutput: 'fallback', + content: [{ text: 'part 1' }], + }; + } + ); + const result = await t({}); + assert.deepStrictEqual(result, { + fallbackOutput: 'fallback', + content: [{ text: 'part 1' }], + }); + }); + }); }); describe('defineTool', () => { diff --git a/js/core/src/registry.ts b/js/core/src/registry.ts index fbe128a241..8036da0491 100644 --- a/js/core/src/registry.ts +++ b/js/core/src/registry.ts @@ -53,6 +53,7 @@ const ACTION_TYPES = [ 'reranker', 'retriever', 'tool', + 'multipart-tool', 'util', 'resource', ] as const; diff --git a/js/genkit/src/genkit.ts b/js/genkit/src/genkit.ts index 040d640113..2cd34a4c7a 100644 --- a/js/genkit/src/genkit.ts +++ b/js/genkit/src/genkit.ts @@ -97,7 +97,12 @@ import { type RetrieverFn, type SimpleRetrieverOptions, } from '@genkit-ai/ai/retriever'; -import { dynamicTool, type ToolFn } from '@genkit-ai/ai/tool'; +import { + dynamicTool, + type MultipartToolAction, + type MultipartToolFn, + type ToolFn, +} from '@genkit-ai/ai/tool'; import { ActionFnArg, GenkitError, @@ -222,6 +227,16 @@ export class Genkit implements HasRegistry { return flow; } + /** + * Defines and registers a tool that can return multiple parts of content. + * + * Tools can be passed to models by name or value during `generate` calls to be called automatically based on the prompt and situation. + */ + defineTool( + config: { multipart: true } & ToolConfig, + fn: MultipartToolFn + ): MultipartToolAction; + /** * Defines and registers a tool. * @@ -230,8 +245,13 @@ export class Genkit implements HasRegistry { defineTool( config: ToolConfig, fn: ToolFn - ): ToolAction { - return defineTool(this.registry, config, fn); + ): ToolAction; + + defineTool( + config: ({ multipart?: true } & ToolConfig) | string, + fn: ToolFn | MultipartToolFn + ): ToolAction | MultipartToolAction { + return defineTool(this.registry, config as any, fn as any); } /** diff --git a/js/genkit/tests/generate_test.ts b/js/genkit/tests/generate_test.ts index 9244851fa6..8114117f46 100644 --- a/js/genkit/tests/generate_test.ts +++ b/js/genkit/tests/generate_test.ts @@ -814,6 +814,58 @@ describe('generate', () => { assert.strictEqual(text, '{"foo":"bar a@b.c"}'); }); + it('calls the multipart tool', async () => { + const t = ai.defineTool( + { name: 'testTool', description: 'description', multipart: true }, + async () => ({ + output: 'tool called', + content: [{ text: 'part 1' }], + }) + ); + + // first response is a tool call, the subsequent responses are just text response from agent b. + let reqCounter = 0; + pm.handleResponse = async (req, sc) => { + return { + message: { + role: 'model', + content: [ + reqCounter++ === 0 + ? { + toolRequest: { + name: 'testTool', + input: {}, + ref: 'ref123', + }, + } + : { text: 'done' }, + ], + }, + }; + }; + + const { text, messages } = await ai.generate({ + prompt: 'call the tool', + tools: [t], + }); + + assert.strictEqual(text, 'done'); + assert.strictEqual(messages.length, 4); + const toolMessage = messages[2]; + assert.strictEqual(toolMessage.role, 'tool'); + assert.deepStrictEqual(toolMessage.content, [ + { + toolResponse: { + name: 'testTool', + ref: 'ref123', + output: 'tool called', + content: [{ text: 'part 1' }], + payloadStrategy: 'both', + }, + }, + ]); + }); + it('streams the tool responses', async () => { ai.defineTool( { name: 'testTool', description: 'description' }, From f5d01493960a30dea764e218fa16f72adc29b334 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 14 Nov 2025 19:55:31 -0500 Subject: [PATCH 03/21] fix --- genkit-tools/common/src/types/document.ts | 2 +- genkit-tools/common/src/types/model.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/genkit-tools/common/src/types/document.ts b/genkit-tools/common/src/types/document.ts index 0778c9bc23..164cbc3920 100644 --- a/genkit-tools/common/src/types/document.ts +++ b/genkit-tools/common/src/types/document.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import z from 'zod'; -import { MediaPartSchema, TextPartSchema } from './parts.js'; +import { MediaPartSchema, TextPartSchema } from './parts'; // // IMPORTANT: Keep this file in sync with genkit/ai/src/document.ts! diff --git a/genkit-tools/common/src/types/model.ts b/genkit-tools/common/src/types/model.ts index fcc426d633..9402fe9562 100644 --- a/genkit-tools/common/src/types/model.ts +++ b/genkit-tools/common/src/types/model.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { z } from 'zod'; -import { DocumentDataSchema } from './document.js'; +import { DocumentDataSchema } from './document'; import { CustomPartSchema, DataPartSchema, @@ -34,7 +34,7 @@ import { type TextPart, type ToolRequestPart, type ToolResponsePart, -} from './parts.js'; +} from './parts'; export { CustomPartSchema, DataPartSchema, From 15449fa7076954dd0d1671b09d3c0dc2d1864d70 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 14 Nov 2025 20:18:00 -0500 Subject: [PATCH 04/21] fix go/py --- genkit-tools/common/src/types/parts.ts | 9 +- genkit-tools/genkit-schema.json | 114 +++--- genkit-tools/scripts/schema-exporter.ts | 1 + go/ai/gen.go | 15 +- .../cmd/jsonschemagen/jsonschemagen.go | 3 + js/ai/src/parts.ts | 6 +- py/packages/genkit/src/genkit/core/typing.py | 359 +++++++++--------- .../genkit/tests/genkit/blocks/prompt_test.py | 3 +- 8 files changed, 270 insertions(+), 240 deletions(-) diff --git a/genkit-tools/common/src/types/parts.ts b/genkit-tools/common/src/types/parts.ts index 36f2bca250..d05d4c3553 100644 --- a/genkit-tools/common/src/types/parts.ts +++ b/genkit-tools/common/src/types/parts.ts @@ -101,10 +101,7 @@ export const ToolRequestPartSchema = EmptyPartSchema.extend({ */ export type ToolRequestPart = z.infer; -/** - * Zod schema of a tool response. - */ -export const ToolResponseSchemaBase = z.object({ +const ToolResponseSchemaBase = z.object({ /** The call id or reference for a specific request. */ ref: z.string().optional(), /** The name of the tool. */ @@ -123,7 +120,9 @@ export type ToolResponsePart = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.any()).optional(), + // TODO: switch to this once we have effective recursive schema support across the board. + // content: z.array(z.lazy(() => PartSchema)).optional(), }); /** diff --git a/genkit-tools/genkit-schema.json b/genkit-tools/genkit-schema.json index c7f9aa34a5..02c5e63136 100644 --- a/genkit-tools/genkit-schema.json +++ b/genkit-tools/genkit-schema.json @@ -683,19 +683,7 @@ "$ref": "#/$defs/CustomPart/properties/text" }, "media": { - "type": "object", - "properties": { - "contentType": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "required": [ - "url" - ], - "additionalProperties": false + "$ref": "#/$defs/Media" }, "toolRequest": { "$ref": "#/$defs/CustomPart/properties/toolRequest" @@ -1172,20 +1160,7 @@ "$ref": "#/$defs/CustomPart/properties/media" }, "toolRequest": { - "type": "object", - "properties": { - "ref": { - "type": "string" - }, - "name": { - "type": "string" - }, - "input": {} - }, - "required": [ - "name" - ], - "additionalProperties": false + "$ref": "#/$defs/ToolRequest" }, "toolResponse": { "$ref": "#/$defs/CustomPart/properties/toolResponse" @@ -1224,33 +1199,7 @@ "$ref": "#/$defs/CustomPart/properties/toolRequest" }, "toolResponse": { - "type": "object", - "properties": { - "ref": { - "type": "string" - }, - "name": { - "type": "string" - }, - "output": {}, - "payloadStrategy": { - "type": "string", - "enum": [ - "both", - "fallback" - ] - }, - "content": { - "type": "array", - "items": { - "$ref": "#/$defs/Part" - } - } - }, - "required": [ - "name" - ], - "additionalProperties": false + "$ref": "#/$defs/ToolResponse" }, "data": { "$ref": "#/$defs/CustomPart/properties/data" @@ -1273,6 +1222,63 @@ ], "additionalProperties": false }, + "Media": { + "type": "object", + "properties": { + "contentType": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false + }, + "ToolRequest": { + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "name": { + "type": "string" + }, + "input": {} + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "ToolResponse": { + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "name": { + "type": "string" + }, + "output": {}, + "payloadStrategy": { + "type": "string", + "enum": [ + "both", + "fallback" + ] + }, + "content": { + "type": "array" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, "CommonRerankerOptions": { "type": "object", "properties": { diff --git a/genkit-tools/scripts/schema-exporter.ts b/genkit-tools/scripts/schema-exporter.ts index 78f310b071..48df79b56a 100644 --- a/genkit-tools/scripts/schema-exporter.ts +++ b/genkit-tools/scripts/schema-exporter.ts @@ -27,6 +27,7 @@ const EXPORTED_TYPE_MODULES = [ '../common/src/types/evaluator.ts', '../common/src/types/error.ts', '../common/src/types/model.ts', + '../common/src/types/parts.ts', '../common/src/types/reranker.ts', '../common/src/types/retriever.ts', '../common/src/types/trace.ts', diff --git a/go/ai/gen.go b/go/ai/gen.go index 5d24d51bf0..5b39121259 100644 --- a/go/ai/gen.go +++ b/go/ai/gen.go @@ -390,11 +390,13 @@ type toolRequestPart struct { // the results of running a specific tool on the arguments passed to the client // by the model in a [ToolRequest]. type ToolResponse struct { - Name string `json:"name,omitempty"` + Content []any `json:"content,omitempty"` + Name string `json:"name,omitempty"` // Output is a JSON object describing the results of running the tool. // An example might be map[string]any{"name":"Thomas Jefferson", "born":1743}. - Output any `json:"output,omitempty"` - Ref string `json:"ref,omitempty"` + Output any `json:"output,omitempty"` + PayloadStrategy ToolResponsePayloadStrategy `json:"payloadStrategy,omitempty"` + Ref string `json:"ref,omitempty"` } type toolResponsePart struct { @@ -402,6 +404,13 @@ type toolResponsePart struct { ToolResponse *ToolResponse `json:"toolResponse,omitempty"` } +type ToolResponsePayloadStrategy string + +const ( + ToolResponsePayloadStrategyBoth ToolResponsePayloadStrategy = "both" + ToolResponsePayloadStrategyFallback ToolResponsePayloadStrategy = "fallback" +) + type TraceMetadata struct { FeatureName string `json:"featureName,omitempty"` Paths []*PathMetadata `json:"paths,omitempty"` diff --git a/go/internal/cmd/jsonschemagen/jsonschemagen.go b/go/internal/cmd/jsonschemagen/jsonschemagen.go index 6714c0dce8..243346f5e4 100644 --- a/go/internal/cmd/jsonschemagen/jsonschemagen.go +++ b/go/internal/cmd/jsonschemagen/jsonschemagen.go @@ -454,6 +454,9 @@ func (g *generator) generateDoc(s *Schema, ic *itemConfig) { // typeExpr returns a Go type expression denoting the type represented by the schema. func (g *generator) typeExpr(s *Schema) (string, error) { // A reference to another type refers to that type by name. Use the name. + if s == nil { + return "any", nil + } if s.Ref != "" { name, ok := strings.CutPrefix(s.Ref, refPrefix) if !ok { diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts index 3011ec9992..32117891da 100644 --- a/js/ai/src/parts.ts +++ b/js/ai/src/parts.ts @@ -100,7 +100,7 @@ export type ToolRequestPart = z.infer; /** * Zod schema of a tool response. */ -export const ToolResponseSchemaBase = z.object({ +const ToolResponseSchemaBase = z.object({ /** The call id or reference for a specific request. */ ref: z.string().optional(), /** The name of the tool. */ @@ -119,7 +119,9 @@ export type ToolResponse = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.any()).optional(), + // TODO: switch to this once we have effective recursive schema support across the board. + // content: z.array(z.lazy(() => PartSchema)).optional(), }); /** diff --git a/py/packages/genkit/src/genkit/core/typing.py b/py/packages/genkit/src/genkit/core/typing.py index 0673136c52..c70cd142c4 100644 --- a/py/packages/genkit/src/genkit/core/typing.py +++ b/py/packages/genkit/src/genkit/core/typing.py @@ -44,54 +44,6 @@ class Model(RootModel[Any]): root: Any -class CustomPart(BaseModel): - """Model for custompart data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - text: Any | None = None - media: Any | None = None - tool_request: Any | None = Field(None, alias='toolRequest') - tool_response: Any | None = Field(None, alias='toolResponse') - data: Any | None = None - metadata: dict[str, Any] | None = None - custom: dict[str, Any] - reasoning: Any | None = None - resource: Any | None = None - - -class Media(BaseModel): - """Model for media data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - content_type: str | None = Field(None, alias='contentType') - url: str - - -class Resource1(BaseModel): - """Model for resource1 data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - uri: str - - -class ToolRequest(BaseModel): - """Model for toolrequest data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - ref: str | None = None - name: str - input: Any | None = None - - -class ToolResponse(BaseModel): - """Model for toolresponse data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - ref: str | None = None - name: str - output: Any | None = None - - class Embedding(BaseModel): """Model for embedding data.""" @@ -155,8 +107,8 @@ class GenkitErrorDetails(BaseModel): trace_id: str = Field(..., alias='traceId') -class Data1(BaseModel): - """Model for data1 data.""" +class Data(BaseModel): + """Model for data data.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) genkit_error_message: str | None = Field(None, alias='genkitErrorMessage') @@ -170,7 +122,7 @@ class GenkitError(BaseModel): message: str stack: str | None = None details: Any | None = None - data: Data1 | None = None + data: Data | None = None class Code(StrEnum): @@ -190,6 +142,21 @@ class CandidateError(BaseModel): message: str | None = None +class CustomPart(BaseModel): + """Model for custompart data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + text: Any | None = None + media: Any | None = None + tool_request: Any | None = Field(None, alias='toolRequest') + tool_response: Any | None = Field(None, alias='toolResponse') + data: Any | None = None + metadata: dict[str, Any] | None = None + custom: dict[str, Any] + reasoning: Any | None = None + resource: Any | None = None + + class FinishReason(StrEnum): """Enumeration of finishreason values.""" @@ -325,6 +292,13 @@ class OutputConfig(BaseModel): content_type: str | None = Field(None, alias='contentType') +class Resource1(BaseModel): + """Model for resource1 data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + uri: str + + class Role(StrEnum): """Enumeration of role values.""" @@ -349,6 +323,41 @@ class ToolDefinition(BaseModel): metadata: dict[str, Any] | None = Field(None, description='additional metadata for this tool definition') +class Media(BaseModel): + """Model for media data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + content_type: str | None = Field(None, alias='contentType') + url: str + + +class ToolRequest(BaseModel): + """Model for toolrequest data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + ref: str | None = None + name: str + input: Any | None = None + + +class PayloadStrategy(StrEnum): + """Enumeration of payloadstrategy values.""" + + BOTH = 'both' + FALLBACK = 'fallback' + + +class ToolResponse(BaseModel): + """Model for toolresponse data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + ref: str | None = None + name: str + output: Any | None = None + payload_strategy: PayloadStrategy | None = Field(None, alias='payloadStrategy') + content: list | None = None + + class CommonRerankerOptions(BaseModel): """Model for commonrerankeroptions data.""" @@ -489,8 +498,8 @@ class TraceIds(RootModel[list[str]]): root: list[str] -class Data(RootModel[Any]): - """Root model for data.""" +class DataModel(RootModel[Any]): + """Root model for datamodel.""" root: Any @@ -603,6 +612,42 @@ class Index(RootModel[float]): root: float +class EmbedResponse(BaseModel): + """Model for embedresponse data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + embeddings: list[Embedding] + + +class BaseEvalDataPoint(BaseModel): + """Model for baseevaldatapoint data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + input: Input | None = None + output: Output | None = None + context: Context | None = None + reference: Reference | None = None + test_case_id: str = Field(..., alias='testCaseId') + trace_ids: TraceIds | None = Field(None, alias='traceIds') + + +class EvalFnResponse(BaseModel): + """Model for evalfnresponse data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + sample_index: float | None = Field(None, alias='sampleIndex') + test_case_id: str = Field(..., alias='testCaseId') + trace_id: str | None = Field(None, alias='traceId') + span_id: str | None = Field(None, alias='spanId') + evaluation: Score | list[Score] + + +class EvalResponse(RootModel[list[EvalFnResponse]]): + """Root model for evalresponse.""" + + root: list[EvalFnResponse] + + class DataPart(BaseModel): """Model for datapart data.""" @@ -626,7 +671,7 @@ class MediaPart(BaseModel): media: Media tool_request: ToolRequestModel | None = Field(None, alias='toolRequest') tool_response: ToolResponseModel | None = Field(None, alias='toolResponse') - data: Data | None = None + data: DataModel | None = None metadata: Metadata | None = None custom: Custom | None = None reasoning: Reasoning | None = None @@ -641,7 +686,7 @@ class ReasoningPart(BaseModel): media: MediaModel | None = None tool_request: ToolRequestModel | None = Field(None, alias='toolRequest') tool_response: ToolResponseModel | None = Field(None, alias='toolResponse') - data: Data | None = None + data: DataModel | None = None metadata: Metadata | None = None custom: Custom | None = None reasoning: str @@ -656,7 +701,7 @@ class ResourcePart(BaseModel): media: MediaModel | None = None tool_request: ToolRequestModel | None = Field(None, alias='toolRequest') tool_response: ToolResponseModel | None = Field(None, alias='toolResponse') - data: Data | None = None + data: DataModel | None = None metadata: Metadata | None = None custom: Custom | None = None reasoning: Reasoning | None = None @@ -671,7 +716,7 @@ class TextPart(BaseModel): media: MediaModel | None = None tool_request: ToolRequestModel | None = Field(None, alias='toolRequest') tool_response: ToolResponseModel | None = Field(None, alias='toolResponse') - data: Data | None = None + data: DataModel | None = None metadata: Metadata | None = None custom: Custom | None = None reasoning: Reasoning | None = None @@ -686,7 +731,7 @@ class ToolRequestPart(BaseModel): media: MediaModel | None = None tool_request: ToolRequest = Field(..., alias='toolRequest') tool_response: ToolResponseModel | None = Field(None, alias='toolResponse') - data: Data | None = None + data: DataModel | None = None metadata: Metadata | None = None custom: Custom | None = None reasoning: Reasoning | None = None @@ -701,70 +746,13 @@ class ToolResponsePart(BaseModel): media: MediaModel | None = None tool_request: ToolRequestModel | None = Field(None, alias='toolRequest') tool_response: ToolResponse = Field(..., alias='toolResponse') - data: Data | None = None + data: DataModel | None = None metadata: Metadata | None = None custom: Custom | None = None reasoning: Reasoning | None = None resource: Resource | None = None -class EmbedResponse(BaseModel): - """Model for embedresponse data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - embeddings: list[Embedding] - - -class BaseEvalDataPoint(BaseModel): - """Model for baseevaldatapoint data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - input: Input | None = None - output: Output | None = None - context: Context | None = None - reference: Reference | None = None - test_case_id: str = Field(..., alias='testCaseId') - trace_ids: TraceIds | None = Field(None, alias='traceIds') - - -class EvalFnResponse(BaseModel): - """Model for evalfnresponse data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - sample_index: float | None = Field(None, alias='sampleIndex') - test_case_id: str = Field(..., alias='testCaseId') - trace_id: str | None = Field(None, alias='traceId') - span_id: str | None = Field(None, alias='spanId') - evaluation: Score | list[Score] - - -class EvalResponse(RootModel[list[EvalFnResponse]]): - """Root model for evalresponse.""" - - root: list[EvalFnResponse] - - -class Resume(BaseModel): - """Model for resume data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - respond: list[ToolResponsePart] | None = None - restart: list[ToolRequestPart] | None = None - metadata: dict[str, Any] | None = None - - -class Part( - RootModel[ - TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart | ReasoningPart | ResourcePart - ] -): - """Root model for part.""" - - root: ( - TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart | ReasoningPart | ResourcePart - ) - - class Link(BaseModel): """Model for link data.""" @@ -814,16 +802,68 @@ class TraceData(BaseModel): spans: dict[str, SpanData] +class DocumentPart(RootModel[TextPart | MediaPart]): + """Root model for documentpart.""" + + root: TextPart | MediaPart + + +class Resume(BaseModel): + """Model for resume data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + respond: list[ToolResponsePart] | None = None + restart: list[ToolRequestPart] | None = None + metadata: dict[str, Any] | None = None + + +class Part( + RootModel[ + TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart | ReasoningPart | ResourcePart + ] +): + """Root model for part.""" + + root: ( + TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart | ReasoningPart | ResourcePart + ) + + +class RankedDocumentData(BaseModel): + """Model for rankeddocumentdata data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + content: list[DocumentPart] + metadata: RankedDocumentMetadata + + +class RerankerResponse(BaseModel): + """Model for rerankerresponse data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + documents: list[RankedDocumentData] + + class Content(RootModel[list[Part]]): """Root model for content.""" root: list[Part] -class DocumentPart(RootModel[TextPart | MediaPart]): - """Root model for documentpart.""" +class DocumentData(BaseModel): + """Model for documentdata data.""" - root: TextPart | MediaPart + model_config = ConfigDict(extra='forbid', populate_by_name=True) + content: list[DocumentPart] + metadata: dict[str, Any] | None = None + + +class EmbedRequest(BaseModel): + """Model for embedrequest data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + input: list[DocumentData] + options: Any | None = None class GenerateResponseChunk(BaseModel): @@ -857,41 +897,40 @@ class ModelResponseChunk(BaseModel): aggregated: Aggregated | None = None -class RankedDocumentData(BaseModel): - """Model for rankeddocumentdata data.""" +class RerankerRequest(BaseModel): + """Model for rerankerrequest data.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) - content: list[DocumentPart] - metadata: RankedDocumentMetadata + query: DocumentData + documents: list[DocumentData] + options: Any | None = None -class RerankerResponse(BaseModel): - """Model for rerankerresponse data.""" +class RetrieverRequest(BaseModel): + """Model for retrieverrequest data.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) - documents: list[RankedDocumentData] + query: DocumentData + options: Any | None = None -class Messages(RootModel[list[Message]]): - """Root model for messages.""" +class RetrieverResponse(BaseModel): + """Model for retrieverresponse data.""" - root: list[Message] + model_config = ConfigDict(extra='forbid', populate_by_name=True) + documents: list[DocumentData] -class DocumentData(BaseModel): - """Model for documentdata data.""" +class Docs(RootModel[list[DocumentData]]): + """Root model for docs.""" - model_config = ConfigDict(extra='forbid', populate_by_name=True) - content: list[DocumentPart] - metadata: dict[str, Any] | None = None + root: list[DocumentData] -class EmbedRequest(BaseModel): - """Model for embedrequest data.""" +class Messages(RootModel[list[Message]]): + """Root model for messages.""" - model_config = ConfigDict(extra='forbid', populate_by_name=True) - input: list[DocumentData] - options: Any | None = None + root: list[Message] class Candidate(BaseModel): @@ -952,42 +991,6 @@ class GenerateResponse(BaseModel): candidates: list[Candidate] | None = None -class RerankerRequest(BaseModel): - """Model for rerankerrequest data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - query: DocumentData - documents: list[DocumentData] - options: Any | None = None - - -class RetrieverRequest(BaseModel): - """Model for retrieverrequest data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - query: DocumentData - options: Any | None = None - - -class RetrieverResponse(BaseModel): - """Model for retrieverresponse data.""" - - model_config = ConfigDict(extra='forbid', populate_by_name=True) - documents: list[DocumentData] - - -class Docs(RootModel[list[DocumentData]]): - """Root model for docs.""" - - root: list[DocumentData] - - -class Request(RootModel[GenerateRequest]): - """Root model for request.""" - - root: GenerateRequest - - class ModelRequest(BaseModel): """Model for modelrequest data.""" @@ -1000,6 +1003,12 @@ class ModelRequest(BaseModel): docs: Docs | None = None +class Request(RootModel[GenerateRequest]): + """Root model for request.""" + + root: GenerateRequest + + class ModelResponse(BaseModel): """Model for modelresponse data.""" diff --git a/py/packages/genkit/tests/genkit/blocks/prompt_test.py b/py/packages/genkit/tests/genkit/blocks/prompt_test.py index 67b09291b5..1b62b92835 100644 --- a/py/packages/genkit/tests/genkit/blocks/prompt_test.py +++ b/py/packages/genkit/tests/genkit/blocks/prompt_test.py @@ -202,7 +202,8 @@ def test_tool(input: ToolInput): ), ] -@pytest.mark.skip(reason="issues when running on CI") + +@pytest.mark.skip(reason='issues when running on CI') @pytest.mark.asyncio @pytest.mark.parametrize( 'test_case, prompt, input, input_option, context, want_rendered', From 0ae5ec57be328cf27de442f2f60f7b82d6b7adf6 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 14 Nov 2025 20:25:35 -0500 Subject: [PATCH 05/21] undo --- py/packages/genkit/tests/genkit/blocks/prompt_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/packages/genkit/tests/genkit/blocks/prompt_test.py b/py/packages/genkit/tests/genkit/blocks/prompt_test.py index 1b62b92835..21eb0b0e33 100644 --- a/py/packages/genkit/tests/genkit/blocks/prompt_test.py +++ b/py/packages/genkit/tests/genkit/blocks/prompt_test.py @@ -202,7 +202,6 @@ def test_tool(input: ToolInput): ), ] - @pytest.mark.skip(reason='issues when running on CI') @pytest.mark.asyncio @pytest.mark.parametrize( From 63ca4e77e1adae2dd3ea8bd00f8c8b7792097992 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 14 Nov 2025 20:27:45 -0500 Subject: [PATCH 06/21] fmt --- py/packages/genkit/tests/genkit/blocks/prompt_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py/packages/genkit/tests/genkit/blocks/prompt_test.py b/py/packages/genkit/tests/genkit/blocks/prompt_test.py index 21eb0b0e33..1b62b92835 100644 --- a/py/packages/genkit/tests/genkit/blocks/prompt_test.py +++ b/py/packages/genkit/tests/genkit/blocks/prompt_test.py @@ -202,6 +202,7 @@ def test_tool(input: ToolInput): ), ] + @pytest.mark.skip(reason='issues when running on CI') @pytest.mark.asyncio @pytest.mark.parametrize( From b9f2f1f6cbe855a2b0c2ac7a5816409e64c9eeaf Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Sun, 16 Nov 2025 21:53:25 -0500 Subject: [PATCH 07/21] tool.v2 --- js/ai/src/generate.ts | 4 ++-- js/ai/src/generate/resolve-tool-requests.ts | 2 +- js/ai/src/tool.ts | 12 ++++++------ js/ai/tests/tool_test.ts | 4 ++-- js/core/src/registry.ts | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/js/ai/src/generate.ts b/js/ai/src/generate.ts index 5b2be557f9..a2abfe8a91 100755 --- a/js/ai/src/generate.ts +++ b/js/ai/src/generate.ts @@ -588,8 +588,8 @@ async function resolveFullToolNames( if (await registry.lookupAction(`/tool/${name}`)) { return [`/tool/${name}`]; } - if (await registry.lookupAction(`/multipart-tool/${name}`)) { - return [`/multipart-tool/${name}`]; + if (await registry.lookupAction(`/tool.v2/${name}`)) { + return [`/tool.v2/${name}`]; } if (await registry.lookupAction(`/prompt/${name}`)) { return [`/prompt/${name}`]; diff --git a/js/ai/src/generate/resolve-tool-requests.ts b/js/ai/src/generate/resolve-tool-requests.ts index 32dabcce0d..0fe23a7b1f 100644 --- a/js/ai/src/generate/resolve-tool-requests.ts +++ b/js/ai/src/generate/resolve-tool-requests.ts @@ -122,7 +122,7 @@ export async function resolveToolRequest( // otherwise, execute the tool and catch interrupts try { const output = await tool(part.toolRequest.input, toRunOptions(part)); - if (tool.__action.actionType === 'multipart-tool') { + if (tool.__action.actionType === 'tool.v2') { const multipartResponse = output as z.infer< typeof MultipartToolResponseSchema >; diff --git a/js/ai/src/tool.ts b/js/ai/src/tool.ts index 44e167b48b..156f22b1ba 100644 --- a/js/ai/src/tool.ts +++ b/js/ai/src/tool.ts @@ -101,7 +101,7 @@ export type ToolAction< }; /** - * An action with a `multipart-tool` type. + * An action with a `tool.v2` type. */ export type MultipartToolAction< I extends z.ZodTypeAny = z.ZodTypeAny, @@ -115,7 +115,7 @@ export type MultipartToolAction< Resumable & { __action: { metadata: { - type: 'multipart-tool'; + type: 'tool.v2'; }; }; }; @@ -238,7 +238,7 @@ export async function lookupToolByName( const tool = (await registry.lookupAction(name)) || (await registry.lookupAction(`/tool/${name}`)) || - (await registry.lookupAction(`/multipart-tool/${name}`)) || + (await registry.lookupAction(`/tool.v2/${name}`)) || (await registry.lookupAction(`/prompt/${name}`)) || (await registry.lookupAction(`/dynamic-action-provider/${name}`)); if (!tool) { @@ -325,7 +325,7 @@ export function defineTool( fn?: ToolFn | MultipartToolFn ): ToolAction | MultipartToolAction { const a = tool(config, fn); - registry.registerAction(config.multipart ? 'multipart-tool' : 'tool', a); + registry.registerAction(config.multipart ? 'tool.v2' : 'tool', a); return a as ToolAction; } @@ -519,8 +519,8 @@ function multipartTool( { ...config, outputSchema: MultipartToolResponseSchema, - actionType: 'multipart-tool', - metadata: { ...(config.metadata || {}), type: 'multipart-tool' }, + actionType: 'tool.v2', + metadata: { ...(config.metadata || {}), type: 'tool.v2' }, }, (i, runOptions) => { const interrupt = interruptTool(runOptions.registry); diff --git a/js/ai/tests/tool_test.ts b/js/ai/tests/tool_test.ts index cd0616b896..d694aa9b98 100644 --- a/js/ai/tests/tool_test.ts +++ b/js/ai/tests/tool_test.ts @@ -120,8 +120,8 @@ describe('defineInterrupt', () => { }; } ); - assert.equal(t.__action.metadata.type, 'multipart-tool'); - assert.equal(t.__action.actionType, 'multipart-tool'); + assert.equal(t.__action.metadata.type, 'tool.v2'); + assert.equal(t.__action.actionType, 'tool.v2'); const result = await t({}); assert.deepStrictEqual(result, { output: 'main output', diff --git a/js/core/src/registry.ts b/js/core/src/registry.ts index 8036da0491..a758a047d1 100644 --- a/js/core/src/registry.ts +++ b/js/core/src/registry.ts @@ -53,7 +53,7 @@ const ACTION_TYPES = [ 'reranker', 'retriever', 'tool', - 'multipart-tool', + 'tool.v2', 'util', 'resource', ] as const; From 3f8ef7c5f25ab4b8d57c99603950ca80cc03dfb2 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Mon, 17 Nov 2025 12:01:31 -0500 Subject: [PATCH 08/21] v2 action --- js/ai/src/tool.ts | 26 +++++++++++++++++++++++--- js/ai/tests/tool_test.ts | 16 ++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/js/ai/src/tool.ts b/js/ai/src/tool.ts index 156f22b1ba..69573add65 100644 --- a/js/ai/src/tool.ts +++ b/js/ai/src/tool.ts @@ -16,6 +16,7 @@ import { action, + ActionFnArg, assertUnstable, isAction, stripUndefinedProps, @@ -279,7 +280,7 @@ export function toToolDefinition( return out; } -export interface ToolFnOptions { +export interface ToolFnOptions extends ActionFnArg { /** * A function that can be called during tool execution that will result in the tool * getting interrupted (immediately) and tool request returned to the upstream caller. @@ -300,7 +301,7 @@ export type MultipartToolFn = ( ) => Promise<{ output?: z.infer; fallbackOutput?: z.infer; - content: Part[]; + content?: Part[]; }>; export function defineTool( @@ -326,6 +327,10 @@ export function defineTool( ): ToolAction | MultipartToolAction { const a = tool(config, fn); registry.registerAction(config.multipart ? 'tool.v2' : 'tool', a); + if (!config.multipart) { + // For non-multipart tools, we register a v2 tool action as well + registry.registerAction('tool.v2', basicToolV2(config, fn as ToolFn)); + } return a as ToolAction; } @@ -505,10 +510,25 @@ function basicTool( return a; } +function basicToolV2( + config: ToolConfig, + fn?: ToolFn +): MultipartToolAction { + return multipartTool(config, async (input, ctx) => { + if (!fn) { + const interrupt = interruptTool(ctx.registry); + return interrupt(); + } + return { + output: await fn(input, ctx), + }; + }); +} + export const MultipartToolResponseSchema = z.object({ output: z.any().optional(), fallbackOutput: z.any().optional(), - content: z.array(PartSchema), + content: z.array(PartSchema).optional(), }); function multipartTool( diff --git a/js/ai/tests/tool_test.ts b/js/ai/tests/tool_test.ts index d694aa9b98..4dfcb4f5b7 100644 --- a/js/ai/tests/tool_test.ts +++ b/js/ai/tests/tool_test.ts @@ -307,4 +307,20 @@ describe('defineTool', () => { ); }); }); + + it('should register a v1 tool as v2 as well', async () => { + defineTool(registry, { name: 'test', description: 'test' }, async () => {}); + assert.ok(await registry.lookupAction('/tool/test')); + assert.ok(await registry.lookupAction('/tool.v2/test')); + }); + + it('should only register a multipart tool as v2', async () => { + defineTool( + registry, + { name: 'test', description: 'test', multipart: true }, + async () => {} + ); + assert.ok(await registry.lookupAction('/tool.v2/test')); + assert.equal(await registry.lookupAction('/tool/test'), undefined); + }); }); From 32964e923218169754e17423c842587b52f78640 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Mon, 17 Nov 2025 12:15:57 -0500 Subject: [PATCH 09/21] another test --- js/ai/tests/tool_test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/js/ai/tests/tool_test.ts b/js/ai/tests/tool_test.ts index 4dfcb4f5b7..2bb71dccca 100644 --- a/js/ai/tests/tool_test.ts +++ b/js/ai/tests/tool_test.ts @@ -323,4 +323,16 @@ describe('defineTool', () => { assert.ok(await registry.lookupAction('/tool.v2/test')); assert.equal(await registry.lookupAction('/tool/test'), undefined); }); + + it('should wrap v1 tool output when called as v2', async () => { + defineTool( + registry, + { name: 'test', description: 'test' }, + async () => 'foo' + ); + const action = await registry.lookupAction('/tool.v2/test'); + assert.ok(action); + const result = await action!({}); + assert.deepStrictEqual(result, { output: 'foo' }); + }); }); From 5cd0a0672d976e32af70b4496be0a35310a51019 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Mon, 17 Nov 2025 12:28:41 -0500 Subject: [PATCH 10/21] added multipart metadata --- js/ai/src/tool.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/js/ai/src/tool.ts b/js/ai/src/tool.ts index 69573add65..f3c79573fa 100644 --- a/js/ai/src/tool.ts +++ b/js/ai/src/tool.ts @@ -540,7 +540,11 @@ function multipartTool( ...config, outputSchema: MultipartToolResponseSchema, actionType: 'tool.v2', - metadata: { ...(config.metadata || {}), type: 'tool.v2' }, + metadata: { + ...(config.metadata || {}), + type: 'tool.v2', + tool: { multipart: true }, + }, }, (i, runOptions) => { const interrupt = interruptTool(runOptions.registry); From 5fd063771fb3778bb6060ab0f87b2c75bf03c335 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Thu, 20 Nov 2025 22:24:13 -0500 Subject: [PATCH 11/21] fix schema --- genkit-tools/common/src/types/parts.ts | 3 ++- genkit-tools/genkit-schema.json | 12 ++++------ go/ai/gen.go | 23 +++++++------------- go/core/schemas.config | 3 +++ js/ai/src/parts.ts | 3 ++- py/packages/genkit/src/genkit/core/typing.py | 9 +------- 6 files changed, 20 insertions(+), 33 deletions(-) diff --git a/genkit-tools/common/src/types/parts.ts b/genkit-tools/common/src/types/parts.ts index d05d4c3553..df93190bd1 100644 --- a/genkit-tools/common/src/types/parts.ts +++ b/genkit-tools/common/src/types/parts.ts @@ -86,6 +86,8 @@ export const ToolRequestSchema = z.object({ name: z.string(), /** The input parameters for the tool, usually a JSON object. */ input: z.unknown().optional(), + /** Whether the request is a partial chunk. */ + partial: z.boolean().optional(), }); /** @@ -108,7 +110,6 @@ const ToolResponseSchemaBase = z.object({ name: z.string(), /** The output data returned from the tool, usually a JSON object. */ output: z.unknown().optional(), - payloadStrategy: z.enum(['both', 'fallback']).optional(), // default: both }); /** diff --git a/genkit-tools/genkit-schema.json b/genkit-tools/genkit-schema.json index 02c5e63136..74499d3468 100644 --- a/genkit-tools/genkit-schema.json +++ b/genkit-tools/genkit-schema.json @@ -1246,7 +1246,10 @@ "name": { "type": "string" }, - "input": {} + "input": {}, + "partial": { + "type": "boolean" + } }, "required": [ "name" @@ -1263,13 +1266,6 @@ "type": "string" }, "output": {}, - "payloadStrategy": { - "type": "string", - "enum": [ - "both", - "fallback" - ] - }, "content": { "type": "array" } diff --git a/go/ai/gen.go b/go/ai/gen.go index 5b39121259..c59a62eee4 100644 --- a/go/ai/gen.go +++ b/go/ai/gen.go @@ -376,9 +376,10 @@ type ToolDefinition struct { type ToolRequest struct { // Input is a JSON object describing the input values to the tool. // An example might be map[string]any{"country":"USA", "president":3}. - Input any `json:"input,omitempty"` - Name string `json:"name,omitempty"` - Ref string `json:"ref,omitempty"` + Input any `json:"input,omitempty"` + Name string `json:"name,omitempty"` + Partial bool `json:"partial,omitempty"` + Ref string `json:"ref,omitempty"` } type toolRequestPart struct { @@ -390,13 +391,12 @@ type toolRequestPart struct { // the results of running a specific tool on the arguments passed to the client // by the model in a [ToolRequest]. type ToolResponse struct { - Content []any `json:"content,omitempty"` - Name string `json:"name,omitempty"` + Content []*Part `json:"content,omitempty"` + Name string `json:"name,omitempty"` // Output is a JSON object describing the results of running the tool. // An example might be map[string]any{"name":"Thomas Jefferson", "born":1743}. - Output any `json:"output,omitempty"` - PayloadStrategy ToolResponsePayloadStrategy `json:"payloadStrategy,omitempty"` - Ref string `json:"ref,omitempty"` + Output any `json:"output,omitempty"` + Ref string `json:"ref,omitempty"` } type toolResponsePart struct { @@ -404,13 +404,6 @@ type toolResponsePart struct { ToolResponse *ToolResponse `json:"toolResponse,omitempty"` } -type ToolResponsePayloadStrategy string - -const ( - ToolResponsePayloadStrategyBoth ToolResponsePayloadStrategy = "both" - ToolResponsePayloadStrategyFallback ToolResponsePayloadStrategy = "fallback" -) - type TraceMetadata struct { FeatureName string `json:"featureName,omitempty"` Paths []*PathMetadata `json:"paths,omitempty"` diff --git a/go/core/schemas.config b/go/core/schemas.config index fba66996b5..2caf39e9f7 100644 --- a/go/core/schemas.config +++ b/go/core/schemas.config @@ -67,6 +67,9 @@ the results of running a specific tool on the arguments passed to the client by the model in a [ToolRequest]. . +ToolResponse.content type []*Part + + Candidate omit DocumentData pkg ai diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts index 32117891da..2cd1cf78c4 100644 --- a/js/ai/src/parts.ts +++ b/js/ai/src/parts.ts @@ -81,6 +81,8 @@ export const ToolRequestSchema = z.object({ name: z.string(), /** The input parameters for the tool, usually a JSON object. */ input: z.unknown().optional(), + /** Whether the request is a partial chunk. */ + partial: z.boolean().optional(), }); export type ToolRequest = z.infer; @@ -107,7 +109,6 @@ const ToolResponseSchemaBase = z.object({ name: z.string(), /** The output data returned from the tool, usually a JSON object. */ output: z.unknown().optional(), - payloadStrategy: z.enum(['both', 'fallback']).optional(), // default: both }); /** diff --git a/py/packages/genkit/src/genkit/core/typing.py b/py/packages/genkit/src/genkit/core/typing.py index c70cd142c4..e8bc222c5e 100644 --- a/py/packages/genkit/src/genkit/core/typing.py +++ b/py/packages/genkit/src/genkit/core/typing.py @@ -338,13 +338,7 @@ class ToolRequest(BaseModel): ref: str | None = None name: str input: Any | None = None - - -class PayloadStrategy(StrEnum): - """Enumeration of payloadstrategy values.""" - - BOTH = 'both' - FALLBACK = 'fallback' + partial: bool | None = None class ToolResponse(BaseModel): @@ -354,7 +348,6 @@ class ToolResponse(BaseModel): ref: str | None = None name: str output: Any | None = None - payload_strategy: PayloadStrategy | None = Field(None, alias='payloadStrategy') content: list | None = None From a908067a5bc08ee57e52a32d1816552ba459ed90 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Fri, 21 Nov 2025 17:28:18 -0500 Subject: [PATCH 12/21] feat(js/plugins/google-genai): added support for tool request streaming for Gemini 3 --- js/plugins/google-genai/package.json | 4 +- .../google-genai/src/common/converters.ts | 196 ++++++++++--- js/plugins/google-genai/src/common/types.ts | 39 ++- js/plugins/google-genai/src/common/utils.ts | 65 ++++- .../google-genai/src/googleai/gemini.ts | 15 +- js/plugins/google-genai/src/googleai/index.ts | 2 +- .../google-genai/src/vertexai/gemini.ts | 18 +- js/plugins/google-genai/src/vertexai/index.ts | 2 +- .../tests/common/converters_test.ts | 261 +++++++++++++++++- .../google-genai/tests/common/utils_test.ts | 124 +++++++++ .../tests/googleai/gemini_test.ts | 10 +- .../tests/vertexai/gemini_test.ts | 14 +- js/pnpm-lock.yaml | 196 ++++++------- js/testapps/basic-gemini/package.json | 1 - .../basic-gemini/src/index-vertexai.ts | 62 ++--- js/testapps/basic-gemini/src/index.ts | 32 +-- js/testapps/basic-gemini/src/types.ts | 48 ++++ 17 files changed, 837 insertions(+), 252 deletions(-) create mode 100644 js/testapps/basic-gemini/src/types.ts diff --git a/js/plugins/google-genai/package.json b/js/plugins/google-genai/package.json index b2c656bfac..142c8f2961 100644 --- a/js/plugins/google-genai/package.json +++ b/js/plugins/google-genai/package.json @@ -31,12 +31,14 @@ "author": "genkit", "license": "Apache-2.0", "dependencies": { - "google-auth-library": "^9.14.2" + "google-auth-library": "^9.14.2", + "jsonpath-plus": "^10.3.0" }, "peerDependencies": { "genkit": "workspace:^" }, "devDependencies": { + "@types/jsonpath-plus": "^5.0.5", "@types/node": "^20.11.16", "@types/sinon": "^17.0.4", "npm-run-all": "^4.1.5", diff --git a/js/plugins/google-genai/src/common/converters.ts b/js/plugins/google-genai/src/common/converters.ts index 49227928b6..fb3971d4a7 100644 --- a/js/plugins/google-genai/src/common/converters.ts +++ b/js/plugins/google-genai/src/common/converters.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { GenkitError, z } from 'genkit'; +import { GenkitError, ToolRequest, z } from 'genkit'; import { CandidateData, MessageData, @@ -23,6 +23,7 @@ import { TextPart, ToolDefinition, } from 'genkit/model'; +import { JSONPath } from 'jsonpath-plus'; import { FunctionCallingMode, FunctionDeclaration, @@ -139,38 +140,41 @@ function toGeminiToolRequest(part: Part): GeminiPart { if (!part.toolRequest?.input) { throw Error('Invalid ToolRequestPart: input was missing.'); } - return maybeAddGeminiThoughtSignature(part, { - functionCall: { - name: part.toolRequest.name, - args: part.toolRequest.input, - }, - }); + const functionCall: GeminiPart['functionCall'] = { + name: part.toolRequest.name, + args: part.toolRequest.input, + }; + if (part.toolRequest.ref) { + functionCall.id = part.toolRequest.ref; + } + return maybeAddGeminiThoughtSignature(part, { functionCall }); } function toGeminiToolResponse(part: Part): GeminiPart { if (!part.toolResponse?.output) { throw Error('Invalid ToolResponsePart: output was missing.'); } - return maybeAddGeminiThoughtSignature(part, { - functionResponse: { + const functionResponse: GeminiPart['functionResponse'] = { + name: part.toolResponse.name, + response: { name: part.toolResponse.name, - response: { - name: part.toolResponse.name, - content: part.toolResponse.output, - }, + content: part.toolResponse.output, }, + }; + if (part.toolResponse.ref) { + functionResponse.id = part.toolResponse.ref; + } + return maybeAddGeminiThoughtSignature(part, { + functionResponse, }); } function toGeminiReasoning(part: Part): GeminiPart { const out: GeminiPart = { thought: true }; - if (typeof part.metadata?.thoughtSignature === 'string') { - out.thoughtSignature = part.metadata.thoughtSignature; - } if (part.reasoning?.length) { out.text = part.reasoning; } - return out; + return maybeAddGeminiThoughtSignature(part, out); } function toGeminiCustom(part: Part): GeminiPart { @@ -354,10 +358,9 @@ function maybeAddThoughtSignature(geminiPart: GeminiPart, part: Part): Part { } function fromGeminiThought(part: GeminiPart): Part { - return { + return maybeAddThoughtSignature(part, { reasoning: part.text || '', - metadata: { thoughtSignature: part.thoughtSignature }, - }; + }); } function fromGeminiInlineData(part: GeminiPart): Part { @@ -400,34 +403,147 @@ function fromGeminiFileData(part: GeminiPart): Part { }); } -function fromGeminiFunctionCall(part: GeminiPart, ref: string): Part { +export function applyStreamingJsonPath(target: object, geminiPart: GeminiPart) { + if (!geminiPart.functionCall?.partialArgs) { + return; + } + + for (const partialArg of geminiPart.functionCall.partialArgs) { + if (!partialArg.jsonPath) { + continue; + } + let value: boolean | string | number | null | undefined; + if (partialArg.boolValue !== undefined) { + value = partialArg.boolValue; + } else if (partialArg.nullValue !== undefined) { + value = null; + } else if (partialArg.numberValue !== undefined) { + value = partialArg.numberValue; + } else if (partialArg.stringValue !== undefined) { + value = partialArg.stringValue; + } + if (value === undefined) { + continue; + } + + let current: any = target; + const path = JSONPath.toPathArray(partialArg.jsonPath); + for (let i = 1; i < path.length - 1; i++) { + const key = path[i]; + const nextKey = path[i + 1]; + if (current[key] === undefined) { + if (!isNaN(parseInt(nextKey, 10))) { + current[key] = []; + } else { + current[key] = {}; + } + } + current = current[key]; + } + + const finalKey = path[path.length - 1]; + if ( + partialArg.stringValue !== undefined && + typeof current[finalKey] === 'string' + ) { + current[finalKey] += partialArg.stringValue; + } else { + current[finalKey] = value as any; + } + } +} + +function fromGeminiFunctionCall( + part: GeminiPart, + previousChunks?: CandidateData[] +): Part { if (!part.functionCall) { throw Error( 'Invalid Gemini Function Call Part: missing function call data' ); } - return maybeAddThoughtSignature(part, { - toolRequest: { - name: part.functionCall.name, - input: part.functionCall.args, - ref, - }, - }); + const req: Partial = { + name: part.functionCall.name, + input: part.functionCall.args, + }; + + if (part.functionCall.id) { + req.ref = part.functionCall.id; + } + + if (part.functionCall.willContinue) { + req.partial = true; + } + + handleFunctionCallPartials(req, part, previousChunks); + + const toolRequest: Part = { toolRequest: req as ToolRequest }; + + return maybeAddThoughtSignature(part, toolRequest); } -function fromGeminiFunctionResponse(part: GeminiPart, ref?: string): Part { +function handleFunctionCallPartials( + req: Partial, + part: GeminiPart, + previousChunks?: CandidateData[] +) { + if (!part.functionCall) { + throw Error( + 'Invalid Gemini Function Call Part: missing function call data' + ); + } + + // we try to find if there's a previous partial tool request part. + const prevPart = previousChunks?.at(-1)?.message.content?.at(-1); + const prevPartialToolRequestPart = + prevPart?.toolRequest && prevPart?.toolRequest.partial + ? prevPart + : undefined; + + // if the current functionCall has partialArgs, we try to apply the diff to the + // potentilly including the previous partial part. + if (part.functionCall.partialArgs) { + const newInput = prevPartialToolRequestPart?.toolRequest?.input + ? JSON.parse(JSON.stringify(prevPartialToolRequestPart.toolRequest.input)) + : {}; + applyStreamingJsonPath(newInput, part); + req.input = newInput; + } + + // If there's a previous partial part, we copy some fields over, because the + // API wil not return these. + if (prevPartialToolRequestPart) { + if (!req.name) { + req.name = prevPartialToolRequestPart.toolRequest.name; + } + if (!req.ref) { + req.ref = prevPartialToolRequestPart.toolRequest.ref; + } + // This is a special case for the final partial function call chunk from the API, + // it will have nothing... so we need to make sure to copy the input + // from the previous. + if (req.input === undefined) { + req.input = prevPartialToolRequestPart.toolRequest.input; + } + } +} + +function fromGeminiFunctionResponse(part: GeminiPart): Part { if (!part.functionResponse) { throw new Error( 'Invalid Gemini Function Call Part: missing function call data' ); } - return maybeAddThoughtSignature(part, { + const toolResponse: Part = { toolResponse: { name: part.functionResponse.name.replace(/__/g, '/'), // restore slashes output: part.functionResponse.response, - ref, }, - }); + }; + if (part.functionResponse.id) { + toolResponse.toolResponse.ref = part.functionResponse.id; + } + return maybeAddThoughtSignature(part, toolResponse); } function fromExecutableCode(part: GeminiPart): Part { @@ -462,20 +578,26 @@ function fromGeminiText(part: GeminiPart): Part { return maybeAddThoughtSignature(part, { text: part.text } as TextPart); } -function fromGeminiPart(part: GeminiPart, ref: string): Part { +function fromGeminiPart( + part: GeminiPart, + previousChunks?: CandidateData[] +): Part { if (part.thought) return fromGeminiThought(part as any); if (typeof part.text === 'string') return fromGeminiText(part); if (part.inlineData) return fromGeminiInlineData(part); if (part.fileData) return fromGeminiFileData(part); - if (part.functionCall) return fromGeminiFunctionCall(part, ref); - if (part.functionResponse) return fromGeminiFunctionResponse(part, ref); + if (part.functionCall) return fromGeminiFunctionCall(part, previousChunks); + if (part.functionResponse) return fromGeminiFunctionResponse(part); if (part.executableCode) return fromExecutableCode(part); if (part.codeExecutionResult) return fromCodeExecutionResult(part); throw new Error('Unsupported GeminiPart type ' + JSON.stringify(part)); } -export function fromGeminiCandidate(candidate: GeminiCandidate): CandidateData { +export function fromGeminiCandidate( + candidate: GeminiCandidate, + previousChunks?: CandidateData[] +): CandidateData { const parts = candidate.content?.parts || []; const genkitCandidate: CandidateData = { index: candidate.index || 0, @@ -484,7 +606,7 @@ export function fromGeminiCandidate(candidate: GeminiCandidate): CandidateData { content: parts // the model sometimes returns empty parts, ignore those. .filter((p) => Object.keys(p).length > 0) - .map((part, index) => fromGeminiPart(part, index.toString())), + .map((part) => fromGeminiPart(part, previousChunks)), }, finishReason: fromGeminiFinishReason(candidate.finishReason), finishMessage: candidate.finishMessage, diff --git a/js/plugins/google-genai/src/common/types.ts b/js/plugins/google-genai/src/common/types.ts index 6ada7b6d58..d702b4a110 100644 --- a/js/plugins/google-genai/src/common/types.ts +++ b/js/plugins/google-genai/src/common/types.ts @@ -296,18 +296,44 @@ export declare interface GenerativeContentBlob { * values. */ export declare interface FunctionCall { + /** + * The unique id of the function call. If populated, the client to execute the + * `function_call` and return the response with the matching `id`. + */ + id?: string; /** The name of the function specified in FunctionDeclaration.name. */ - name: string; + name?: string; /** The arguments to pass to the function. */ - args: object; + args?: object; + /** Optional. The partial argument value of the function call. If provided, represents the arguments/fields that are streamed incrementally. This field is not supported in Gemini API. */ + partialArgs?: PartialArg[]; + /** Optional. Whether this is the last part of the FunctionCall. If true, another partial message for the current FunctionCall is expected to follow. This field is not supported in Gemini API. */ + willContinue?: boolean; +} + +/** Partial argument value of the function call. This data type is not supported in Gemini API. */ +export declare interface PartialArg { + /** Optional. Represents a null value. */ + nullValue?: 'NULL_VALUE'; + /** Optional. Represents a double value. */ + numberValue?: number; + /** Optional. Represents a string value. */ + stringValue?: string; + /** Optional. Represents a boolean value. */ + boolValue?: boolean; + /** Required. A JSON Path (RFC 9535) to the argument being streamed. https://datatracker.ietf.org/doc/html/rfc9535. e.g. "$.foo.bar[0].data". */ + jsonPath?: string; + /** Optional. Whether this is not the last part of the same json_path. If true, another PartialArg message for the current json_path is expected to follow. */ + willContinue?: boolean; } - /** * The result output of a FunctionCall that contains a string representing * the FunctionDeclaration.name and a structured JSON object containing any * output from the function call. It is used as context to the model. */ export declare interface FunctionResponse { + /** Optional. The id of the function call this response is for. Populated by the client to match the corresponding function call `id`. */ + id?: string; /** The name of the function specified in FunctionDeclaration.name. */ name: string; /** The expected response from the model. */ @@ -1051,6 +1077,13 @@ export declare interface FunctionCallingConfig { * will predict a function call from the set of function names provided. */ allowedFunctionNames?: string[]; + + /** + * When set to true, arguments of a single function call will be streamed out + * in multiple parts/contents/responses. Partial parameter results will be + * returned in the [FunctionCall.partial_args] field. + */ + streamFunctionCallArguments?: boolean; } export declare interface LatLng { diff --git a/js/plugins/google-genai/src/common/utils.ts b/js/plugins/google-genai/src/common/utils.ts index 2864b7511c..9759f1b144 100644 --- a/js/plugins/google-genai/src/common/utils.ts +++ b/js/plugins/google-genai/src/common/utils.ts @@ -25,6 +25,7 @@ import { z, } from 'genkit'; import { GenerateRequest } from 'genkit/model'; +import { applyStreamingJsonPath } from './converters.js'; import { GenerateContentCandidate, GenerateContentResponse, @@ -416,7 +417,58 @@ async function getResponsePromise( } } -function aggregateResponses( +function handleFunctionCall( + part: Part, + newPart: Partial, + activePartialToolRequest: Part | null +): { + shouldContinue: boolean; + newActivePartialToolRequest: Part | null; +} { + // If there's an active partial tool request, we're in the middle of a stream. + if (activePartialToolRequest) { + applyStreamingJsonPath(activePartialToolRequest.functionCall!.args!, part); + // If `willContinue` is false, this is the end of the stream. + if (!part.functionCall!.willContinue) { + newPart.thoughtSignature = activePartialToolRequest.thoughtSignature; + part.functionCall = activePartialToolRequest.functionCall; + delete part.functionCall!.willContinue; + activePartialToolRequest = null; + } else { + // If `willContinue` is true, we're still in the middle of a stream. + // This is a partial result, so we skip adding it to the parts list. + return { + shouldContinue: true, + newActivePartialToolRequest: activePartialToolRequest, + }; + } + // If `willContinue` is true on a part and there's no active partial request, + // this is the start of a new streaming tool call. + } else if (part.functionCall!.willContinue) { + activePartialToolRequest = { + ...part, + functionCall: { + ...part.functionCall, + args: part.functionCall!.args || {}, + }, + }; + applyStreamingJsonPath(activePartialToolRequest.functionCall!.args!, part); + // This is the start of a partial, so we skip adding it to the parts list. + return { + shouldContinue: true, + newActivePartialToolRequest: activePartialToolRequest, + }; + } + + // If we're here, it's a regular, non-streaming tool call. + newPart.functionCall = part.functionCall; + return { + shouldContinue: false, + newActivePartialToolRequest: activePartialToolRequest, + }; +} + +export function aggregateResponses( responses: GenerateContentResponse[] ): GenerateContentResponse { const lastResponse = responses.at(-1); @@ -429,6 +481,7 @@ function aggregateResponses( if (lastResponse.promptFeedback) { aggregatedResponse.promptFeedback = lastResponse.promptFeedback; } + let activePartialToolRequest: Part | null = null; for (const response of responses) { for (const candidate of response.candidates ?? []) { const index = candidate.index ?? 0; @@ -486,7 +539,15 @@ function aggregateResponses( newPart.text = part.text; } if (part.functionCall) { - newPart.functionCall = part.functionCall; + // function calls are special, there can be partials, so we need aggregate + // the partials into final functionCall. + const { shouldContinue, newActivePartialToolRequest } = + handleFunctionCall(part, newPart, activePartialToolRequest); + if (shouldContinue) { + activePartialToolRequest = newActivePartialToolRequest; + continue; + } + activePartialToolRequest = newActivePartialToolRequest; } if (part.executableCode) { newPart.executableCode = part.executableCode; diff --git a/js/plugins/google-genai/src/googleai/gemini.ts b/js/plugins/google-genai/src/googleai/gemini.ts index bfe336f1db..d8b2a013b7 100644 --- a/js/plugins/google-genai/src/googleai/gemini.ts +++ b/js/plugins/google-genai/src/googleai/gemini.ts @@ -16,6 +16,7 @@ import { ActionMetadata, GenkitError, modelActionMetadata, z } from 'genkit'; import { + CandidateData, GenerationCommonConfigDescriptions, GenerationCommonConfigSchema, ModelAction, @@ -424,7 +425,7 @@ const KNOWN_MODELS = { ...KNOWN_GEMMA_MODELS, }; -export function model( +export function googleaiModelRef( version: string, config: GeminiConfig | GeminiTtsConfig | GemmaConfig = {} ): ModelReference { @@ -465,7 +466,7 @@ export function listActions(models: Model[]): ActionMetadata[] { // Filter out deprecated .filter((m) => !m.description || !m.description.includes('deprecated')) .map((m) => { - const ref = model(m.name); + const ref = googleaiModelRef(m.name); return modelActionMetadata({ name: ref.name, info: ref.info, @@ -489,7 +490,7 @@ export function defineModel( pluginOptions?: GoogleAIPluginOptions ): ModelAction { checkApiKey(pluginOptions?.apiKey); - const ref = model(name); + const ref = googleaiModelRef(name); const clientOptions: ClientOptions = { apiVersion: pluginOptions?.apiVersion, baseUrl: pluginOptions?.baseUrl, @@ -661,10 +662,11 @@ export function defineModel( generateContentRequest, clientOpt ); - + const chunks: CandidateData[] = []; for await (const item of result.stream) { item.candidates?.forEach((candidate) => { - const c = fromGeminiCandidate(candidate); + const c = fromGeminiCandidate(candidate, chunks); + chunks.push(c); sendChunk({ index: c.index, content: c.message.content, @@ -692,7 +694,8 @@ export function defineModel( }); } - const candidateData = candidates.map(fromGeminiCandidate) || []; + const candidateData = + candidates.map((c) => fromGeminiCandidate(c)) || []; return { candidates: candidateData, diff --git a/js/plugins/google-genai/src/googleai/index.ts b/js/plugins/google-genai/src/googleai/index.ts index 220854ce04..3af11f6105 100644 --- a/js/plugins/google-genai/src/googleai/index.ts +++ b/js/plugins/google-genai/src/googleai/index.ts @@ -164,7 +164,7 @@ export const googleAI = googleAIPlugin as GoogleAIPlugin; return imagen.model(name, config); } // gemma, tts, gemini and unknown model families. - return gemini.model(name, config); + return gemini.googleaiModelRef(name, config); }; googleAI.embedder = ( name: string, diff --git a/js/plugins/google-genai/src/vertexai/gemini.ts b/js/plugins/google-genai/src/vertexai/gemini.ts index 65ef0ca2d3..99605d4865 100644 --- a/js/plugins/google-genai/src/vertexai/gemini.ts +++ b/js/plugins/google-genai/src/vertexai/gemini.ts @@ -16,6 +16,7 @@ import { ActionMetadata, GenkitError, modelActionMetadata, z } from 'genkit'; import { + CandidateData, GenerationCommonConfigDescriptions, GenerationCommonConfigSchema, ModelAction, @@ -252,6 +253,12 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({ .object({ mode: z.enum(['MODE_UNSPECIFIED', 'AUTO', 'ANY', 'NONE']).optional(), allowedFunctionNames: z.array(z.string()).optional(), + /** + * When set to true, arguments of a single function call will be streamed out in + * multiple parts/contents/responses. Partial parameter results will be returned in the + * [FunctionCall.partial_args] field. This field is not supported in Gemini API. + */ + streamFunctionCallArguments: z.boolean().optional(), }) .describe( 'Controls how the model uses the provided tools (function declarations). ' + @@ -399,7 +406,7 @@ export function isGeminiModelName(value?: string): value is GeminiModelName { return !!value?.startsWith('gemini-') && !value.includes('embedding'); } -export function model( +export function vertexModelRef( version: string, options: GeminiConfig = {} ): ModelReference { @@ -430,7 +437,7 @@ export function listActions(models: Model[]): ActionMetadata[] { !KNOWN_DECOMISSIONED_MODELS.includes(modelName(m.name) || '') ) .map((m) => { - const ref = model(m.name); + const ref = vertexModelRef(m.name); return modelActionMetadata({ name: ref.name, info: ref.info, @@ -456,7 +463,7 @@ export function defineModel( clientOptions: ClientOptions, pluginOptions?: VertexPluginOptions ): ModelAction { - const ref = model(name); + const ref = vertexModelRef(name); const middlewares: ModelMiddleware[] = []; if (ref.info?.supports?.media) { // the gemini api doesn't support downloading media from http(s) @@ -536,6 +543,7 @@ export function defineModel( if (functionCallingConfig) { toolConfig = { functionCallingConfig: { + ...functionCallingConfig, allowedFunctionNames: functionCallingConfig.allowedFunctionNames, mode: toGeminiFunctionModeEnum(functionCallingConfig.mode), }, @@ -646,10 +654,12 @@ export function defineModel( clientOpt ); + const chunks: CandidateData[] = []; for await (const item of result.stream) { (item as GenerateContentResponse).candidates?.forEach( (candidate) => { - const c = fromGeminiCandidate(candidate); + const c = fromGeminiCandidate(candidate, chunks); + chunks.push(c); sendChunk({ index: c.index, content: c.message.content, diff --git a/js/plugins/google-genai/src/vertexai/index.ts b/js/plugins/google-genai/src/vertexai/index.ts index 6b061f12ad..14fcd46250 100644 --- a/js/plugins/google-genai/src/vertexai/index.ts +++ b/js/plugins/google-genai/src/vertexai/index.ts @@ -167,7 +167,7 @@ export const vertexAI = vertexAIPlugin as VertexAIPlugin; return veo.model(name, config); } // gemini and unknown model families - return gemini.model(name, config); + return gemini.vertexModelRef(name, config); }; vertexAI.embedder = ( name: string, diff --git a/js/plugins/google-genai/tests/common/converters_test.ts b/js/plugins/google-genai/tests/common/converters_test.ts index 1da39d49ab..a69a0fc0e3 100644 --- a/js/plugins/google-genai/tests/common/converters_test.ts +++ b/js/plugins/google-genai/tests/common/converters_test.ts @@ -13,17 +13,22 @@ limitations under the License. */ import * as assert from 'assert'; import { z } from 'genkit'; -import type { MessageData } from 'genkit/model'; +import type { CandidateData, MessageData } from 'genkit/model'; import { toJsonSchema } from 'genkit/schema'; import { describe, it } from 'node:test'; import { + applyStreamingJsonPath, fromGeminiCandidate, toGeminiFunctionModeEnum, toGeminiMessage, toGeminiSystemInstruction, toGeminiTool, } from '../../src/common/converters.js'; -import type { GenerateContentCandidate } from '../../src/common/types.js'; +import type { + FunctionCall, + GenerateContentCandidate, + Part, +} from '../../src/common/types.js'; import { ExecutableCodeLanguage, FunctionCallingMode, @@ -87,6 +92,7 @@ describe('toGeminiMessage', () => { parts: [ { functionResponse: { + id: '0', name: 'tellAFunnyJoke', response: { name: 'tellAFunnyJoke', @@ -96,6 +102,7 @@ describe('toGeminiMessage', () => { }, { functionResponse: { + id: '1', name: 'tellAFunnyJoke', response: { name: 'tellAFunnyJoke', @@ -515,14 +522,12 @@ describe('fromGeminiCandidate', () => { toolRequest: { name: 'tellAFunnyJoke', input: { topic: 'dog' }, - ref: '0', }, }, { toolRequest: { name: 'my__tool__name', // Expected no conversion for functionCall input: { param: 'value' }, - ref: '1', }, }, ], @@ -842,6 +847,129 @@ describe('fromGeminiCandidate', () => { assert.deepStrictEqual(result, test.expectedOutput); }); } + + describe('fromGeminiFunctionCall partial tool requests', () => { + it('should handle streaming function calls', () => { + const chunks: CandidateData[] = []; + // First chunk, defines the function call + let result = fromGeminiCandidate( + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + name: 'getWeather', + id: '1234', + args: {}, + willContinue: true, + }, + thoughtSignature: 'thoughtSignature1234', + }, + ], + }, + }, + chunks + ); + chunks.push(result); + + assert.deepStrictEqual(result.message.content[0].toolRequest, { + name: 'getWeather', + ref: '1234', + input: {}, + partial: true, + }); + + // Second chunk, adds a partial argument + result = fromGeminiCandidate( + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + partialArgs: [ + { + jsonPath: '$.location', + stringValue: 'Paris, France', + }, + ], + willContinue: true, + }, + }, + ], + }, + }, + chunks + ); + chunks.push(result); + + assert.deepStrictEqual(result.message.content[0].toolRequest, { + name: 'getWeather', + ref: '1234', + input: { location: 'Paris, France' }, + partial: true, + }); + + // Third chunk, adds another partial argument + result = fromGeminiCandidate( + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + name: '', + partialArgs: [ + { + jsonPath: '$.unit', + stringValue: 'celsius', + }, + ], + willContinue: true, + }, + }, + ], + }, + }, + chunks + ); + chunks.push(result); + + assert.deepStrictEqual(result.message.content[0].toolRequest, { + name: 'getWeather', + ref: '1234', + input: { location: 'Paris, France', unit: 'celsius' }, + partial: true, + }); + + // Final chunk, finishes the call + result = fromGeminiCandidate( + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: {}, + }, + ], + }, + }, + chunks + ); + chunks.push(result); + + assert.deepStrictEqual(result.message.content[0].toolRequest, { + name: 'getWeather', + ref: '1234', + input: { location: 'Paris, France', unit: 'celsius' }, + }); + }); + }); }); describe('toGeminiTool', () => { @@ -959,3 +1087,128 @@ describe('toGeminiFunctionModeEnum', () => { ); }); }); + +describe('applyStreamingJsonPath', () => { + const testCases = [ + { + should: 'apply a simple string value', + initialArgs: {}, + partialArgs: [{ jsonPath: '$.foo', stringValue: 'bar' }], + expectedArgs: { foo: 'bar' }, + }, + { + should: 'apply a simple number value', + initialArgs: {}, + partialArgs: [{ jsonPath: '$.count', numberValue: 42 }], + expectedArgs: { count: 42 }, + }, + { + should: 'apply a simple boolean value', + initialArgs: {}, + partialArgs: [{ jsonPath: '$.enabled', boolValue: true }], + expectedArgs: { enabled: true }, + }, + { + should: 'apply a null value', + initialArgs: { key: 'not-null' }, + partialArgs: [{ jsonPath: '$.key', nullValue: 'NULL_VALUE' as const }], + expectedArgs: { key: null }, + }, + { + should: 'apply a value to a nested object', + initialArgs: {}, + partialArgs: [{ jsonPath: '$.config.setting', stringValue: 'value' }], + expectedArgs: { config: { setting: 'value' } }, + }, + { + should: 'apply a value to an array index', + initialArgs: { items: ['a', 'b'] }, + partialArgs: [{ jsonPath: '$.items[1]', stringValue: 'c' }], + expectedArgs: { items: ['a', 'bc'] }, + }, + { + should: 'create and apply a value to an array index', + initialArgs: { items: [] }, + partialArgs: [{ jsonPath: '$.items[0]', stringValue: 'a' }], + expectedArgs: { items: ['a'] }, + }, + { + should: 'apply a value to a nested object within an array', + initialArgs: { data: [{ id: 1, value: 'old' }] }, + partialArgs: [{ jsonPath: '$.data[0].value', stringValue: 'new' }], + expectedArgs: { data: [{ id: 1, value: 'oldnew' }] }, + }, + { + should: 'apply multiple partial args', + initialArgs: {}, + partialArgs: [ + { jsonPath: '$.name', stringValue: 'test' }, + { jsonPath: '$.config.version', numberValue: 2 }, + ], + expectedArgs: { name: 'test', config: { version: 2 } }, + }, + { + should: 'overwrite an existing value', + initialArgs: { property: 'initial' }, + partialArgs: [{ jsonPath: '$.property', stringValue: 'updated' }], + expectedArgs: { property: 'initialupdated' }, + }, + { + should: 'create deeply nested objects and arrays', + initialArgs: {}, + partialArgs: [{ jsonPath: '$.a.b[0].c', stringValue: 'deep' }], + expectedArgs: { a: { b: [{ c: 'deep' }] } }, + }, + { + should: 'create a nested array', + initialArgs: {}, + partialArgs: [{ jsonPath: '$.data[0][0]', stringValue: 'nested' }], + expectedArgs: { data: [['nested']] }, + }, + { + should: 'apply a value to a multi-dimensional array', + initialArgs: { + matrix: [ + [1, 2], + [3, 4], + ], + }, + partialArgs: [{ jsonPath: '$.matrix[1][0]', numberValue: 5 }], + expectedArgs: { + matrix: [ + [1, 2], + [5, 4], + ], + }, + }, + { + should: 'add a new property to an existing object', + initialArgs: { user: { name: 'John' } }, + partialArgs: [{ jsonPath: '$.user.age', numberValue: 30 }], + expectedArgs: { user: { name: 'John', age: 30 } }, + }, + { + should: 'add a new element to an existing array', + initialArgs: { scores: [10, 20] }, + partialArgs: [{ jsonPath: '$.scores[2]', numberValue: 30 }], + expectedArgs: { scores: [10, 20, 30] }, + }, + ]; + for (const test of testCases) { + it(`should ${test.should}`, () => { + const functionCall: FunctionCall = { + name: 'test', + args: test.initialArgs, + }; + const part: Part = { + functionCall: { + name: 'test', + args: {}, + partialArgs: test.partialArgs, + }, + }; + applyStreamingJsonPath(functionCall.args!, part); + assert.deepStrictEqual(functionCall.args, test.expectedArgs); + }); + } +}); diff --git a/js/plugins/google-genai/tests/common/utils_test.ts b/js/plugins/google-genai/tests/common/utils_test.ts index aa66161c25..060f52a12d 100644 --- a/js/plugins/google-genai/tests/common/utils_test.ts +++ b/js/plugins/google-genai/tests/common/utils_test.ts @@ -18,7 +18,9 @@ import * as assert from 'assert'; import { GenkitError, embedderRef, modelRef } from 'genkit'; import { GenerateRequest } from 'genkit/model'; import { describe, it } from 'node:test'; +import { GenerateContentResponse } from '../../src/common/types.js'; import { + aggregateResponses, checkModelName, checkSupportedMimeType, cleanSchema, @@ -508,4 +510,126 @@ describe('Common Utils', () => { assert.deepStrictEqual(cleaned, schema); }); }); + + describe('aggregateResponses', () => { + it('should aggregate streaming function call parts', () => { + const responses: GenerateContentResponse[] = [ + { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + name: 'findFlights', + id: '1234', + willContinue: true, + }, + thoughtSignature: 'thoughtSignature1234', + }, + ], + }, + }, + ], + }, + { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + willContinue: true, + partialArgs: [ + { + jsonPath: '$.flights[0].departure_airport', + stringValue: 'SFO', + }, + ], + }, + }, + ], + }, + }, + ], + }, + { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + willContinue: true, + partialArgs: [ + { + jsonPath: '$.flights[0].arrival_airport', + stringValue: 'JFK', + }, + ], + }, + }, + ], + }, + }, + ], + }, + { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + name: 'findFlights', + }, + }, + ], + }, + }, + ], + }, + ]; + + const aggregated = aggregateResponses(responses); + + const expected = { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [ + { + functionCall: { + name: 'findFlights', + id: '1234', + args: { + flights: [ + { + departure_airport: 'SFO', + arrival_airport: 'JFK', + }, + ], + }, + }, + thoughtSignature: 'thoughtSignature1234', + }, + ], + }, + }, + ], + }; + + assert.deepStrictEqual(aggregated, expected); + }); + }); }); diff --git a/js/plugins/google-genai/tests/googleai/gemini_test.ts b/js/plugins/google-genai/tests/googleai/gemini_test.ts index a70d24ab32..100b322c7c 100644 --- a/js/plugins/google-genai/tests/googleai/gemini_test.ts +++ b/js/plugins/google-genai/tests/googleai/gemini_test.ts @@ -23,7 +23,7 @@ import { GeminiConfigSchema, GeminiTtsConfigSchema, defineModel, - model, + googleaiModelRef, } from '../../src/googleai/gemini.js'; import { FinishReason, @@ -392,7 +392,7 @@ describe('Google AI Gemini', () => { describe('gemini() function', () => { it('returns a ModelReference for a known model string', () => { const name = 'gemini-2.0-flash'; - const modelRef = model(name); + const modelRef = googleaiModelRef(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -400,7 +400,7 @@ describe('Google AI Gemini', () => { it('returns a ModelReference for a tts type model string', () => { const name = 'gemini-2.5-flash-preview-tts'; - const modelRef = model(name); + const modelRef = googleaiModelRef(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, false); assert.strictEqual(modelRef.configSchema, GeminiTtsConfigSchema); @@ -408,7 +408,7 @@ describe('Google AI Gemini', () => { it('returns a ModelReference for an image type model string', () => { const name = 'gemini-2.5-flash-image'; - const modelRef = model(name); + const modelRef = googleaiModelRef(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -416,7 +416,7 @@ describe('Google AI Gemini', () => { it('returns a ModelReference for an unknown model string', () => { const name = 'gemini-42.0-flash'; - const modelRef = model(name); + const modelRef = googleaiModelRef(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); diff --git a/js/plugins/google-genai/tests/vertexai/gemini_test.ts b/js/plugins/google-genai/tests/vertexai/gemini_test.ts index affcdfd19e..55bcb7165f 100644 --- a/js/plugins/google-genai/tests/vertexai/gemini_test.ts +++ b/js/plugins/google-genai/tests/vertexai/gemini_test.ts @@ -26,7 +26,7 @@ import { getVertexAIUrl } from '../../src/vertexai/client.js'; import { GeminiConfigSchema, defineModel, - model, + vertexModelRef, } from '../../src/vertexai/gemini.js'; import { ClientOptions, @@ -125,7 +125,8 @@ describe('Vertex AI Gemini', () => { describe('gemini() function', () => { it('returns a ModelReference for a known model string', () => { const name = 'gemini-2.0-flash'; - const modelRef: ModelReference = model(name); + const modelRef: ModelReference = + vertexModelRef(name); assert.strictEqual(modelRef.name, `vertexai/${name}`); assert.ok(modelRef.info?.supports?.multiturn); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -133,7 +134,8 @@ describe('Vertex AI Gemini', () => { it('returns a ModelReference for an unknown model string', () => { const name = 'gemini-new-model'; - const modelRef: ModelReference = model(name); + const modelRef: ModelReference = + vertexModelRef(name); assert.strictEqual(modelRef.name, `vertexai/${name}`); assert.ok(modelRef.info?.supports?.multiturn); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -141,10 +143,8 @@ describe('Vertex AI Gemini', () => { it('applies options to the ModelReference', () => { const options = { temperature: 0.9, topK: 20 }; - const modelRef: ModelReference = model( - 'gemini-2.0-flash', - options - ); + const modelRef: ModelReference = + vertexModelRef('gemini-2.0-flash', options); assert.deepStrictEqual(modelRef.config, options); }); }); diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 00f3027cfd..233670dbe9 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -181,7 +181,7 @@ importers: optionalDependencies: '@genkit-ai/firebase': specifier: ^1.16.1 - version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) doc-snippets: dependencies: @@ -670,7 +670,13 @@ importers: google-auth-library: specifier: ^9.14.2 version: 9.15.1(encoding@0.1.13) + jsonpath-plus: + specifier: ^10.3.0 + version: 10.3.0 devDependencies: + '@types/jsonpath-plus': + specifier: ^5.0.5 + version: 5.0.5 '@types/node': specifier: ^20.11.16 version: 20.19.1 @@ -999,9 +1005,6 @@ importers: '@genkit-ai/google-genai': specifier: workspace:* version: link:../../plugins/google-genai - '@google/genai': - specifier: ^1.29.0 - version: 1.29.0 express: specifier: ^4.20.0 version: 4.21.2 @@ -1029,7 +1032,7 @@ importers: version: link:../../plugins/compat-oai '@genkit-ai/express': specifier: ^1.1.0 - version: 1.12.0(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit) + version: 1.12.0(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit) genkit: specifier: workspace:* version: link:../../genkit @@ -1596,7 +1599,7 @@ importers: version: link:../../plugins/ollama genkitx-openai: specifier: ^0.10.1 - version: 0.10.1(@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3) + version: 0.10.1(@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3) devDependencies: rimraf: specifier: ^6.0.1 @@ -2625,11 +2628,11 @@ packages: '@firebase/webchannel-wrapper@1.0.3': resolution: {integrity: sha512-2xCRM9q9FlzGZCdgDMJwc0gyUkWFtkosy7Xxr6sFgQwn+wMNIWd7xIvYNauU1r64B5L5rsGKy/n9TKJ0aAFeqQ==} - '@genkit-ai/ai@1.22.0': - resolution: {integrity: sha512-TDKO+zWyM5YI8zE4a0IlqlpgHuLB4B4islzgWDvzdQlbjtyJp0ayODAMFhS2ruQ6+a/UdXDySRrOX/RcqF4yjA==} + '@genkit-ai/ai@1.24.0': + resolution: {integrity: sha512-Rv2eZqvJA8awIfLKiZL+P1hlBPGFiBFk1r01hRk0BSp1HmpZmlzSx+MM+X2H54xMgRXBRAelzU6xUXXzN5U57Q==} - '@genkit-ai/core@1.22.0': - resolution: {integrity: sha512-etVlpwJkPoy91xR6H5+S/AWZPJMeovb7N35+B90md1+6xWcodQF7WZ3chKcH31Xamlz+jTIvd3riiZGY9RFumg==} + '@genkit-ai/core@1.24.0': + resolution: {integrity: sha512-JGmwdcC066OpbwShXeOOwvinj9b4yA0BKfKjPrVqqbWt9hvu81I60UNNiZC5y8dx+TvEhdEPUAXRvoOup2vC0w==} '@genkit-ai/express@1.12.0': resolution: {integrity: sha512-QAxSS07dX5ovSfsUB4s90KaDnv4zg1wnoxCZCa+jBsYUyv9NvCCTsOk25xAQgGxc7xi3+MD+3AsPier5oZILIg==} @@ -2743,15 +2746,6 @@ packages: resolution: {integrity: sha512-HqYqoivNtkq59po8m7KI0n+lWKdz4kabENncYQXZCX/hBWJfXtKAfR/2nUQsP+TwSfHKoA7zDL2RrJYIv/j3VQ==} engines: {node: '>=18.0.0'} - '@google/genai@1.29.0': - resolution: {integrity: sha512-cQP7Ssa06W+MSAyVtL/812FBtZDoDehnFObIpK1xo5Uv4XvqBcVZ8OhXgihOIXWn7xvPQGvLclR8+yt3Ysnd9g==} - engines: {node: '>=20.0.0'} - peerDependencies: - '@modelcontextprotocol/sdk': ^1.20.1 - peerDependenciesMeta: - '@modelcontextprotocol/sdk': - optional: true - '@google/generative-ai@0.15.0': resolution: {integrity: sha512-zs37judcTYFJf1U7tnuqnh7gdzF6dcWj9pNRxjA5JTONRoiQ0htrRdbefRFiewOIfXwhun5t9hbd2ray7812eQ==} engines: {node: '>=18.0.0'} @@ -3011,6 +3005,18 @@ packages: '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@jsep-plugin/assignment@1.3.0': + resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==} + engines: {node: '>= 10.16.0'} + peerDependencies: + jsep: ^0.4.0||^1.0.0 + + '@jsep-plugin/regex@1.0.4': + resolution: {integrity: sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==} + engines: {node: '>= 10.16.0'} + peerDependencies: + jsep: ^0.4.0||^1.0.0 + '@langchain/community@0.0.53': resolution: {integrity: sha512-iFqZPt4MRssGYsQoKSXWJQaYTZCC7WNuilp2JCCs3wKmJK3l6mR0eV+PDrnT+TaDHUVxt/b0rwgM0sOiy0j2jA==} engines: {node: '>=18'} @@ -4159,6 +4165,9 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/jsonpath-plus@5.0.5': + resolution: {integrity: sha512-aaqqDf5LcGOtAfEntO5qKZS6ixT0MpNhUXNwbVPdLf7ET9hKsufJq+buZr7eXSnWoLRyGzVj2Yz2hbjVSK3wsA==} + '@types/jsonwebtoken@9.0.10': resolution: {integrity: sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==} @@ -5259,10 +5268,6 @@ packages: resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} engines: {node: '>=14'} - gaxios@7.1.3: - resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} - engines: {node: '>=18'} - gcp-metadata@5.3.0: resolution: {integrity: sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==} engines: {node: '>=12'} @@ -5271,12 +5276,8 @@ packages: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} engines: {node: '>=14'} - gcp-metadata@8.1.2: - resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} - engines: {node: '>=18'} - - genkit@1.22.0: - resolution: {integrity: sha512-GoVVO3EnNHrjkMkUPRvgx1MjBHKvOlZAu/ffMIJgLFxrH7rrUbvfHXE6Nk7uh5BNvET7+DApyhbhqz9G8sy+mQ==} + genkit@1.24.0: + resolution: {integrity: sha512-9BjPrLULfWdzauibKkbZcCf2LVu0mW2EW6EXHK/cTber6QdiP/guA5mXahaaDWBo+qdIKpInXAYDEzcRUmCVSA==} genkitx-openai@0.10.1: resolution: {integrity: sha512-E9/DzyQcBUSTy81xT2pvEmdnn9Q/cKoojEt6lD/EdOeinhqE9oa59d/kuXTokCMekTrj3Rk7LtNBQIDjnyjNOA==} @@ -5353,10 +5354,6 @@ packages: resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} engines: {node: '>= 0.4'} - google-auth-library@10.5.0: - resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} - engines: {node: '>=18'} - google-auth-library@8.9.0: resolution: {integrity: sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==} engines: {node: '>=12'} @@ -5369,10 +5366,6 @@ packages: resolution: {integrity: sha512-V6eky/xz2mcKfAd1Ioxyd6nmA61gao3n01C+YeuIwu3vzM9EDR6wcVzMSIbLMDXWeoi9SHYctXuKYC5uJUT3eQ==} engines: {node: '>=14'} - google-logging-utils@1.1.2: - resolution: {integrity: sha512-YsFPGVgDFf4IzSwbwIR0iaFJQFmR5Jp7V1WuYSjuRgAm9yWqsMhKE9YPlL+wvFLnc/wMiFV4SQUD9Y/JMpxIxQ==} - engines: {node: '>=14'} - google-p12-pem@4.0.1: resolution: {integrity: sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==} engines: {node: '>=12.0.0'} @@ -5406,10 +5399,6 @@ packages: resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} engines: {node: '>=14.0.0'} - gtoken@8.0.0: - resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} - engines: {node: '>=18'} - handlebars@4.7.8: resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} engines: {node: '>=0.4.7'} @@ -5885,6 +5874,10 @@ packages: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true + jsep@1.4.0: + resolution: {integrity: sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==} + engines: {node: '>= 10.16.0'} + jsesc@3.0.2: resolution: {integrity: sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==} engines: {node: '>=6'} @@ -5917,6 +5910,11 @@ packages: resolution: {integrity: sha512-WhQB5tXQ32qjkx2GYHFw2XbL90u+LLzjofAYwi+86g6SyZeXHz9F1Q0amy3dWRYczshOC3Haok9J4pOCgHtwyQ==} engines: {node: '>= 8'} + jsonpath-plus@10.3.0: + resolution: {integrity: sha512-8TNmfeTCk2Le33A3vRRwtuworG/L5RrgMvdjhKZxvyShO+mBu2fP50OWUjRLNtvw344DdDarFh9buFAZs5ujeA==} + engines: {node: '>=18.0.0'} + hasBin: true + jsonpointer@5.0.1: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} @@ -6988,10 +6986,6 @@ packages: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} - rimraf@5.0.10: - resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} - hasBin: true - rimraf@6.0.1: resolution: {integrity: sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==} engines: {node: 20 || >=22} @@ -8508,9 +8502,9 @@ snapshots: '@firebase/webchannel-wrapper@1.0.3': {} - '@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) '@opentelemetry/api': 1.9.0 '@types/node': 20.19.1 colorette: 2.0.20 @@ -8529,9 +8523,9 @@ snapshots: - supports-color optional: true - '@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': + '@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': dependencies: - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) '@opentelemetry/api': 1.9.0 '@types/node': 20.19.1 colorette: 2.0.20 @@ -8549,7 +8543,7 @@ snapshots: - genkit - supports-color - '@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) @@ -8571,7 +8565,7 @@ snapshots: zod: 3.25.67 zod-to-json-schema: 3.24.5(zod@3.25.67) optionalDependencies: - '@genkit-ai/firebase': 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/firebase': 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) transitivePeerDependencies: - '@google-cloud/firestore' - encoding @@ -8581,7 +8575,7 @@ snapshots: - supports-color optional: true - '@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': + '@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) @@ -8612,9 +8606,9 @@ snapshots: - genkit - supports-color - '@genkit-ai/express@1.12.0(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit)': + '@genkit-ai/express@1.12.0(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit)': dependencies: - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) body-parser: 1.20.3 cors: 2.8.5 express: 5.1.0 @@ -8622,12 +8616,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@genkit-ai/firebase@1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/firebase@1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: - '@genkit-ai/google-cloud': 1.16.1(encoding@0.1.13)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/google-cloud': 1.16.1(encoding@0.1.13)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) '@google-cloud/firestore': 7.11.1(encoding@0.1.13) firebase-admin: 13.4.0(encoding@0.1.13) - genkit: 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) + genkit: 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) optionalDependencies: firebase: 11.9.1 transitivePeerDependencies: @@ -8648,7 +8642,7 @@ snapshots: - supports-color optional: true - '@genkit-ai/google-cloud@1.16.1(encoding@0.1.13)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/google-cloud@1.16.1(encoding@0.1.13)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: '@google-cloud/logging-winston': 6.0.1(encoding@0.1.13)(winston@3.17.0) '@google-cloud/opentelemetry-cloud-monitoring-exporter': 0.19.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) @@ -8664,7 +8658,7 @@ snapshots: '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': 0.52.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) - genkit: 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) + genkit: 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) google-auth-library: 9.15.1(encoding@0.1.13) node-fetch: 3.3.2 winston: 3.17.0 @@ -8891,15 +8885,6 @@ snapshots: - encoding - supports-color - '@google/genai@1.29.0': - dependencies: - google-auth-library: 10.5.0 - ws: 8.18.3 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate - '@google/generative-ai@0.15.0': {} '@google/generative-ai@0.21.0': {} @@ -9268,6 +9253,14 @@ snapshots: '@js-sdsl/ordered-map@4.4.2': {} + '@jsep-plugin/assignment@1.3.0(jsep@1.4.0)': + dependencies: + jsep: 1.4.0 + + '@jsep-plugin/regex@1.0.4(jsep@1.4.0)': + dependencies: + jsep: 1.4.0 + '@langchain/community@0.0.53(@pinecone-database/pinecone@2.2.2)(chromadb@1.9.2(encoding@0.1.13)(openai@4.104.0(encoding@0.1.13)(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(firebase-admin@12.3.1(encoding@0.1.13))(google-auth-library@8.9.0(encoding@0.1.13))(jsonwebtoken@9.0.2)(lodash@4.17.21)(pg@8.16.2)(ws@8.18.3)': dependencies: '@langchain/core': 0.1.63 @@ -10339,6 +10332,8 @@ snapshots: '@types/json-schema@7.0.15': {} + '@types/jsonpath-plus@5.0.5': {} + '@types/jsonwebtoken@9.0.10': dependencies: '@types/ms': 2.1.0 @@ -11679,15 +11674,6 @@ snapshots: - encoding - supports-color - gaxios@7.1.3: - dependencies: - extend: 3.0.2 - https-proxy-agent: 7.0.6 - node-fetch: 3.3.2 - rimraf: 5.0.10 - transitivePeerDependencies: - - supports-color - gcp-metadata@5.3.0(encoding@0.1.13): dependencies: gaxios: 5.1.3(encoding@0.1.13) @@ -11705,18 +11691,10 @@ snapshots: - encoding - supports-color - gcp-metadata@8.1.2: + genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): dependencies: - gaxios: 7.1.3 - google-logging-utils: 1.1.2 - json-bigint: 1.0.0 - transitivePeerDependencies: - - supports-color - - genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): - dependencies: - '@genkit-ai/ai': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/ai': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) uuid: 10.0.0 transitivePeerDependencies: - '@google-cloud/firestore' @@ -11726,10 +11704,10 @@ snapshots: - supports-color optional: true - genkitx-openai@0.10.1(@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3): + genkitx-openai@0.10.1(@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3): dependencies: - '@genkit-ai/ai': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/ai': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) openai: 4.104.0(encoding@0.1.13)(ws@8.18.3)(zod@3.25.67) zod: 3.25.67 transitivePeerDependencies: @@ -11823,18 +11801,6 @@ snapshots: define-properties: 1.2.1 gopd: 1.2.0 - google-auth-library@10.5.0: - dependencies: - base64-js: 1.5.1 - ecdsa-sig-formatter: 1.0.11 - gaxios: 7.1.3 - gcp-metadata: 8.1.2 - google-logging-utils: 1.1.2 - gtoken: 8.0.0 - jws: 4.0.0 - transitivePeerDependencies: - - supports-color - google-auth-library@8.9.0(encoding@0.1.13): dependencies: arrify: 2.0.1 @@ -11881,8 +11847,6 @@ snapshots: - encoding - supports-color - google-logging-utils@1.1.2: {} - google-p12-pem@4.0.1: dependencies: node-forge: 1.3.1 @@ -11938,13 +11902,6 @@ snapshots: - encoding - supports-color - gtoken@8.0.0: - dependencies: - gaxios: 7.1.3 - jws: 4.0.0 - transitivePeerDependencies: - - supports-color - handlebars@4.7.8: dependencies: minimist: 1.2.8 @@ -12688,6 +12645,8 @@ snapshots: dependencies: argparse: 2.0.1 + jsep@1.4.0: {} + jsesc@3.0.2: {} json-bigint@1.0.0: @@ -12708,6 +12667,12 @@ snapshots: jsonata@2.0.6: {} + jsonpath-plus@10.3.0: + dependencies: + '@jsep-plugin/assignment': 1.3.0(jsep@1.4.0) + '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) + jsep: 1.4.0 + jsonpointer@5.0.1: {} jsonwebtoken@9.0.2: @@ -13724,10 +13689,6 @@ snapshots: retry@0.13.1: {} - rimraf@5.0.10: - dependencies: - glob: 10.3.12 - rimraf@6.0.1: dependencies: glob: 11.0.0 @@ -14659,7 +14620,8 @@ snapshots: imurmurhash: 0.1.4 signal-exit: 3.0.7 - ws@8.18.3: {} + ws@8.18.3: + optional: true xorshift@1.2.0: {} diff --git a/js/testapps/basic-gemini/package.json b/js/testapps/basic-gemini/package.json index afaeffc52f..7842e9a8fb 100644 --- a/js/testapps/basic-gemini/package.json +++ b/js/testapps/basic-gemini/package.json @@ -19,7 +19,6 @@ "@genkit-ai/firebase": "workspace:*", "@genkit-ai/google-cloud": "workspace:*", "@genkit-ai/google-genai": "workspace:*", - "@google/genai": "^1.29.0", "express": "^4.20.0", "node-fetch": "3.3.2", "wav": "^1.0.2" diff --git a/js/testapps/basic-gemini/src/index-vertexai.ts b/js/testapps/basic-gemini/src/index-vertexai.ts index b182ec3a90..e9c29ef44e 100644 --- a/js/testapps/basic-gemini/src/index-vertexai.ts +++ b/js/testapps/basic-gemini/src/index-vertexai.ts @@ -18,6 +18,7 @@ import { vertexAI } from '@genkit-ai/google-genai'; import * as fs from 'fs'; import { genkit, Operation, Part, StreamingCallback, z } from 'genkit'; import wav from 'wav'; +import { RpgCharacterSchema } from './types'; const ai = genkit({ plugins: [ @@ -270,6 +271,34 @@ ai.defineFlow( } ); +ai.defineFlow( + { + name: 'streamingToolCalling', + inputSchema: z.string().default('Paris, France'), + outputSchema: z.string(), + streamSchema: z.any(), + }, + async (location, { sendChunk }) => { + const { response, stream } = ai.generateStream({ + model: vertexAI.model('gemini-3-pro-preview'), + config: { + temperature: 1, + functionCallingConfig: { + streamFunctionCallArguments: true, + }, + }, + tools: [getWeather, celsiusToFahrenheit], + prompt: `What's the weather in ${location}? Convert the temperature to Fahrenheit.`, + }); + + for await (const chunk of stream) { + sendChunk(chunk); + } + + return (await response).text; + } +); + // Tool calling with structured output ai.defineFlow( { @@ -307,37 +336,6 @@ ai.defineFlow( } ); -const baseCategorySchema = z.object({ - name: z.string(), -}); - -type Category = z.infer & { - subcategories?: Category[]; -}; - -const categorySchema: z.ZodType = baseCategorySchema.extend({ - subcategories: z.lazy(() => - categorySchema - .array() - .describe('make sure there are at least 2-3 levels of subcategories') - .optional() - ), -}); - -const WeaponSchema = z.object({ - name: z.string(), - damage: z.number(), - category: categorySchema, -}); - -const RpgCharacterSchema = z.object({ - name: z.string().describe('name of the character'), - backstory: z.string().describe("character's backstory, about a paragraph"), - weapons: z.array(WeaponSchema), - class: z.enum(['RANGER', 'WIZZARD', 'TANK', 'HEALER', 'ENGINEER']), - affiliation: z.string().optional(), -}); - // A simple example of structured output. ai.defineFlow( { @@ -352,7 +350,7 @@ ai.defineFlow( temperature: 2, // we want creativity }, output: { schema: RpgCharacterSchema }, - prompt: `Generate an RPC character called ${name}`, + prompt: `Generate sample data`, }); for await (const chunk of stream) { diff --git a/js/testapps/basic-gemini/src/index.ts b/js/testapps/basic-gemini/src/index.ts index 81aabaa6bf..c613356796 100644 --- a/js/testapps/basic-gemini/src/index.ts +++ b/js/testapps/basic-gemini/src/index.ts @@ -32,6 +32,7 @@ import { deleteFileSearchStore, uploadBlobToFileSearchStore, } from './helper.js'; +import { RpgCharacterSchema } from './types.js'; const ai = genkit({ plugins: [ @@ -304,37 +305,6 @@ ai.defineFlow( } ); -const baseCategorySchema = z.object({ - name: z.string(), -}); - -type Category = z.infer & { - subcategories?: Category[]; -}; - -const categorySchema: z.ZodType = baseCategorySchema.extend({ - subcategories: z.lazy(() => - categorySchema - .array() - .describe('make sure there are at least 2-3 levels of subcategories') - .optional() - ), -}); - -const WeaponSchema = z.object({ - name: z.string(), - damage: z.number(), - category: categorySchema, -}); - -const RpgCharacterSchema = z.object({ - name: z.string().describe('name of the character'), - backstory: z.string().describe("character's backstory, about a paragraph"), - weapons: z.array(WeaponSchema), - class: z.enum(['RANGER', 'WIZZARD', 'TANK', 'HEALER', 'ENGINEER']), - affiliation: z.string().optional(), -}); - // A simple example of structured output. ai.defineFlow( { diff --git a/js/testapps/basic-gemini/src/types.ts b/js/testapps/basic-gemini/src/types.ts new file mode 100644 index 0000000000..81d9c2d9e3 --- /dev/null +++ b/js/testapps/basic-gemini/src/types.ts @@ -0,0 +1,48 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'genkit'; + +const baseCategorySchema = z.object({ + name: z.string(), +}); + +type Category = z.infer & { + subcategories?: Category[]; +}; + +const categorySchema: z.ZodType = baseCategorySchema.extend({ + subcategories: z.lazy(() => + categorySchema + .array() + .describe('make sure there are at least 2-3 levels of subcategories') + .optional() + ), +}); + +export const WeaponSchema = z.object({ + name: z.string(), + damage: z.number(), + category: categorySchema, +}); + +export const RpgCharacterSchema = z.object({ + name: z.string().describe('name of the character'), + backstory: z.string().describe("character's backstory, about a paragraph"), + weapons: z.array(WeaponSchema), + class: z.enum(['RANGER', 'WIZZARD', 'TANK', 'HEALER', 'ENGINEER']), + affiliation: z.string().optional(), +}); From 9779fd3818b9272ab97d4b6a428b2221d2ca91d7 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Mon, 24 Nov 2025 15:44:13 -0500 Subject: [PATCH 13/21] removed payload strategy --- js/ai/src/generate/resolve-tool-requests.ts | 4 +- js/ai/src/tool.ts | 2 - js/ai/tests/generate/generate_test.ts | 4 +- js/ai/tests/tool_test.ts | 2 - js/genkit/tests/generate_test.ts | 1 - js/pnpm-lock.yaml | 90 ++++++++++++++++++++- js/testapps/basic-gemini/package.json | 1 + 7 files changed, 91 insertions(+), 13 deletions(-) diff --git a/js/ai/src/generate/resolve-tool-requests.ts b/js/ai/src/generate/resolve-tool-requests.ts index 0fe23a7b1f..22a6c216ed 100644 --- a/js/ai/src/generate/resolve-tool-requests.ts +++ b/js/ai/src/generate/resolve-tool-requests.ts @@ -126,14 +126,12 @@ export async function resolveToolRequest( const multipartResponse = output as z.infer< typeof MultipartToolResponseSchema >; - const strategy = multipartResponse.fallbackOutput ? 'fallback' : 'both'; const response = stripUndefinedProps({ toolResponse: { name: part.toolRequest.name, ref: part.toolRequest.ref, - output: multipartResponse.output || multipartResponse.fallbackOutput, + output: multipartResponse.output, content: multipartResponse.content, - payloadStrategy: strategy, } as ToolResponse, }); diff --git a/js/ai/src/tool.ts b/js/ai/src/tool.ts index f3c79573fa..066f44b2d9 100644 --- a/js/ai/src/tool.ts +++ b/js/ai/src/tool.ts @@ -300,7 +300,6 @@ export type MultipartToolFn = ( ctx: ToolFnOptions & ToolRunOptions ) => Promise<{ output?: z.infer; - fallbackOutput?: z.infer; content?: Part[]; }>; @@ -527,7 +526,6 @@ function basicToolV2( export const MultipartToolResponseSchema = z.object({ output: z.any().optional(), - fallbackOutput: z.any().optional(), content: z.array(PartSchema).optional(), }); diff --git a/js/ai/tests/generate/generate_test.ts b/js/ai/tests/generate/generate_test.ts index 089edf170e..052acee642 100644 --- a/js/ai/tests/generate/generate_test.ts +++ b/js/ai/tests/generate/generate_test.ts @@ -681,7 +681,6 @@ describe('generate', () => { text: 'part 1', }, ], - payloadStrategy: 'both', }, }, ], @@ -707,7 +706,7 @@ describe('generate', () => { }, async () => { return { - fallbackOutput: 'fallback output', + output: 'fallback output', content: [{ text: 'part 1' }], }; } @@ -775,7 +774,6 @@ describe('generate', () => { text: 'part 1', }, ], - payloadStrategy: 'fallback', }, }, ], diff --git a/js/ai/tests/tool_test.ts b/js/ai/tests/tool_test.ts index 2bb71dccca..b934374f11 100644 --- a/js/ai/tests/tool_test.ts +++ b/js/ai/tests/tool_test.ts @@ -135,14 +135,12 @@ describe('defineInterrupt', () => { { name: 'test', description: 'test', multipart: true }, async () => { return { - fallbackOutput: 'fallback', content: [{ text: 'part 1' }], }; } ); const result = await t({}); assert.deepStrictEqual(result, { - fallbackOutput: 'fallback', content: [{ text: 'part 1' }], }); }); diff --git a/js/genkit/tests/generate_test.ts b/js/genkit/tests/generate_test.ts index 8114117f46..ecdcb783a8 100644 --- a/js/genkit/tests/generate_test.ts +++ b/js/genkit/tests/generate_test.ts @@ -860,7 +860,6 @@ describe('generate', () => { ref: 'ref123', output: 'tool called', content: [{ text: 'part 1' }], - payloadStrategy: 'both', }, }, ]); diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 233670dbe9..871e47f24a 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -1005,6 +1005,9 @@ importers: '@genkit-ai/google-genai': specifier: workspace:* version: link:../../plugins/google-genai + '@google/genai': + specifier: 1.30.0 + version: 1.30.0 express: specifier: ^4.20.0 version: 4.21.2 @@ -2746,6 +2749,15 @@ packages: resolution: {integrity: sha512-HqYqoivNtkq59po8m7KI0n+lWKdz4kabENncYQXZCX/hBWJfXtKAfR/2nUQsP+TwSfHKoA7zDL2RrJYIv/j3VQ==} engines: {node: '>=18.0.0'} + '@google/genai@1.30.0': + resolution: {integrity: sha512-3MRcgczBFbUat1wIlZoLJ0vCCfXgm7Qxjh59cZi2X08RgWLtm9hKOspzp7TOg1TV2e26/MLxR2GR5yD5GmBV2w==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.20.1 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + '@google/generative-ai@0.15.0': resolution: {integrity: sha512-zs37judcTYFJf1U7tnuqnh7gdzF6dcWj9pNRxjA5JTONRoiQ0htrRdbefRFiewOIfXwhun5t9hbd2ray7812eQ==} engines: {node: '>=18.0.0'} @@ -5268,6 +5280,10 @@ packages: resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} engines: {node: '>=14'} + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} + gcp-metadata@5.3.0: resolution: {integrity: sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==} engines: {node: '>=12'} @@ -5276,6 +5292,10 @@ packages: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} engines: {node: '>=14'} + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + genkit@1.24.0: resolution: {integrity: sha512-9BjPrLULfWdzauibKkbZcCf2LVu0mW2EW6EXHK/cTber6QdiP/guA5mXahaaDWBo+qdIKpInXAYDEzcRUmCVSA==} @@ -5354,6 +5374,10 @@ packages: resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} engines: {node: '>= 0.4'} + google-auth-library@10.5.0: + resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} + engines: {node: '>=18'} + google-auth-library@8.9.0: resolution: {integrity: sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==} engines: {node: '>=12'} @@ -5366,6 +5390,10 @@ packages: resolution: {integrity: sha512-V6eky/xz2mcKfAd1Ioxyd6nmA61gao3n01C+YeuIwu3vzM9EDR6wcVzMSIbLMDXWeoi9SHYctXuKYC5uJUT3eQ==} engines: {node: '>=14'} + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + google-p12-pem@4.0.1: resolution: {integrity: sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==} engines: {node: '>=12.0.0'} @@ -5399,6 +5427,10 @@ packages: resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} engines: {node: '>=14.0.0'} + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} + handlebars@4.7.8: resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} engines: {node: '>=0.4.7'} @@ -6986,6 +7018,10 @@ packages: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + rimraf@6.0.1: resolution: {integrity: sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==} engines: {node: 20 || >=22} @@ -8885,6 +8921,15 @@ snapshots: - encoding - supports-color + '@google/genai@1.30.0': + dependencies: + google-auth-library: 10.5.0 + ws: 8.18.3 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + '@google/generative-ai@0.15.0': {} '@google/generative-ai@0.21.0': {} @@ -11674,6 +11719,15 @@ snapshots: - encoding - supports-color + gaxios@7.1.3: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + rimraf: 5.0.10 + transitivePeerDependencies: + - supports-color + gcp-metadata@5.3.0(encoding@0.1.13): dependencies: gaxios: 5.1.3(encoding@0.1.13) @@ -11691,6 +11745,14 @@ snapshots: - encoding - supports-color + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.1.3 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): dependencies: '@genkit-ai/ai': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) @@ -11801,6 +11863,18 @@ snapshots: define-properties: 1.2.1 gopd: 1.2.0 + google-auth-library@10.5.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.1.3 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + gtoken: 8.0.0 + jws: 4.0.0 + transitivePeerDependencies: + - supports-color + google-auth-library@8.9.0(encoding@0.1.13): dependencies: arrify: 2.0.1 @@ -11847,6 +11921,8 @@ snapshots: - encoding - supports-color + google-logging-utils@1.1.3: {} + google-p12-pem@4.0.1: dependencies: node-forge: 1.3.1 @@ -11902,6 +11978,13 @@ snapshots: - encoding - supports-color + gtoken@8.0.0: + dependencies: + gaxios: 7.1.3 + jws: 4.0.0 + transitivePeerDependencies: + - supports-color + handlebars@4.7.8: dependencies: minimist: 1.2.8 @@ -13689,6 +13772,10 @@ snapshots: retry@0.13.1: {} + rimraf@5.0.10: + dependencies: + glob: 10.3.12 + rimraf@6.0.1: dependencies: glob: 11.0.0 @@ -14620,8 +14707,7 @@ snapshots: imurmurhash: 0.1.4 signal-exit: 3.0.7 - ws@8.18.3: - optional: true + ws@8.18.3: {} xorshift@1.2.0: {} diff --git a/js/testapps/basic-gemini/package.json b/js/testapps/basic-gemini/package.json index 7842e9a8fb..f04f7ec934 100644 --- a/js/testapps/basic-gemini/package.json +++ b/js/testapps/basic-gemini/package.json @@ -16,6 +16,7 @@ "license": "ISC", "dependencies": { "genkit": "workspace:*", + "@google/genai": "1.30.0", "@genkit-ai/firebase": "workspace:*", "@genkit-ai/google-cloud": "workspace:*", "@genkit-ai/google-genai": "workspace:*", From 6b5b6ea7d3064ee6151dbb027d65117b4109cc0f Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Mon, 24 Nov 2025 17:22:44 -0500 Subject: [PATCH 14/21] fix tests --- go/ai/gen.go | 4 +- go/core/schemas.config | 3 +- js/pnpm-lock.yaml | 68 +++++++++++++-------------- js/testapps/basic-gemini/package.json | 2 +- 4 files changed, 39 insertions(+), 38 deletions(-) diff --git a/go/ai/gen.go b/go/ai/gen.go index c59a62eee4..4ff147356b 100644 --- a/go/ai/gen.go +++ b/go/ai/gen.go @@ -391,8 +391,8 @@ type toolRequestPart struct { // the results of running a specific tool on the arguments passed to the client // by the model in a [ToolRequest]. type ToolResponse struct { - Content []*Part `json:"content,omitempty"` - Name string `json:"name,omitempty"` + Content []any `json:"content,omitempty"` + Name string `json:"name,omitempty"` // Output is a JSON object describing the results of running the tool. // An example might be map[string]any{"name":"Thomas Jefferson", "born":1743}. Output any `json:"output,omitempty"` diff --git a/go/core/schemas.config b/go/core/schemas.config index 2caf39e9f7..ec2f22a7f4 100644 --- a/go/core/schemas.config +++ b/go/core/schemas.config @@ -67,7 +67,8 @@ the results of running a specific tool on the arguments passed to the client by the model in a [ToolRequest]. . -ToolResponse.content type []*Part +# TODO: make this work... otherwise we end up with `any` type for the field. +# ToolResponse.content type []*Part Candidate omit diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index 00f3027cfd..d6eeb42c7f 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -181,7 +181,7 @@ importers: optionalDependencies: '@genkit-ai/firebase': specifier: ^1.16.1 - version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + version: 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) doc-snippets: dependencies: @@ -1000,8 +1000,8 @@ importers: specifier: workspace:* version: link:../../plugins/google-genai '@google/genai': - specifier: ^1.29.0 - version: 1.29.0 + specifier: ^1.30.0 + version: 1.30.0 express: specifier: ^4.20.0 version: 4.21.2 @@ -1029,7 +1029,7 @@ importers: version: link:../../plugins/compat-oai '@genkit-ai/express': specifier: ^1.1.0 - version: 1.12.0(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit) + version: 1.12.0(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit) genkit: specifier: workspace:* version: link:../../genkit @@ -1596,7 +1596,7 @@ importers: version: link:../../plugins/ollama genkitx-openai: specifier: ^0.10.1 - version: 0.10.1(@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3) + version: 0.10.1(@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3) devDependencies: rimraf: specifier: ^6.0.1 @@ -2625,11 +2625,11 @@ packages: '@firebase/webchannel-wrapper@1.0.3': resolution: {integrity: sha512-2xCRM9q9FlzGZCdgDMJwc0gyUkWFtkosy7Xxr6sFgQwn+wMNIWd7xIvYNauU1r64B5L5rsGKy/n9TKJ0aAFeqQ==} - '@genkit-ai/ai@1.22.0': - resolution: {integrity: sha512-TDKO+zWyM5YI8zE4a0IlqlpgHuLB4B4islzgWDvzdQlbjtyJp0ayODAMFhS2ruQ6+a/UdXDySRrOX/RcqF4yjA==} + '@genkit-ai/ai@1.24.0': + resolution: {integrity: sha512-Rv2eZqvJA8awIfLKiZL+P1hlBPGFiBFk1r01hRk0BSp1HmpZmlzSx+MM+X2H54xMgRXBRAelzU6xUXXzN5U57Q==} - '@genkit-ai/core@1.22.0': - resolution: {integrity: sha512-etVlpwJkPoy91xR6H5+S/AWZPJMeovb7N35+B90md1+6xWcodQF7WZ3chKcH31Xamlz+jTIvd3riiZGY9RFumg==} + '@genkit-ai/core@1.24.0': + resolution: {integrity: sha512-JGmwdcC066OpbwShXeOOwvinj9b4yA0BKfKjPrVqqbWt9hvu81I60UNNiZC5y8dx+TvEhdEPUAXRvoOup2vC0w==} '@genkit-ai/express@1.12.0': resolution: {integrity: sha512-QAxSS07dX5ovSfsUB4s90KaDnv4zg1wnoxCZCa+jBsYUyv9NvCCTsOk25xAQgGxc7xi3+MD+3AsPier5oZILIg==} @@ -2743,8 +2743,8 @@ packages: resolution: {integrity: sha512-HqYqoivNtkq59po8m7KI0n+lWKdz4kabENncYQXZCX/hBWJfXtKAfR/2nUQsP+TwSfHKoA7zDL2RrJYIv/j3VQ==} engines: {node: '>=18.0.0'} - '@google/genai@1.29.0': - resolution: {integrity: sha512-cQP7Ssa06W+MSAyVtL/812FBtZDoDehnFObIpK1xo5Uv4XvqBcVZ8OhXgihOIXWn7xvPQGvLclR8+yt3Ysnd9g==} + '@google/genai@1.30.0': + resolution: {integrity: sha512-3MRcgczBFbUat1wIlZoLJ0vCCfXgm7Qxjh59cZi2X08RgWLtm9hKOspzp7TOg1TV2e26/MLxR2GR5yD5GmBV2w==} engines: {node: '>=20.0.0'} peerDependencies: '@modelcontextprotocol/sdk': ^1.20.1 @@ -5275,8 +5275,8 @@ packages: resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} engines: {node: '>=18'} - genkit@1.22.0: - resolution: {integrity: sha512-GoVVO3EnNHrjkMkUPRvgx1MjBHKvOlZAu/ffMIJgLFxrH7rrUbvfHXE6Nk7uh5BNvET7+DApyhbhqz9G8sy+mQ==} + genkit@1.24.0: + resolution: {integrity: sha512-9BjPrLULfWdzauibKkbZcCf2LVu0mW2EW6EXHK/cTber6QdiP/guA5mXahaaDWBo+qdIKpInXAYDEzcRUmCVSA==} genkitx-openai@0.10.1: resolution: {integrity: sha512-E9/DzyQcBUSTy81xT2pvEmdnn9Q/cKoojEt6lD/EdOeinhqE9oa59d/kuXTokCMekTrj3Rk7LtNBQIDjnyjNOA==} @@ -8508,9 +8508,9 @@ snapshots: '@firebase/webchannel-wrapper@1.0.3': {} - '@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) '@opentelemetry/api': 1.9.0 '@types/node': 20.19.1 colorette: 2.0.20 @@ -8529,9 +8529,9 @@ snapshots: - supports-color optional: true - '@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': + '@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': dependencies: - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) '@opentelemetry/api': 1.9.0 '@types/node': 20.19.1 colorette: 2.0.20 @@ -8549,7 +8549,7 @@ snapshots: - genkit - supports-color - '@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) @@ -8571,7 +8571,7 @@ snapshots: zod: 3.25.67 zod-to-json-schema: 3.24.5(zod@3.25.67) optionalDependencies: - '@genkit-ai/firebase': 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/firebase': 1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) transitivePeerDependencies: - '@google-cloud/firestore' - encoding @@ -8581,7 +8581,7 @@ snapshots: - supports-color optional: true - '@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': + '@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.25.1(@opentelemetry/api@1.9.0) @@ -8612,9 +8612,9 @@ snapshots: - genkit - supports-color - '@genkit-ai/express@1.12.0(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit)': + '@genkit-ai/express@1.12.0(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(express@5.1.0)(genkit@genkit)': dependencies: - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) body-parser: 1.20.3 cors: 2.8.5 express: 5.1.0 @@ -8622,12 +8622,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@genkit-ai/firebase@1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/firebase@1.16.1(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: - '@genkit-ai/google-cloud': 1.16.1(encoding@0.1.13)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/google-cloud': 1.16.1(encoding@0.1.13)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) '@google-cloud/firestore': 7.11.1(encoding@0.1.13) firebase-admin: 13.4.0(encoding@0.1.13) - genkit: 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) + genkit: 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) optionalDependencies: firebase: 11.9.1 transitivePeerDependencies: @@ -8648,7 +8648,7 @@ snapshots: - supports-color optional: true - '@genkit-ai/google-cloud@1.16.1(encoding@0.1.13)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': + '@genkit-ai/google-cloud@1.16.1(encoding@0.1.13)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1))': dependencies: '@google-cloud/logging-winston': 6.0.1(encoding@0.1.13)(winston@3.17.0) '@google-cloud/opentelemetry-cloud-monitoring-exporter': 0.19.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@1.25.1(@opentelemetry/api@1.9.0))(encoding@0.1.13) @@ -8664,7 +8664,7 @@ snapshots: '@opentelemetry/sdk-metrics': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-node': 0.52.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) - genkit: 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) + genkit: 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1) google-auth-library: 9.15.1(encoding@0.1.13) node-fetch: 3.3.2 winston: 3.17.0 @@ -8891,7 +8891,7 @@ snapshots: - encoding - supports-color - '@google/genai@1.29.0': + '@google/genai@1.30.0': dependencies: google-auth-library: 10.5.0 ws: 8.18.3 @@ -11713,10 +11713,10 @@ snapshots: transitivePeerDependencies: - supports-color - genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): + genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1): dependencies: - '@genkit-ai/ai': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/ai': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)) uuid: 10.0.0 transitivePeerDependencies: - '@google-cloud/firestore' @@ -11726,10 +11726,10 @@ snapshots: - supports-color optional: true - genkitx-openai@0.10.1(@genkit-ai/ai@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3): + genkitx-openai@0.10.1(@genkit-ai/ai@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(@genkit-ai/core@1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit))(encoding@0.1.13)(ws@8.18.3): dependencies: - '@genkit-ai/ai': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) - '@genkit-ai/core': 1.22.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/ai': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) + '@genkit-ai/core': 1.24.0(@google-cloud/firestore@7.11.1(encoding@0.1.13))(encoding@0.1.13)(firebase-admin@13.4.0(encoding@0.1.13))(firebase@11.9.1)(genkit@genkit) openai: 4.104.0(encoding@0.1.13)(ws@8.18.3)(zod@3.25.67) zod: 3.25.67 transitivePeerDependencies: diff --git a/js/testapps/basic-gemini/package.json b/js/testapps/basic-gemini/package.json index afaeffc52f..5956ffb13b 100644 --- a/js/testapps/basic-gemini/package.json +++ b/js/testapps/basic-gemini/package.json @@ -19,7 +19,7 @@ "@genkit-ai/firebase": "workspace:*", "@genkit-ai/google-cloud": "workspace:*", "@genkit-ai/google-genai": "workspace:*", - "@google/genai": "^1.29.0", + "@google/genai": "^1.30.0", "express": "^4.20.0", "node-fetch": "3.3.2", "wav": "^1.0.2" From 1797153b67eee969f4b27381754a6afec210841b Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 20:33:49 -0500 Subject: [PATCH 15/21] try to fix schema at the generator --- genkit-tools/common/src/types/parts.ts | 4 +--- genkit-tools/genkit-schema.json | 5 ++++- go/core/schemas.config | 2 +- js/ai/src/parts.ts | 4 +--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/genkit-tools/common/src/types/parts.ts b/genkit-tools/common/src/types/parts.ts index df93190bd1..222cdebad6 100644 --- a/genkit-tools/common/src/types/parts.ts +++ b/genkit-tools/common/src/types/parts.ts @@ -121,9 +121,7 @@ export type ToolResponsePart = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.any()).optional(), - // TODO: switch to this once we have effective recursive schema support across the board. - // content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.lazy(() => PartSchema)).optional(), }); /** diff --git a/genkit-tools/genkit-schema.json b/genkit-tools/genkit-schema.json index 74499d3468..5db0c25895 100644 --- a/genkit-tools/genkit-schema.json +++ b/genkit-tools/genkit-schema.json @@ -1267,7 +1267,10 @@ }, "output": {}, "content": { - "type": "array" + "type": "array", + "items": { + "$ref": "#/$defs/Part" + } } }, "required": [ diff --git a/go/core/schemas.config b/go/core/schemas.config index ec2f22a7f4..738b8dd30d 100644 --- a/go/core/schemas.config +++ b/go/core/schemas.config @@ -68,7 +68,7 @@ by the model in a [ToolRequest]. . # TODO: make this work... otherwise we end up with `any` type for the field. -# ToolResponse.content type []*Part +ToolResponse.content type []any Candidate omit diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts index 2cd1cf78c4..fc32ac4e41 100644 --- a/js/ai/src/parts.ts +++ b/js/ai/src/parts.ts @@ -120,9 +120,7 @@ export type ToolResponse = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.any()).optional(), - // TODO: switch to this once we have effective recursive schema support across the board. - // content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.lazy(() => PartSchema)).optional(), }); /** From 514aa9df198f3c95ee8fab5235671cfdc97dcae5 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 20:36:37 -0500 Subject: [PATCH 16/21] Revert "try to fix schema at the generator" This reverts commit 1797153b67eee969f4b27381754a6afec210841b. --- genkit-tools/common/src/types/parts.ts | 4 +++- genkit-tools/genkit-schema.json | 5 +---- go/core/schemas.config | 2 +- js/ai/src/parts.ts | 4 +++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/genkit-tools/common/src/types/parts.ts b/genkit-tools/common/src/types/parts.ts index 222cdebad6..df93190bd1 100644 --- a/genkit-tools/common/src/types/parts.ts +++ b/genkit-tools/common/src/types/parts.ts @@ -121,7 +121,9 @@ export type ToolResponsePart = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.any()).optional(), + // TODO: switch to this once we have effective recursive schema support across the board. + // content: z.array(z.lazy(() => PartSchema)).optional(), }); /** diff --git a/genkit-tools/genkit-schema.json b/genkit-tools/genkit-schema.json index 5db0c25895..74499d3468 100644 --- a/genkit-tools/genkit-schema.json +++ b/genkit-tools/genkit-schema.json @@ -1267,10 +1267,7 @@ }, "output": {}, "content": { - "type": "array", - "items": { - "$ref": "#/$defs/Part" - } + "type": "array" } }, "required": [ diff --git a/go/core/schemas.config b/go/core/schemas.config index 738b8dd30d..ec2f22a7f4 100644 --- a/go/core/schemas.config +++ b/go/core/schemas.config @@ -68,7 +68,7 @@ by the model in a [ToolRequest]. . # TODO: make this work... otherwise we end up with `any` type for the field. -ToolResponse.content type []any +# ToolResponse.content type []*Part Candidate omit diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts index fc32ac4e41..2cd1cf78c4 100644 --- a/js/ai/src/parts.ts +++ b/js/ai/src/parts.ts @@ -120,7 +120,9 @@ export type ToolResponse = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.any()).optional(), + // TODO: switch to this once we have effective recursive schema support across the board. + // content: z.array(z.lazy(() => PartSchema)).optional(), }); /** From 9309638143fb6f11f9cbb24d4ca6fb04750812b3 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 20:37:10 -0500 Subject: [PATCH 17/21] smaller change for js only --- js/ai/src/parts.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts index 2cd1cf78c4..fc32ac4e41 100644 --- a/js/ai/src/parts.ts +++ b/js/ai/src/parts.ts @@ -120,9 +120,7 @@ export type ToolResponse = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.any()).optional(), - // TODO: switch to this once we have effective recursive schema support across the board. - // content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.lazy(() => PartSchema)).optional(), }); /** From f8488596f11dbae7c01df3d44f24b4bfa02fa3e0 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 20:40:01 -0500 Subject: [PATCH 18/21] Revert "smaller change for js only" This reverts commit 9309638143fb6f11f9cbb24d4ca6fb04750812b3. --- js/ai/src/parts.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/ai/src/parts.ts b/js/ai/src/parts.ts index fc32ac4e41..2cd1cf78c4 100644 --- a/js/ai/src/parts.ts +++ b/js/ai/src/parts.ts @@ -120,7 +120,9 @@ export type ToolResponse = z.infer & { export const ToolResponseSchema: z.ZodType = ToolResponseSchemaBase.extend({ - content: z.array(z.lazy(() => PartSchema)).optional(), + content: z.array(z.any()).optional(), + // TODO: switch to this once we have effective recursive schema support across the board. + // content: z.array(z.lazy(() => PartSchema)).optional(), }); /** From c840b4f89134d87f12ec5176061b99e4529884ff Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 22:25:13 -0500 Subject: [PATCH 19/21] cleanup --- js/plugins/google-genai/src/googleai/gemini.ts | 6 +++--- js/plugins/google-genai/src/googleai/index.ts | 2 +- js/plugins/google-genai/src/vertexai/gemini.ts | 6 +++--- js/plugins/google-genai/src/vertexai/index.ts | 2 +- js/plugins/google-genai/tests/googleai/gemini_test.ts | 10 +++++----- js/plugins/google-genai/tests/vertexai/gemini_test.ts | 8 ++++---- js/testapps/basic-gemini/src/index-vertexai.ts | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/js/plugins/google-genai/src/googleai/gemini.ts b/js/plugins/google-genai/src/googleai/gemini.ts index 64d8ddfdc5..162d329b85 100644 --- a/js/plugins/google-genai/src/googleai/gemini.ts +++ b/js/plugins/google-genai/src/googleai/gemini.ts @@ -430,7 +430,7 @@ const KNOWN_MODELS = { ...KNOWN_GEMMA_MODELS, }; -export function googleaiModelRef( +export function model( version: string, config: GeminiConfig | GeminiTtsConfig | GemmaConfig = {} ): ModelReference { @@ -471,7 +471,7 @@ export function listActions(models: Model[]): ActionMetadata[] { // Filter out deprecated .filter((m) => !m.description || !m.description.includes('deprecated')) .map((m) => { - const ref = googleaiModelRef(m.name); + const ref = model(m.name); return modelActionMetadata({ name: ref.name, info: ref.info, @@ -495,7 +495,7 @@ export function defineModel( pluginOptions?: GoogleAIPluginOptions ): ModelAction { checkApiKey(pluginOptions?.apiKey); - const ref = googleaiModelRef(name); + const ref = model(name); const clientOptions: ClientOptions = { apiVersion: pluginOptions?.apiVersion, baseUrl: pluginOptions?.baseUrl, diff --git a/js/plugins/google-genai/src/googleai/index.ts b/js/plugins/google-genai/src/googleai/index.ts index 3af11f6105..220854ce04 100644 --- a/js/plugins/google-genai/src/googleai/index.ts +++ b/js/plugins/google-genai/src/googleai/index.ts @@ -164,7 +164,7 @@ export const googleAI = googleAIPlugin as GoogleAIPlugin; return imagen.model(name, config); } // gemma, tts, gemini and unknown model families. - return gemini.googleaiModelRef(name, config); + return gemini.model(name, config); }; googleAI.embedder = ( name: string, diff --git a/js/plugins/google-genai/src/vertexai/gemini.ts b/js/plugins/google-genai/src/vertexai/gemini.ts index 99605d4865..7c65310503 100644 --- a/js/plugins/google-genai/src/vertexai/gemini.ts +++ b/js/plugins/google-genai/src/vertexai/gemini.ts @@ -406,7 +406,7 @@ export function isGeminiModelName(value?: string): value is GeminiModelName { return !!value?.startsWith('gemini-') && !value.includes('embedding'); } -export function vertexModelRef( +export function model( version: string, options: GeminiConfig = {} ): ModelReference { @@ -437,7 +437,7 @@ export function listActions(models: Model[]): ActionMetadata[] { !KNOWN_DECOMISSIONED_MODELS.includes(modelName(m.name) || '') ) .map((m) => { - const ref = vertexModelRef(m.name); + const ref = model(m.name); return modelActionMetadata({ name: ref.name, info: ref.info, @@ -463,7 +463,7 @@ export function defineModel( clientOptions: ClientOptions, pluginOptions?: VertexPluginOptions ): ModelAction { - const ref = vertexModelRef(name); + const ref = model(name); const middlewares: ModelMiddleware[] = []; if (ref.info?.supports?.media) { // the gemini api doesn't support downloading media from http(s) diff --git a/js/plugins/google-genai/src/vertexai/index.ts b/js/plugins/google-genai/src/vertexai/index.ts index 14fcd46250..6b061f12ad 100644 --- a/js/plugins/google-genai/src/vertexai/index.ts +++ b/js/plugins/google-genai/src/vertexai/index.ts @@ -167,7 +167,7 @@ export const vertexAI = vertexAIPlugin as VertexAIPlugin; return veo.model(name, config); } // gemini and unknown model families - return gemini.vertexModelRef(name, config); + return gemini.model(name, config); }; vertexAI.embedder = ( name: string, diff --git a/js/plugins/google-genai/tests/googleai/gemini_test.ts b/js/plugins/google-genai/tests/googleai/gemini_test.ts index f70c8073ce..31de1081a9 100644 --- a/js/plugins/google-genai/tests/googleai/gemini_test.ts +++ b/js/plugins/google-genai/tests/googleai/gemini_test.ts @@ -23,7 +23,7 @@ import { GeminiConfigSchema, GeminiTtsConfigSchema, defineModel, - googleaiModelRef, + model, } from '../../src/googleai/gemini.js'; import { FinishReason, @@ -396,7 +396,7 @@ describe('Google AI Gemini', () => { describe('gemini() function', () => { it('returns a ModelReference for a known model string', () => { const name = 'gemini-2.0-flash'; - const modelRef = googleaiModelRef(name); + const modelRef = model(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -404,7 +404,7 @@ describe('Google AI Gemini', () => { it('returns a ModelReference for a tts type model string', () => { const name = 'gemini-2.5-flash-preview-tts'; - const modelRef = googleaiModelRef(name); + const modelRef = model(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, false); assert.strictEqual(modelRef.configSchema, GeminiTtsConfigSchema); @@ -412,7 +412,7 @@ describe('Google AI Gemini', () => { it('returns a ModelReference for an image type model string', () => { const name = 'gemini-2.5-flash-image'; - const modelRef = googleaiModelRef(name); + const modelRef = model(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -420,7 +420,7 @@ describe('Google AI Gemini', () => { it('returns a ModelReference for an unknown model string', () => { const name = 'gemini-42.0-flash'; - const modelRef = googleaiModelRef(name); + const modelRef = model(name); assert.strictEqual(modelRef.name, `googleai/${name}`); assert.strictEqual(modelRef.info?.supports?.multiturn, true); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); diff --git a/js/plugins/google-genai/tests/vertexai/gemini_test.ts b/js/plugins/google-genai/tests/vertexai/gemini_test.ts index 55bcb7165f..b2ee643ee4 100644 --- a/js/plugins/google-genai/tests/vertexai/gemini_test.ts +++ b/js/plugins/google-genai/tests/vertexai/gemini_test.ts @@ -26,7 +26,7 @@ import { getVertexAIUrl } from '../../src/vertexai/client.js'; import { GeminiConfigSchema, defineModel, - vertexModelRef, + model, } from '../../src/vertexai/gemini.js'; import { ClientOptions, @@ -126,7 +126,7 @@ describe('Vertex AI Gemini', () => { it('returns a ModelReference for a known model string', () => { const name = 'gemini-2.0-flash'; const modelRef: ModelReference = - vertexModelRef(name); + model(name); assert.strictEqual(modelRef.name, `vertexai/${name}`); assert.ok(modelRef.info?.supports?.multiturn); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -135,7 +135,7 @@ describe('Vertex AI Gemini', () => { it('returns a ModelReference for an unknown model string', () => { const name = 'gemini-new-model'; const modelRef: ModelReference = - vertexModelRef(name); + model(name); assert.strictEqual(modelRef.name, `vertexai/${name}`); assert.ok(modelRef.info?.supports?.multiturn); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -144,7 +144,7 @@ describe('Vertex AI Gemini', () => { it('applies options to the ModelReference', () => { const options = { temperature: 0.9, topK: 20 }; const modelRef: ModelReference = - vertexModelRef('gemini-2.0-flash', options); + model('gemini-2.0-flash', options); assert.deepStrictEqual(modelRef.config, options); }); }); diff --git a/js/testapps/basic-gemini/src/index-vertexai.ts b/js/testapps/basic-gemini/src/index-vertexai.ts index e9c29ef44e..a3be81297f 100644 --- a/js/testapps/basic-gemini/src/index-vertexai.ts +++ b/js/testapps/basic-gemini/src/index-vertexai.ts @@ -350,7 +350,7 @@ ai.defineFlow( temperature: 2, // we want creativity }, output: { schema: RpgCharacterSchema }, - prompt: `Generate sample data`, + prompt: `Generate an RPC character called ${name}`, }); for await (const chunk of stream) { From a64e940a392a9d73a75db14afe92214e2f299d26 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 22:25:44 -0500 Subject: [PATCH 20/21] fmt --- .../google-genai/tests/vertexai/gemini_test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/js/plugins/google-genai/tests/vertexai/gemini_test.ts b/js/plugins/google-genai/tests/vertexai/gemini_test.ts index b2ee643ee4..affcdfd19e 100644 --- a/js/plugins/google-genai/tests/vertexai/gemini_test.ts +++ b/js/plugins/google-genai/tests/vertexai/gemini_test.ts @@ -125,8 +125,7 @@ describe('Vertex AI Gemini', () => { describe('gemini() function', () => { it('returns a ModelReference for a known model string', () => { const name = 'gemini-2.0-flash'; - const modelRef: ModelReference = - model(name); + const modelRef: ModelReference = model(name); assert.strictEqual(modelRef.name, `vertexai/${name}`); assert.ok(modelRef.info?.supports?.multiturn); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -134,8 +133,7 @@ describe('Vertex AI Gemini', () => { it('returns a ModelReference for an unknown model string', () => { const name = 'gemini-new-model'; - const modelRef: ModelReference = - model(name); + const modelRef: ModelReference = model(name); assert.strictEqual(modelRef.name, `vertexai/${name}`); assert.ok(modelRef.info?.supports?.multiturn); assert.strictEqual(modelRef.configSchema, GeminiConfigSchema); @@ -143,8 +141,10 @@ describe('Vertex AI Gemini', () => { it('applies options to the ModelReference', () => { const options = { temperature: 0.9, topK: 20 }; - const modelRef: ModelReference = - model('gemini-2.0-flash', options); + const modelRef: ModelReference = model( + 'gemini-2.0-flash', + options + ); assert.deepStrictEqual(modelRef.config, options); }); }); From f632c059d04c53308766af4091acfc43f32fae31 Mon Sep 17 00:00:00 2001 From: Pavel Jbanov Date: Tue, 25 Nov 2025 22:47:13 -0500 Subject: [PATCH 21/21] implemented in google-genai --- .../google-genai/src/common/converters.ts | 3 ++ js/plugins/google-genai/src/common/types.ts | 35 ++++++++++++++ .../tests/common/converters_test.ts | 47 +++++++++++++++++++ js/testapps/basic-gemini/src/index.ts | 41 ++++++++++++++++ 4 files changed, 126 insertions(+) diff --git a/js/plugins/google-genai/src/common/converters.ts b/js/plugins/google-genai/src/common/converters.ts index fee41107f5..e61ce6807a 100644 --- a/js/plugins/google-genai/src/common/converters.ts +++ b/js/plugins/google-genai/src/common/converters.ts @@ -162,6 +162,9 @@ function toGeminiToolResponse(part: Part): GeminiPart { content: part.toolResponse.output, }, }; + if (part.toolResponse.content) { + functionResponse.parts = part.toolResponse.content.map(toGeminiPart); + } if (part.toolResponse.ref) { functionResponse.id = part.toolResponse.ref; } diff --git a/js/plugins/google-genai/src/common/types.ts b/js/plugins/google-genai/src/common/types.ts index 9f72ccf5fd..c69b0c4264 100644 --- a/js/plugins/google-genai/src/common/types.ts +++ b/js/plugins/google-genai/src/common/types.ts @@ -342,6 +342,41 @@ export declare interface FunctionResponse { name: string; /** The expected response from the model. */ response: object; + /** List of parts that constitute a function response. Each part may + have a different IANA MIME type. */ + parts?: FunctionResponsePart[]; +} + +/** + * A datatype containing media that is part of a `FunctionResponse` message. + * + * A `FunctionResponsePart` consists of data which has an associated datatype. A + * `FunctionResponsePart` can only contain one of the accepted types in + * `FunctionResponsePart.data`. + * + * A `FunctionResponsePart` must have a fixed IANA MIME type identifying the + * type and subtype of the media if the `inline_data` field is filled with raw + * bytes. + */ +export class FunctionResponsePart { + /** Optional. Inline media bytes. */ + inlineData?: FunctionResponseBlob; +} + +/** + * Raw media bytes for function response. + * + * Text should not be sent as raw bytes, use the FunctionResponse.response field. + */ +export class FunctionResponseBlob { + /** Required. The IANA standard MIME type of the source data. */ + mimeType?: string; + /** Required. Inline media bytes. + * @remarks Encoded as base64 string. */ + data?: string; + /** Optional. Display name of the blob. + Used to provide a label or filename to distinguish blobs. */ + displayName?: string; } /** diff --git a/js/plugins/google-genai/tests/common/converters_test.ts b/js/plugins/google-genai/tests/common/converters_test.ts index ddd228cb11..e1053aec8b 100644 --- a/js/plugins/google-genai/tests/common/converters_test.ts +++ b/js/plugins/google-genai/tests/common/converters_test.ts @@ -113,6 +113,53 @@ describe('toGeminiMessage', () => { ], }, }, + { + should: + 'should transform genkit message (tool response with media content) correctly', + inputMessage: { + role: 'tool', + content: [ + { + toolResponse: { + name: 'screenshot', + output: 'success', + ref: '0', + content: [ + { + media: { + contentType: 'image/png', + url: 'data:image/png;base64,SHORTENED_BASE64_DATA', + }, + }, + ], + }, + }, + ], + }, + expectedOutput: { + role: 'function', + parts: [ + { + functionResponse: { + id: '0', + name: 'screenshot', + response: { + name: 'screenshot', + content: 'success', + }, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'SHORTENED_BASE64_DATA', + }, + }, + ], + }, + }, + ], + }, + }, { should: 'should transform genkit message (inline base64 image content) correctly', diff --git a/js/testapps/basic-gemini/src/index.ts b/js/testapps/basic-gemini/src/index.ts index 611f6d8898..a21f850d78 100644 --- a/js/testapps/basic-gemini/src/index.ts +++ b/js/testapps/basic-gemini/src/index.ts @@ -248,6 +248,22 @@ const getWeather = ai.defineTool( } ); +const screenshot = ai.defineTool( + { + name: 'screenshot', + multipart: true, + description: 'takes a screenshot', + }, + async () => { + // pretend we call an actual API + const picture = fs.readFileSync('my_room.png', { encoding: 'base64' }); + return { + output: 'success', + content: [{ media: { url: `data:image/png;base64,${picture}` } }], + }; + } +); + const celsiusToFahrenheit = ai.defineTool( { name: 'celsiusToFahrenheit', @@ -287,6 +303,31 @@ ai.defineFlow( } ); +// Multipart tool calling +ai.defineFlow( + { + name: 'multipart-tool-calling', + outputSchema: z.string(), + streamSchema: z.any(), + }, + async (_, { sendChunk }) => { + const { response, stream } = ai.generateStream({ + model: googleAI.model('gemini-3-pro-preview'), + config: { + temperature: 1, + }, + tools: [screenshot], + prompt: `Tell me what I'm seeing on the screen.`, + }); + + for await (const chunk of stream) { + sendChunk(chunk.output); + } + + return (await response).text; + } +); + // Tool calling with structured output ai.defineFlow( {