[JS] Flow isn't producing constrained output #1701
Description
Describe the bug
When we run our flows, the LLM is not responding with structured output matching the schema specified in the dotprompt file.
We're using 1.0.0-rc.9. This doesn't happen for 0.9.12.
To Reproduce
See the repo at https://github.com/gspencergoog/genkit_bugs/tree/main/unconstrained_output_1701 for a minimal repro case.
If you run the storyGeneratorFlow flow in the UI, you will see that the result is an error like the trace below:
Bad Flow Trace
{
"traceId": "a9d561190244ab76fa2090a3c01b9070",
"displayName": "storyGeneratorFlow",
"startTime": 1738206265313,
"endTime": 1738206269611.1777,
"spans": {
"566c08d85b61719a": {
"spanId": "566c08d85b61719a",
"traceId": "a9d561190244ab76fa2090a3c01b9070",
"parentSpanId": "358556ed10b22a2c",
"startTime": 1738206265331,
"endTime": 1738206269609.4993,
"attributes": {
"genkit:type": "util",
"genkit:name": "generate",
"genkit:path": "/{storyGeneratorFlow,t:flow}/{generate,t:util}",
"genkit:input": "{\"model\":\"googleai/gemini-2.0-flash-exp\",\"messages\":[{\"role\":\"system\",\"content\":[{\"text\":\"\\nWrite a two-paragraph story for each of the character names provided by the user.\\n\"}]},{\"role\":\"user\",\"content\":[{\"text\":\"\\nGreg\\nPaul\\n\"}]}],\"config\":{\"responseMimeType\":\"application/json\"},\"output\":{\"jsonSchema\":{\"type\":\"object\",\"properties\":{\"stories\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"The generated stories, one for each character name.\"}},\"required\":[\"stories\"],\"additionalProperties\":true,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}}",
"genkit:state": "error"
},
"displayName": "generate",
"links": [],
"instrumentationLibrary": {
"name": "genkit-tracer",
"version": "v1"
},
"spanKind": "INTERNAL",
"sameProcessAsParentSpan": {
"value": true
},
"status": {
"code": 2,
"message": "INVALID_ARGUMENT: Schema validation failed. Parse Errors:\n\n- (root): must be object\n\nProvided data:\n\nnull\n\nRequired JSON schema:\n\n{\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"description\": \"The generated stories, one for each character name.\"\n }\n },\n \"required\": [\n \"stories\"\n ],\n \"additionalProperties\": true,\n \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}"
},
"timeEvents": {
"timeEvent": [
{
"time": 1738206269609.4119,
"annotation": {
"attributes": {
"exception.type": "GenkitError",
"exception.message": "INVALID_ARGUMENT: Schema validation failed. Parse Errors:\n\n- (root): must be object\n\nProvided data:\n\nnull\n\nRequired JSON schema:\n\n{\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"description\": \"The generated stories, one for each character name.\"\n }\n },\n \"required\": [\n \"stories\"\n ],\n \"additionalProperties\": true,\n \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}",
"exception.stacktrace": "GenkitError: INVALID_ARGUMENT: Schema validation failed. Parse Errors:\n\n- (root): must be object\n\nProvided data:\n\nnull\n\nRequired JSON schema:\n\n{\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"description\": \"The generated stories, one for each character name.\"\n }\n },\n \"required\": [\n \"stories\"\n ],\n \"additionalProperties\": true,\n \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}\n at parseSchema (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/core/src/schema.ts:133:21)\n at GenerateResponse.assertValidSchema (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/response.ts:101:7)\n at generate (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/action.ts:296:45)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at <anonymous> (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/action.ts:129:22)\n at <anonymous> (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/core/src/tracing/instrumentation.ts:115:24)\n at runInNewSpan (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/core/src/tracing/instrumentation.ts:103:10)\n at generateHelper (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/action.ts:116:10)\n at <anonymous> (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate.ts:394:24)\n at generate (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate.ts:390:10)"
},
"description": "exception"
}
}
]
}
},
"358556ed10b22a2c": {
"spanId": "358556ed10b22a2c",
"traceId": "a9d561190244ab76fa2090a3c01b9070",
"startTime": 1738206265313,
"endTime": 1738206269611.1777,
"attributes": {
"genkit:type": "action",
"genkit:metadata:subtype": "flow",
"genkit:name": "storyGeneratorFlow",
"genkit:isRoot": true,
"genkit:path": "/{storyGeneratorFlow,t:flow}",
"genkit:metadata:context": "{}",
"genkit:input": "{\"names\":[\"Greg\",\"Paul\"]}",
"genkit:state": "error"
},
"displayName": "storyGeneratorFlow",
"links": [],
"instrumentationLibrary": {
"name": "genkit-tracer",
"version": "v1"
},
"spanKind": "INTERNAL",
"sameProcessAsParentSpan": {
"value": true
},
"status": {
"code": 2,
"message": "INVALID_ARGUMENT: Schema validation failed. Parse Errors:\n\n- (root): must be object\n\nProvided data:\n\nnull\n\nRequired JSON schema:\n\n{\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"description\": \"The generated stories, one for each character name.\"\n }\n },\n \"required\": [\n \"stories\"\n ],\n \"additionalProperties\": true,\n \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}"
},
"timeEvents": {
"timeEvent": [
{
"time": 1738206269610.9758,
"annotation": {
"attributes": {
"exception.type": "GenkitError",
"exception.message": "INVALID_ARGUMENT: Schema validation failed. Parse Errors:\n\n- (root): must be object\n\nProvided data:\n\nnull\n\nRequired JSON schema:\n\n{\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"description\": \"The generated stories, one for each character name.\"\n }\n },\n \"required\": [\n \"stories\"\n ],\n \"additionalProperties\": true,\n \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}",
"exception.stacktrace": "GenkitError: INVALID_ARGUMENT: Schema validation failed. Parse Errors:\n\n- (root): must be object\n\nProvided data:\n\nnull\n\nRequired JSON schema:\n\n{\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"description\": \"The generated stories, one for each character name.\"\n }\n },\n \"required\": [\n \"stories\"\n ],\n \"additionalProperties\": true,\n \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}\n at parseSchema (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/core/src/schema.ts:133:21)\n at GenerateResponse.assertValidSchema (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/response.ts:101:7)\n at generate (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/action.ts:296:45)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at <anonymous> (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/action.ts:129:22)\n at <anonymous> (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/core/src/tracing/instrumentation.ts:115:24)\n at runInNewSpan (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/core/src/tracing/instrumentation.ts:103:10)\n at generateHelper (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate/action.ts:116:10)\n at <anonymous> (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate.ts:394:24)\n at generate (/usr/local/google/home/gspencer/code/genkit_bugs/bad_llm_output/node_modules/@genkit-ai/ai/src/generate.ts:390:10)"
},
"description": "exception"
}
}
]
}
}
}
}The LLM was given the following input by GenKit:
Bad LLM Input
{
"messages": [
{
"role": "system",
"content": [
{
"text": "\nWrite a two-paragraph story for each of the character names provided by the user.\n"
}
]
},
{
"role": "user",
"content": [
{
"text": "\nGreg\nPaul\n"
}
]
}
],
"config": {
"responseMimeType": "application/json"
},
"tools": [],
"output": {
"schema": {
"type": "object",
"properties": {
"stories": {
"type": "array",
"items": {
"type": "string"
},
"description": "The generated stories, one for each character name."
}
},
"required": [
"stories"
],
"additionalProperties": true,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
}Which appears to be missing this stanza from the system instructions:
{
"text": "Output should be in JSON format and conform to the following schema:\n\n```\n{\"type\":\"object\",\"properties\":{\"stories\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"The generated stories, one for each character name.\"}},\"required\":[\"stories\"],\"additionalProperties\":true,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}\n```\n",
"metadata": {
"purpose": "output"
}
}And the output section is missing this:
"output": {
"format": "json",
"contentType": "application/json",
"constrained": true,
// ...
}So the LLM produced the following output that was trying to be validated:
Bad Output
{
"candidates": [
{
"index": 0,
"message": {
"role": "model",
"content": [
{
"text": "Okay, here are two two-paragraph stories for each of the names you provided:\n\n**Greg**\n\nGreg adjusted his glasses, the fluorescent lights of the library reflecting in their lenses. He loved the quiet hum of the place, the comforting scent of old paper and ink. Today, he was on a mission: to find the definitive text on the migratory patterns of the Lesser Spotted Bumblebee. He'd been obsessed ever since he'd seen one flitting through his garden last week, and the internet just hadn't been cutting it. He carefully ran a finger along the spines, a small thrill coursing through him as he pulled out a promising volume, its cover adorned with detailed illustrations.\n\nHe settled into a worn armchair by the window, the afternoon sun casting long shadows across the page. Greg lost himself in the intricacies of the bee's journey, the scientific language painting a vivid picture in his mind. He could almost see the tiny creatures, their wings a blur as they navigated vast landscapes. A contented sigh escaped him. This, he thought, was precisely the kind of adventure he lived for.\n\n**Paul**\n\nPaul stared at the half-finished sculpture, a lump of clay that stubbornly refused to take the shape he envisioned. Frustration gnawed at him, the rhythmic thud of his hands against the clay becoming increasingly forceful. He had imagined a sleek, powerful figure, but what emerged was lopsided and awkward. He knew he had the vision, the inspiration – he just lacked the skill, or maybe the patience, to bring it to life. He ran a hand through his hair, scattering clay dust into the air.\n\nHe stepped back, taking a deep breath, and decided to try a new approach. Instead of forcing the clay, he would let it guide him, explore the potential forms hidden within the lump. He started gently, his fingers coaxing the clay into curves and valleys, allowing his frustrations to morph into something more playful, less rigid. A smile crept onto his face as he began to see something new emerge, something he hadn't planned, yet felt strangely right. This, he realized, was the real beauty of creation – the unexpected detours and happy accidents.\n"
}
]
},
"finishReason": "stop",
"custom": {
"safetyRatings": [
{
"category": "HARM_CATEGORY_HATE_SPEECH",
"probability": "NEGLIGIBLE"
},
{
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
"probability": "NEGLIGIBLE"
},
{
"category": "HARM_CATEGORY_HARASSMENT",
"probability": "NEGLIGIBLE"
},
{
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
"probability": "NEGLIGIBLE"
}
]
}
}
],
"custom": {
"candidates": [
{
"content": {
"parts": [
{
"text": "Okay, here are two two-paragraph stories for each of the names you provided:\n\n**Greg**\n\nGreg adjusted his glasses, the fluorescent lights of the library reflecting in their lenses. He loved the quiet hum of the place, the comforting scent of old paper and ink. Today, he was on a mission: to find the definitive text on the migratory patterns of the Lesser Spotted Bumblebee. He'd been obsessed ever since he'd seen one flitting through his garden last week, and the internet just hadn't been cutting it. He carefully ran a finger along the spines, a small thrill coursing through him as he pulled out a promising volume, its cover adorned with detailed illustrations.\n\nHe settled into a worn armchair by the window, the afternoon sun casting long shadows across the page. Greg lost himself in the intricacies of the bee's journey, the scientific language painting a vivid picture in his mind. He could almost see the tiny creatures, their wings a blur as they navigated vast landscapes. A contented sigh escaped him. This, he thought, was precisely the kind of adventure he lived for.\n\n**Paul**\n\nPaul stared at the half-finished sculpture, a lump of clay that stubbornly refused to take the shape he envisioned. Frustration gnawed at him, the rhythmic thud of his hands against the clay becoming increasingly forceful. He had imagined a sleek, powerful figure, but what emerged was lopsided and awkward. He knew he had the vision, the inspiration – he just lacked the skill, or maybe the patience, to bring it to life. He ran a hand through his hair, scattering clay dust into the air.\n\nHe stepped back, taking a deep breath, and decided to try a new approach. Instead of forcing the clay, he would let it guide him, explore the potential forms hidden within the lump. He started gently, his fingers coaxing the clay into curves and valleys, allowing his frustrations to morph into something more playful, less rigid. A smile crept onto his face as he began to see something new emerge, something he hadn't planned, yet felt strangely right. This, he realized, was the real beauty of creation – the unexpected detours and happy accidents.\n"
}
],
"role": "model"
},
"finishReason": "STOP",
"safetyRatings": [
{
"category": "HARM_CATEGORY_HATE_SPEECH",
"probability": "NEGLIGIBLE"
},
{
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
"probability": "NEGLIGIBLE"
},
{
"category": "HARM_CATEGORY_HARASSMENT",
"probability": "NEGLIGIBLE"
},
{
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
"probability": "NEGLIGIBLE"
}
],
"avgLogprobs": -0.7351282631478658
}
],
"usageMetadata": {
"promptTokenCount": 26,
"candidatesTokenCount": 451,
"totalTokenCount": 477,
"promptTokensDetails": [
{
"modality": "TEXT",
"tokenCount": 24
}
],
"candidatesTokensDetails": [
{
"modality": "TEXT",
"tokenCount": 451
}
]
},
"modelVersion": "gemini-2.0-flash-exp"
},
"usage": {
"inputCharacters": 94,
"inputImages": 0,
"inputVideos": 0,
"inputAudioFiles": 0,
"outputCharacters": 2140,
"outputImages": 0,
"outputVideos": 0,
"outputAudioFiles": 0,
"inputTokens": 26,
"outputTokens": 451,
"totalTokens": 477
},
"latencyMs": 3825.3562129735947
}Which doesn't bear any resemblance to the output schema specified:
export const storyGeneratorOutputSchema = z.object({
stories: z
.array(z.string())
.describe("The generated stories, one for each character name."),
});Expected behavior
The LLM to produce constrained output matching the output schema.
Runtime (please complete the following information):
- OS: Linux (Debian)
- Version: 1.0.0-rc.9
** Node version
- v18.20.1