Skip to content

Commit c9b9862

Browse files
authored
Watsonx image support (#8324)
* watsonx - add support for image inputs * mistral models - autodetect image support
1 parent 427ac07 commit c9b9862

File tree

2 files changed

+27
-5
lines changed

2 files changed

+27
-5
lines changed

core/llm/autodetect.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ const MODEL_SUPPORTS_IMAGES: RegExp[] = [
105105
/\bgemma-?3(?!n)/, // gemma3 supports vision, but gemma3n doesn't!
106106
/\b(pali|med)gemma/,
107107
/qwen(.*)vl/,
108+
/mistral-small/,
109+
/mistral-medium/,
108110
];
109111

110112
function modelSupportsImages(

core/llm/llms/WatsonX.ts

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
Chunk,
66
CompletionOptions,
77
LLMOptions,
8+
TextMessagePart,
89
ToolCallDelta,
910
ToolResultChatMessage,
1011
} from "../../index.js";
@@ -103,11 +104,30 @@ class WatsonX extends BaseLLM {
103104
}));
104105
delete message_.toolCalls;
105106
delete message_.content;
106-
} else if (
107-
message_.role === "user" &&
108-
typeof message_.content === "string"
109-
) {
110-
message_.content = [{ type: "text", text: message_.content }];
107+
} else if (message_.role === "user") {
108+
if (typeof message.content === "string") {
109+
message_.content = [{ type: "text", text: message_.content }];
110+
} else {
111+
return {
112+
role: "user",
113+
content: !message.content.some((item) => item.type !== "text")
114+
? message.content
115+
.map((item) => (item as TextMessagePart).text)
116+
.join("") || " "
117+
: message.content.map((part) => {
118+
if (part.type === "imageUrl") {
119+
return {
120+
type: "image_url" as const,
121+
image_url: {
122+
url: part.imageUrl.url,
123+
detail: "auto" as const,
124+
},
125+
};
126+
}
127+
return part;
128+
}),
129+
};
130+
}
111131
}
112132
return message_;
113133
}

0 commit comments

Comments
 (0)