Add DeepSeek V3.2 DSML format support with synthetic template

include/minja/chat-template.hpp

@@ -194,10 +194,9 @@ class chat_template {
         const json dummy_args_obj {{"argument_needle", "print('Hello, World!')"}};
         const auto contains_arg_needle = [&](const std::string & out_str) {
             return contains(out_str, "<parameter=argument_needle>")
-                || contains(out_str, "\"argument_needle\":")
+                || contains(out_str, "\"argument_needle\"")
                 || contains(out_str, "'argument_needle':")
-                || contains(out_str, ">argument_needle<")
-                || contains(out_str, "<parameter name=\"argument_needle\">");
+                || contains(out_str, ">argument_needle<");
         };
 
         // Note: the arguments are rendered in both cases, but may be double-escaped, which we don't want.

scripts/fetch_templates_and_goldens.py

@@ -194,10 +194,9 @@ def make_tool_call(tool_name, arguments):
         dummy_args_obj = {"argument_needle": "print('Hello, World!')"}
         contains_arg_needle = lambda out_str: (
             "<parameter=argument_needle>" in out_str
-            or '"argument_needle":' in out_str
+            or '"argument_needle"' in out_str
             or "'argument_needle':" in out_str
             or ">argument_needle<" in out_str
-            or "<parameter name=\"argument_needle\">" in out_str
         )
 
         out = self.try_raw_render([
@@ -432,6 +431,16 @@ async def async_hf_download(repo_id: str, filename: str) -> str:
 async def process_model(output_folder: str, model_id: str, contexts: list[Context]):
     try:
         print(f"Processing model {model_id}...", file=sys.stderr)
+
+        # Handle local .jinja files directly (for synthetic test templates)
+        if model_id.endswith('.jinja') and os.path.isfile(model_id):
+            async with aiofiles.open(model_id, 'r', encoding='utf-8') as f:
+                chat_template = await f.read()
+            # Use filename without extension as model_id for output naming
+            synthetic_id = os.path.basename(model_id).replace('.jinja', '')
+            await handle_chat_template(output_folder, synthetic_id, None, chat_template, contexts)
+            return
+
         config_str = await async_hf_download(model_id, "tokenizer_config.json")
 
         try:

tests/CMakeLists.txt

@@ -143,6 +143,7 @@ set(MODEL_IDS
     deepseek-ai/DeepSeek-V2-Lite
     deepseek-ai/DeepSeek-V2.5
     deepseek-ai/DeepSeek-V3
+    # deepseek-ai/DeepSeek-V3.2  # No Jinja template; see synthetic below
     deepseek-ai/deepseek-coder-7b-instruct-v1.5
     dicta-il/dictalm2.0-instruct
     ehristoforu/Falcon3-8B-Franken-Basestruct
@@ -195,6 +196,9 @@ set(MODEL_IDS
     xwen-team/Xwen-7B-Chat
     zai-org/GLM-4.6
 
+    # Synthetic templates for models without Jinja templates
+    ${CMAKE_CURRENT_SOURCE_DIR}/synthetic-deepseek-v3.2-dsml.jinja
+
     # Broken, TODO:
     # ai21labs/AI21-Jamba-1.5-Large        # https://github.com/google/minja/issues/8
     # Almawave/Velvet-14B

tests/synthetic-deepseek-v3.2-dsml.jinja

@@ -0,0 +1,42 @@
+{# Synthetic template based on DeepSeek V3.2 DSML format (encoding_dsv32.py) #}
+{# V3.2 doesn't provide a Jinja template, so this replicates its Python encoding logic #}
+{%- set bos_token = "<｜begin▁of▁sentence｜>" -%}
+{%- set eos_token = "<｜end▁of▁sentence｜>" -%}
+{%- set dsml_token = "｜DSML｜" -%}
+{{ bos_token }}
+{%- for message in messages -%}
+{%- if message.role == 'system' -%}
+{{ message.content }}
+{%- elif message.role == 'user' -%}
+<｜User｜>{{ message.content }}<｜Assistant｜>
+{%- elif message.role == 'assistant' -%}
+{%- if message.tool_calls is defined and message.tool_calls -%}
+<{{ dsml_token }}function_calls>
+{%- for tool_call in message.tool_calls -%}
+{%- if tool_call.type == 'function' -%}
+<{{ dsml_token }}invoke name="{{ tool_call.function.name }}">
+{%- if tool_call.function.arguments is mapping -%}
+{%- for key, value in tool_call.function.arguments.items() -%}
+{%- if value is string -%}
+<{{ dsml_token }}parameter name="{{ key }}" string="true">{{ value }}</{{ dsml_token }}parameter>
+{%- else -%}
+<{{ dsml_token }}parameter name="{{ key }}" string="false">{{ value | tojson }}</{{ dsml_token }}parameter>
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
+</{{ dsml_token }}invoke>
+{%- endif -%}
+{%- endfor -%}
+</{{ dsml_token }}function_calls>
+{%- endif -%}
+{%- if message.content -%}
+{{ message.content }}
+{%- endif -%}
+{{ eos_token }}
+{%- elif message.role == 'tool' -%}
+<{{ dsml_token }}tool_result>{{ message.content }}</{{ dsml_token }}tool_result>
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+<｜Assistant｜>
+{%- endif -%}

tests/test-capabilities.cpp

@@ -287,3 +287,20 @@ TEST(CapabilitiesTest, GLM46) {
     EXPECT_FALSE(caps.requires_non_null_content);
     EXPECT_FALSE(caps.requires_typed_content);
 }
+
+// Synthetic template based on DeepSeek V3.2's DSML format (encoding_dsv32.py)
+// V3.2 doesn't provide a Jinja template, so we replicate its Python encoding logic
+// DSML format: <｜DSML｜parameter name="argument_needle" string="true">
+TEST(CapabilitiesTest, SyntheticDeepSeekV3_2_DSML) {
+    auto caps = get_caps("tests/synthetic-deepseek-v3.2-dsml.jinja");
+    EXPECT_TRUE(caps.supports_system_role);
+    EXPECT_FALSE(caps.supports_tools);         // No native tools block in template
+    EXPECT_TRUE(caps.supports_tool_calls);     // Has tool_calls rendering with DSML format
+    EXPECT_FALSE(caps.supports_tool_call_id);
+    EXPECT_TRUE(caps.supports_tool_responses);
+    EXPECT_TRUE(caps.supports_parallel_tool_calls);  // Iterates over tool_calls array
+    EXPECT_TRUE(caps.requires_object_arguments);     // DSML iterates over argument keys
+    EXPECT_FALSE(caps.requires_non_null_content);
+    EXPECT_FALSE(caps.requires_typed_content);
+}
+