[WIP] tool-call: experimental migration of all parsers to peg-parser infra (w/ better test coverage)

common/CMakeLists.txt

@@ -44,18 +44,23 @@ endif()
 
 set(TARGET common)
 
+# Glob chat parser files from the chat-parsers directory
+file(GLOB CHAT_SYNTAX_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/chat-parsers/*.cpp")
+
 add_library(${TARGET} STATIC
     arg.cpp
     arg.h
     base64.hpp
     chat-parser.cpp
     chat-parser.h
-    chat-parser-xml-toolcall.h
     chat-parser-xml-toolcall.cpp
+    chat-parser-xml-toolcall.h
     chat-peg-parser.cpp
     chat-peg-parser.h
+    chat-parsers-internal.h
     chat.cpp
     chat.h
+    ${CHAT_SYNTAX_SOURCES}
     common.cpp
     common.h
     console.cpp

common/arg.cpp

@@ -2880,6 +2880,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.prefill_assistant = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PREFILL_ASSISTANT"));
+    add_opt(common_arg(
+        {"--experimental-new-parsers"},
+        "use experimental new PEG parsers instead of legacy parsers for chat template output parsing (default: disabled)",
+        [](common_params & params) {
+            params.experimental_new_parsers = true;
+            params.use_jinja = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_EXPERIMENTAL_NEW_PARSERS"));
     add_opt(common_arg(
         {"-sps", "--slot-prompt-similarity"}, "SIMILARITY",
         string_format("how much the prompt of a request must match the prompt of a slot in order to use that slot (default: %.2f, 0.0 = disabled)\n", params.slot_prompt_similarity),

common/chat-parser-xml-toolcall.cpp

@@ -1,3 +1,5 @@
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
+#include "chat-parser-xml-toolcall.h"
 #include "chat.h"
 #include "chat-parser.h"
 #include "common.h"

common/chat-parser-xml-toolcall.h

@@ -1,3 +1,4 @@
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 #pragma once
 
 #include "chat.h"

common/chat-parser.cpp

@@ -1,5 +1,6 @@
 #include "chat-parser.h"
 #include "chat-peg-parser.h"
+#include "chat.h"
 #include "common.h"
 #include "log.h"
 #include "peg-parser.h"
@@ -8,6 +9,7 @@
 #include <algorithm>
 #include <cctype>
 #include <optional>
+#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <string_view>
@@ -653,6 +655,7 @@ void common_chat_msg_parser::clear_tools() {
  * All common_chat_parse_* moved from chat.cpp to chat-parser.cpp below
  * to reduce incremental compile time for parser changes.
  */
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_generic(common_chat_msg_parser & builder) {
     if (!builder.syntax().parse_tool_calls) {
         builder.add_content(builder.consume_rest());
@@ -685,6 +688,7 @@ static void common_chat_parse_generic(common_chat_msg_parser & builder) {
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_mistral_nemo(common_chat_msg_parser & builder) {
     if (!builder.syntax().parse_tool_calls) {
         builder.add_content(builder.consume_rest());
@@ -695,6 +699,7 @@ static void common_chat_parse_mistral_nemo(common_chat_msg_parser & builder) {
     parse_prefixed_json_tool_call_array(builder, prefix);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_magistral(common_chat_msg_parser & builder) {
     builder.try_parse_reasoning("[THINK]", "[/THINK]");
 
@@ -707,6 +712,7 @@ static void common_chat_parse_magistral(common_chat_msg_parser & builder) {
     parse_prefixed_json_tool_call_array(builder, prefix);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_command_r7b(common_chat_msg_parser & builder) {
     builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>");
 
@@ -740,6 +746,7 @@ static void common_chat_parse_command_r7b(common_chat_msg_parser & builder) {
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_llama_3_1(common_chat_msg_parser & builder, bool with_builtin_tools = false) {
     builder.try_parse_reasoning("<think>", "</think>");
 
@@ -798,6 +805,7 @@ static void common_chat_parse_llama_3_1(common_chat_msg_parser & builder, bool w
 
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) {
     builder.try_parse_reasoning("<think>", "</think>");
     if (!builder.syntax().parse_tool_calls) {
@@ -819,6 +827,7 @@ static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) {
         tool_calls_end);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & builder) {
     static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?([^\\n<]+)(?:<｜tool▁sep｜>)");
 
@@ -843,6 +852,7 @@ static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & bui
         tool_calls_end);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) {
     // DeepSeek V3.1 outputs reasoning content between "<think>" and "</think>" tags, followed by regular content
     // First try to parse using the standard reasoning parsing method
@@ -879,6 +889,7 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) {
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_minimax_m2(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form {
         /* form.scope_start = */ "<minimax:tool_call>",
@@ -893,6 +904,7 @@ static void common_chat_parse_minimax_m2(common_chat_msg_parser & builder) {
     builder.consume_reasoning_with_xml_tool_calls(form, "<think>", "</think>");
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_qwen3_coder_xml(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form = ([]() {
         xml_tool_call_format form {};
@@ -910,6 +922,7 @@ static void common_chat_parse_qwen3_coder_xml(common_chat_msg_parser & builder)
     builder.consume_reasoning_with_xml_tool_calls(form);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_kimi_k2(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form = ([]() {
         xml_tool_call_format form {};
@@ -929,6 +942,7 @@ static void common_chat_parse_kimi_k2(common_chat_msg_parser & builder) {
     builder.consume_reasoning_with_xml_tool_calls(form, "<think>", "</think>");
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_apriel_1_5(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form = ([]() {
         xml_tool_call_format form {};
@@ -948,6 +962,7 @@ static void common_chat_parse_apriel_1_5(common_chat_msg_parser & builder) {
     builder.consume_reasoning_with_xml_tool_calls(form, "<thinking>", "</thinking>");
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_xiaomi_mimo(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form = ([]() {
         xml_tool_call_format form {};
@@ -966,6 +981,7 @@ static void common_chat_parse_xiaomi_mimo(common_chat_msg_parser & builder) {
     builder.consume_reasoning_with_xml_tool_calls(form);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_gpt_oss(common_chat_msg_parser & builder) {
     static const std::string constraint = "(?: (<\\|constrain\\|>)?([a-zA-Z0-9_-]+))";
     static const std::string recipient("(?: to=functions\\.([^<\\s]+))");
@@ -1054,6 +1070,7 @@ static void common_chat_parse_gpt_oss(common_chat_msg_parser & builder) {
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form {
         /* form.scope_start  = */ "",
@@ -1069,6 +1086,7 @@ static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) {
     builder.consume_reasoning_with_xml_tool_calls(form, "<think>", "</think>");
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_firefunction_v2(common_chat_msg_parser & builder) {
     if (!builder.syntax().parse_tool_calls) {
         builder.add_content(builder.consume_rest());
@@ -1078,6 +1096,7 @@ static void common_chat_parse_firefunction_v2(common_chat_msg_parser & builder)
     parse_prefixed_json_tool_call_array(builder, prefix, /* rstrip_prefix= */ 1);
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_functionary_v3_2(common_chat_msg_parser & builder) {
     static const common_regex function_regex_start_only(R"((\w+\n\{|python\n|all\n))");
     static const common_regex function_regex(R"(>>>(\w+\n\{|python\n|all\n))");
@@ -1107,6 +1126,7 @@ static void common_chat_parse_functionary_v3_2(common_chat_msg_parser & builder)
         });
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_functionary_v3_1_llama_3_1(common_chat_msg_parser & builder) {
     if (!builder.syntax().parse_tool_calls) {
         builder.add_content(builder.consume_rest());
@@ -1133,6 +1153,7 @@ static void common_chat_parse_functionary_v3_1_llama_3_1(common_chat_msg_parser
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_hermes_2_pro(common_chat_msg_parser & builder) {
     builder.try_parse_reasoning("<think>", "</think>");
     if (!builder.syntax().parse_tool_calls) {
@@ -1211,6 +1232,7 @@ static void common_chat_parse_hermes_2_pro(common_chat_msg_parser & builder) {
     builder.add_content(builder.consume_rest());
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_granite(common_chat_msg_parser & builder) {
     // Parse thinking tags
     static const common_regex start_think_regex(regex_escape("<think>"));
@@ -1258,6 +1280,7 @@ static void common_chat_parse_granite(common_chat_msg_parser & builder) {
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_nemotron_v2(common_chat_msg_parser & builder) {
     // Parse thinking tags
     builder.try_parse_reasoning("<think>", "</think>");
@@ -1285,6 +1308,7 @@ static void common_chat_parse_nemotron_v2(common_chat_msg_parser & builder) {
     builder.add_content(builder.consume_rest());
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_apertus(common_chat_msg_parser & builder) {
     // Parse thinking tags
     builder.try_parse_reasoning("<|inner_prefix|>", "<|inner_suffix|>");
@@ -1317,6 +1341,7 @@ static void common_chat_parse_apertus(common_chat_msg_parser & builder) {
 }
 
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_lfm2(common_chat_msg_parser & builder) {
     if (!builder.syntax().parse_tool_calls) {
         builder.add_content(builder.consume_rest());
@@ -1381,6 +1406,7 @@ static void common_chat_parse_lfm2(common_chat_msg_parser & builder) {
     }
 }
 
+// TODO(ochafik): remove once --experimental-new-parsers graduates.
 static void common_chat_parse_seed_oss(common_chat_msg_parser & builder) {
     static const xml_tool_call_format form {
         /* form.scope_start = */ "<seed:tool_call>",
@@ -1486,11 +1512,15 @@ static void common_chat_parse(common_chat_msg_parser & builder) {
 }
 
 common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax) {
+    // Use PEG parser if format explicitly requires it (backward compatibility)
     if (syntax.format == COMMON_CHAT_FORMAT_PEG_SIMPLE ||
         syntax.format == COMMON_CHAT_FORMAT_PEG_NATIVE ||
         syntax.format == COMMON_CHAT_FORMAT_PEG_CONSTRUCTED) {
         return common_chat_peg_parse(syntax.parser, input, is_partial, syntax);
     }
+
+    // TODO(ochafik): remove once --experimental-new-parsers graduates.
+    // Legacy non-PEG parsing path
     common_chat_msg_parser builder(input, is_partial, syntax);
     try {
         common_chat_parse(builder);
@@ -1519,7 +1549,20 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std
     common_peg_parse_context ctx(input, is_partial);
     auto result = parser.parse(ctx);
     if (result.fail()) {
-        throw std::runtime_error(std::string("Failed to parse input at pos ") + std::to_string(result.end));
+        std::ostringstream oss;
+        oss << "Failed to parse input at pos " << result.end;
+        oss << " (format: " << common_chat_format_name(syntax.format) << ")";
+        oss << "\n\nInput (" << input.size() << " chars):\n" << input;
+        if (result.end < input.size()) {
+            oss << "\n\nContext around failure (pos " << result.end << "):\n";
+            size_t start = result.end > 20 ? result.end - 20 : 0;
+            size_t end = std::min(result.end + 20, input.size());
+            if (start > 0) oss << "...";
+            oss << input.substr(start, end - start);
+            if (end < input.size()) oss << "...";
+            oss << "\n" << std::string(start > 0 ? 3 : 0, ' ') << std::string(result.end - start, ' ') << "^";
+        }
+        throw std::runtime_error(oss.str());
     }
 
     common_chat_msg msg;