Skip to content

Commit 9c8ba73

Browse files
aldehiriamwavecut
aldehir
authored and
iamwavecut
committed
common : add gemma 4 specialized parser (ggml-org#21418)
* common : add gemma4 dedicated parser * cont : add '<|tool_response>' as eog * cont : emit JSON from Gemma4 tool call AST * cont : more fixes * cont : refactor convert function * cont : refine rules and mapping * cont : add more tests * cont : clean up * cont : remove autoparser gemma4 implementation * cont : more cleanup * cont : rename gemma4.jinja to match the others * cont : add custom template to support interleaved thinking * cont : preserve reasoning in model turns * cont : fix initializer error * cont : fix unused vars * cont : fix accidental static * cont : fix specialized_template signature * fix extra semicolon * remove debug line and extra space [no ci]
1 parent 82e0721 commit 9c8ba73

13 files changed

+743
-400
lines changed

‎common/chat-auto-parser-generator.cpp‎

Lines changed: 1 addition & 244 deletions
Original file line numberDiff line numberDiff line change
@@ -8,109 +8,11 @@
88
#include "nlohmann/json.hpp"
99
#include "peg-parser.h"
1010

11-
#include <algorithm>
1211
#include <stdexcept>
1312
#include <string>
1413

1514
using json = nlohmann::ordered_json;
1615

17-
namespace {
18-
19-
// Gemma4-specific PEG builder extending the standard chat builder.
20-
// Adds value type parsers that use <|\"|> as string delimiters
21-
// instead of JSON's double quotes, and disables json-to-schema
22-
// conversion for these types.
23-
class common_peg_gemma4_builder {
24-
common_chat_peg_builder & p_;
25-
static constexpr const char * QUOTE = "<|\"|>";
26-
27-
public:
28-
explicit common_peg_gemma4_builder(common_chat_peg_builder & p) : p_(p) {}
29-
30-
common_peg_parser gemma4_string() {
31-
return p_.rule("gemma4-string", [&]() {
32-
return p_.literal(QUOTE) + p_.until(QUOTE) + p_.literal(QUOTE);
33-
});
34-
}
35-
36-
common_peg_parser gemma4_number() {
37-
return p_.rule("gemma4-number", [&]() {
38-
auto digit1_9 = p_.chars("[1-9]", 1, 1);
39-
auto digits = p_.chars("[0-9]");
40-
auto int_part = p_.choice({p_.literal("0"), p_.sequence({digit1_9, p_.chars("[0-9]", 0, -1)})});
41-
auto frac = p_.sequence({p_.literal("."), digits});
42-
auto exp = p_.sequence({p_.choice({p_.literal("e"), p_.literal("E")}),
43-
p_.optional(p_.chars("[+-]", 1, 1)), digits});
44-
auto not_number_continuation = p_.negate(p_.chars("[0-9.eE+-]", 1, 1));
45-
return p_.sequence({p_.optional(p_.literal("-")), int_part, p_.optional(frac),
46-
p_.optional(exp), not_number_continuation});
47-
});
48-
}
49-
50-
common_peg_parser gemma4_bool() {
51-
return p_.rule("gemma4-bool", [&]() {
52-
return p_.choice({p_.literal("true"), p_.literal("false")});
53-
});
54-
}
55-
56-
common_peg_parser gemma4_null() {
57-
return p_.rule("gemma4-null", [&]() {
58-
return p_.literal("null");
59-
});
60-
}
61-
62-
common_peg_parser gemma4_dict() {
63-
return p_.rule("gemma4-dict", [&]() {
64-
auto ws = p_.space();
65-
auto key = p_.until(":");
66-
auto member = p_.sequence({key, p_.literal(":"), ws, gemma4_value()});
67-
auto members = p_.sequence({member, p_.zero_or_more(p_.sequence({p_.literal(","), ws, member}))});
68-
return p_.sequence({
69-
p_.literal("{"), ws,
70-
p_.choice({p_.literal("}"), p_.sequence({members, ws, p_.literal("}")})})
71-
});
72-
});
73-
}
74-
75-
common_peg_parser gemma4_array() {
76-
return p_.rule("gemma4-array", [&]() {
77-
auto ws = p_.space();
78-
auto elements = p_.sequence({gemma4_value(), p_.zero_or_more(p_.sequence({p_.literal(","), ws, gemma4_value()}))});
79-
return p_.sequence({
80-
p_.literal("["), ws,
81-
p_.choice({p_.literal("]"), p_.sequence({elements, ws, p_.literal("]")})})
82-
});
83-
});
84-
}
85-
86-
common_peg_parser gemma4_value() {
87-
return p_.rule("gemma4-value", [&]() {
88-
return p_.choice({gemma4_string(), gemma4_dict(), gemma4_array(),
89-
gemma4_number(), gemma4_bool(), gemma4_null()});
90-
});
91-
}
92-
93-
// Select the appropriate value parser based on JSON schema type.
94-
// Does NOT use schema() - the gemma4 types are pure PEG without
95-
// JSON schema metadata, so GBNF is generated directly from the
96-
// PEG structure.
97-
common_peg_parser gemma4_value_for_type(const json & schema) {
98-
if (!schema.contains("type") || !schema.at("type").is_string()) {
99-
return gemma4_value();
100-
}
101-
std::string type = schema.at("type").get<std::string>();
102-
if (type == "string") { return gemma4_string(); }
103-
if (type == "number") { return gemma4_number(); }
104-
if (type == "integer") { return gemma4_number(); }
105-
if (type == "boolean") { return gemma4_bool(); }
106-
if (type == "object") { return gemma4_dict(); }
107-
if (type == "array") { return gemma4_array(); }
108-
return gemma4_value();
109-
}
110-
};
111-
112-
} // anonymous namespace
113-
11416
// Helper to iterate over tools/functions
11517
static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
11618
for (const auto & tool : tools) {
@@ -142,9 +44,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
14244
// Create the result structure
14345
common_chat_params data;
14446
data.prompt = common_chat_template_direct_apply(tmpl, inputs);
145-
data.format = (autoparser.tools.format.mode == tool_format::TAG_WITH_GEMMA4_DICT)
146-
? COMMON_CHAT_FORMAT_PEG_GEMMA4
147-
: COMMON_CHAT_FORMAT_PEG_NATIVE;
47+
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
14848
data.preserved_tokens = autoparser.preserved_tokens;
14949

15050
auto parser = autoparser.build_parser(inputs);
@@ -271,8 +171,6 @@ common_peg_parser analyze_tools::build_parser(parser_build_context & ctx) const
271171
return build_tool_parser_tag_json(ctx);
272172
case tool_format::TAG_WITH_TAGGED:
273173
return build_tool_parser_tag_tagged(ctx);
274-
case tool_format::TAG_WITH_GEMMA4_DICT:
275-
return build_tool_parser_tag_gemma4_dict(ctx);
276174
default:
277175
LOG_ERR("[ERROR] Template seems to support tool calls, but failed to determine tool format. Tool calling will not work properly. "
278176
"Check for a fixed template for your model in the models/templates directory of your llama.cpp installation or "
@@ -586,145 +484,4 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
586484
p.end();
587485
}
588486

589-
common_peg_parser analyze_tools::build_tool_parser_tag_gemma4_dict(parser_build_context & ctx) const {
590-
auto & p = ctx.p;
591-
const auto & inputs = ctx.inputs;
592-
bool force_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
593-
594-
common_peg_gemma4_builder g4(p);
595-
static const std::string QUOTE = "<|\"|>";
596-
597-
common_peg_parser tool_choice = p.choice();
598-
599-
foreach_function(inputs.tools, [&](const json & tool) {
600-
const auto & func = tool.at("function");
601-
std::string name = func.at("name");
602-
const auto & params = func.at("parameters");
603-
604-
if (!params.contains("properties") || !params.at("properties").is_object()) {
605-
auto func_parser = p.atomic(
606-
p.tool_open(p.literal(function.name_prefix) + p.tool_name(p.literal(name)) + p.literal("{")) +
607-
p.tool_args(p.eps()) +
608-
p.tool_close(p.literal("}")));
609-
tool_choice |= p.rule("tool-" + name, func_parser);
610-
return;
611-
}
612-
613-
const auto & properties = params.at("properties");
614-
std::set<std::string> required;
615-
if (params.contains("required") && params.at("required").is_array()) {
616-
params.at("required").get_to(required);
617-
}
618-
619-
// Build per-argument parsers, sorted alphabetically (matching template's dictsort)
620-
struct arg_entry {
621-
std::string param_name;
622-
common_peg_parser parser;
623-
};
624-
std::vector<arg_entry> arg_entries;
625-
626-
for (const auto & [param_name, param_schema] : properties.items()) {
627-
std::string type = "object";
628-
if (param_schema.contains("type")) {
629-
const auto & type_v = param_schema.at("type");
630-
if (type_v.is_string()) {
631-
type_v.get_to(type);
632-
} else if (type_v.is_array()) {
633-
// Handle nullable types like ["string", "null"]
634-
for (const auto & t : type_v) {
635-
if (t.is_string() && t.get<std::string>() != "null") {
636-
type = t.get<std::string>();
637-
break;
638-
}
639-
}
640-
}
641-
}
642-
// Infer string type from enum values when type is unspecified
643-
if (type == "object" && param_schema.contains("enum")) {
644-
const auto & enum_vals = param_schema.at("enum");
645-
if (enum_vals.is_array()) {
646-
for (const auto & v : enum_vals) {
647-
if (v.is_string()) {
648-
type = "string";
649-
break;
650-
}
651-
}
652-
}
653-
}
654-
655-
common_peg_parser value_parser = p.eps();
656-
if (type == "string") {
657-
// String values are delimited by <|"|>...<|"|>
658-
value_parser =
659-
p.literal(QUOTE) +
660-
p.tool_arg_string_value(p.schema(p.until(QUOTE),
661-
"tool-" + name + "-arg-" + param_name + "-schema", param_schema, true)) +
662-
p.literal(QUOTE);
663-
} else if (type == "number" || type == "integer") {
664-
value_parser = p.tool_arg_value(g4.gemma4_number());
665-
} else if (type == "boolean") {
666-
value_parser = p.tool_arg_value(g4.gemma4_bool());
667-
} else if (type == "null") {
668-
value_parser = p.tool_arg_value(g4.gemma4_null());
669-
} else if (type == "object") {
670-
value_parser = p.tool_arg_value(g4.gemma4_dict());
671-
} else if (type == "array") {
672-
value_parser = p.tool_arg_value(g4.gemma4_array());
673-
} else {
674-
value_parser = p.tool_arg_value(g4.gemma4_value());
675-
}
676-
677-
auto arg = p.tool_arg(
678-
p.tool_arg_open(p.tool_arg_name(p.literal(param_name)) + p.literal(":")) +
679-
value_parser +
680-
p.tool_arg_close(p.eps()));
681-
682-
arg_entries.push_back({param_name, p.rule("tool-" + name + "-arg-" + param_name, arg)});
683-
}
684-
685-
// Sort alphabetically to match Jinja's dictsort
686-
std::sort(arg_entries.begin(), arg_entries.end(), [](const auto & a, const auto & b) {
687-
return a.param_name < b.param_name;
688-
});
689-
690-
// Build arg sequence: any arg, then zero-or-more comma-separated additional args
691-
common_peg_parser args_seq = p.eps();
692-
if (!arg_entries.empty()) {
693-
common_peg_parser any_arg = p.choice();
694-
for (auto & entry : arg_entries) {
695-
any_arg |= entry.parser;
696-
}
697-
args_seq = p.optional(
698-
any_arg + p.repeat(p.literal(",") + any_arg, 0, (int) arg_entries.size() - 1));
699-
}
700-
701-
// Full parser: call:name{args}
702-
auto func_parser = p.atomic(
703-
p.tool_open(p.literal(function.name_prefix) + p.tool_name(p.literal(name)) + p.literal("{")) +
704-
p.tool_args(args_seq) +
705-
p.tool_close(p.literal("}")));
706-
707-
tool_choice |= p.rule("tool-" + name, func_parser);
708-
});
709-
710-
// Wrap each call in <|tool_call>...</tool_call|>
711-
auto wrapped_call = p.literal(format.per_call_start) + tool_choice + p.literal(format.per_call_end);
712-
713-
common_peg_parser tool_calls = p.eps();
714-
if (inputs.parallel_tool_calls) {
715-
tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call));
716-
} else {
717-
tool_calls = p.trigger_rule("tool-call", wrapped_call);
718-
}
719-
720-
if (!force_tools) {
721-
tool_calls = p.optional(tool_calls);
722-
}
723-
724-
auto content_before_tools = p.until_one_of({ format.per_call_start, ctx.reasoning->start });
725-
return ctx.reasoning_parser +
726-
(force_tools ? p.eps() : p.optional(p.content(content_before_tools) + p.optional(ctx.reasoning_parser))) +
727-
tool_calls + p.end();
728-
}
729-
730487
} // namespace autoparser

‎common/chat-auto-parser.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,6 @@ enum class tool_format {
145145
JSON_NATIVE, // Pure JSON: {"name": "X", "arguments": {...}}
146146
TAG_WITH_JSON, // Tag-based with JSON args: <function=X>{...}</function>
147147
TAG_WITH_TAGGED, // Tag-based with tagged args: <param=key>value</param>
148-
TAG_WITH_GEMMA4_DICT, // Gemma4 custom dict: <|tool_call>call:name{key:<|"|>val<|"|>}<tool_call|>
149148
};
150149

151150
inline std::ostream & operator<<(std::ostream & os, const tool_format & format) {
@@ -158,8 +157,6 @@ inline std::ostream & operator<<(std::ostream & os, const tool_format & format)
158157
return os << "TAG_WITH_JSON";
159158
case tool_format::TAG_WITH_TAGGED:
160159
return os << "TAG_WITH_TAGGED";
161-
case tool_format::TAG_WITH_GEMMA4_DICT:
162-
return os << "TAG_WITH_GEMMA4_DICT";
163160
default:
164161
return os << "UNKNOWN";
165162
}
@@ -363,7 +360,6 @@ struct analyze_tools : analyze_base {
363360
const common_peg_parser & call_id_section, bool have_call_id,
364361
const common_peg_parser & args,
365362
std::optional<common_peg_parser> atomic_peek) const;
366-
common_peg_parser build_tool_parser_tag_gemma4_dict(parser_build_context & ctx) const;
367363
};
368364

369365
// ============================================================================

‎common/chat-diff-analyzer.cpp‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -95,34 +95,6 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autopar
9595
LOG_DBG(ANSI_ORANGE "[Patch: Functionary 3.1]\n" ANSI_RESET);
9696
}
9797
},
98-
// Gemma4 - custom dict format: <|tool_call>call:name{key:<|"|>val<|"|>}<tool_call|>
99-
[](const common_chat_template & tmpl, autoparser & analysis) -> void {
100-
if (tmpl.src.find("'<|tool_call>call:'") != std::string::npos) {
101-
analysis.tools.format.mode = tool_format::TAG_WITH_GEMMA4_DICT;
102-
analysis.tools.format.per_call_start = "<|tool_call>";
103-
analysis.tools.format.per_call_end = "<tool_call|>";
104-
analysis.tools.format.section_start = "";
105-
analysis.tools.format.section_end = "";
106-
analysis.tools.function.name_prefix = "call:";
107-
analysis.tools.function.name_suffix = "";
108-
analysis.tools.arguments.start = "{";
109-
analysis.tools.arguments.end = "}";
110-
analysis.tools.arguments.name_prefix = "";
111-
analysis.tools.arguments.name_suffix = ":";
112-
analysis.tools.arguments.separator = ",";
113-
analysis.reasoning.mode = reasoning_mode::TAG_BASED;
114-
analysis.reasoning.start = "<|channel>thought";
115-
analysis.reasoning.end = "<channel|>";
116-
analysis.preserved_tokens.clear();
117-
analysis.preserved_tokens.push_back("<|tool_call>");
118-
analysis.preserved_tokens.push_back("<tool_call|>");
119-
analysis.preserved_tokens.push_back("<|tool_response>");
120-
analysis.preserved_tokens.push_back("<tool_response|>");
121-
analysis.preserved_tokens.push_back("<|\"|>");
122-
analysis.preserved_tokens.push_back("<|turn>");
123-
LOG_DBG(ANSI_ORANGE "[Patch: Gemma4]\n" ANSI_RESET);
124-
}
125-
},
12698
// DeepSeek-R1-Distill-Qwen
12799
[](const common_chat_template & tmpl, autoparser & analysis) -> void {
128100
if (tmpl.src.find(

0 commit comments

Comments
 (0)