From 7a16a6ce326f69752aafaf11468b3103331e26d9 Mon Sep 17 00:00:00 2001 From: Clint Herron Date: Sun, 13 Sep 2026 17:56:46 -0400 Subject: [PATCH] grammar : coalesce find + insert into a single insert and adjust move/copy mechanics (#26885) 1) Combine two consecutive lookups (find + insert) into a single insert-attempt/lookup routine so that we don't per form two O(log(n)) lookup operations in a row anymore -- we only need to do it once and then see if the insert succeeded. 2) Instead of copying every potential stack (expensive) and then moving it (cheap) to new_stacks when it's a final output state, we switch the order so that we move every potential stack (cheap), and then only copy it (expensive) to new stacks when it's a final output state. There are a LOT of intermediate states that get generated, and unless they become final output states, then all of these expensive intermediate copies are wasted. Before: lookup -> lookup/insert + copy -> optional move to output New: lookup/insert + move -> optional copy to output --- src/llama-grammar.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp index 6aa03c766..deffec8c0 100644 --- a/src/llama-grammar.cpp +++ b/src/llama-grammar.cpp @@ -871,17 +871,18 @@ static void llama_grammar_advance_stack( std::set seen(stack_cmp); while (!todo.empty()) { - llama_grammar_stack curr_stack = std::move(todo.back()); + llama_grammar_stack curr_stack_candidate = std::move(todo.back()); todo.pop_back(); - if (seen.find( curr_stack) != seen.end()) { + auto [curr_stack_it, inserted] = seen.insert(std::move(curr_stack_candidate)); + if (!inserted) { continue; } - seen.insert(curr_stack); + const llama_grammar_stack & curr_stack = *curr_stack_it; if (curr_stack.empty()) { if (std::find(new_stacks.begin(), new_stacks.end(), curr_stack) == new_stacks.end()) { - new_stacks.emplace_back(std::move(curr_stack)); + new_stacks.emplace_back(curr_stack); } continue; } @@ -924,7 +925,7 @@ static void llama_grammar_advance_stack( case LLAMA_GRETYPE_TOKEN_NOT: if (std::find(new_stacks.begin(), new_stacks.end(), curr_stack) == new_stacks.end()) { // only add the stack if it's not a duplicate of one we already have - new_stacks.emplace_back(std::move(curr_stack)); + new_stacks.emplace_back(curr_stack); } break; default: