From 9534049d523691688db2362bf1222685b35d6971 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 18 Aug 2026 11:51:30 +0200 Subject: [PATCH] (sip) allow inject messge to generation --- tools/server/README.md | 6 +- tools/server/server-context.cpp | 56 +++++++++++++++++- tools/server/server-task.h | 1 + .../app/chat/ChatScreen/ChatScreen.svelte | 5 +- .../app/chat/ChatScreen/ChatScreenForm.svelte | 3 +- .../constants/control-actions.constants.ts | 3 +- .../lib/constants/settings-keys.constants.ts | 2 + .../constants/settings-registry.constants.ts | 18 ++++++ tools/ui/src/lib/services/chat.service.ts | 52 +++++++++++++++++ tools/ui/src/lib/stores/chat.svelte.ts | 57 +++++++++++++++++-- 10 files changed, 188 insertions(+), 15 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index 67e52b1db8..3e2507d52e 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1443,7 +1443,11 @@ Acts on an in-flight completion identified by its `id` (the `id` field streamed `id`: (Required) The chat completion id to act on. A completion that has already finished matches nothing and the call is a no-op. -`action`: (Required) The control action to perform. Currently the only supported value is `reasoning_end`, which forces the end of the current reasoning block so the model moves on to the final answer. Requires `reasoning_control: true` on the original completion request. +`action`: (Required) The control action to perform. Supported values: +- `reasoning_end`: forces the end of the current reasoning block so the model moves on to the final answer. Requires `reasoning_control: true` on the original completion request. +- `inject`: appends the given `text` to the generated output as if the model produced it, then generation continues from there. Useful to steer the model mid reasoning by injecting a thought. Fails if the completion is still processing the prompt, or if it is constrained by a grammar / JSON schema. Injected tokens are streamed back like sampled tokens and count towards `max_tokens`. + +`text`: (Required for `inject`) The text to inject. `model`: (Required in router mode) The model name, used to route the request to the right instance. Ignored in single model mode. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 842e4203cd..063697bf93 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -305,6 +305,9 @@ struct server_slot { llama_token sampled; // in speculative mode, this is the last accepted token + // tokens queued by the control endpoint "inject" action, consumed instead of sampling + llama_tokens inject_tokens; + // for TTS models, this is the embd generated from prev step, decode this to generate next hidden state // corresponding to one token position (size = n_embd) std::vector inp_embd; @@ -342,6 +345,7 @@ struct server_slot { } generated_tokens.clear(); generated_token_probs.clear(); + inject_tokens.clear(); json_schema = json(); task_prev = std::move(task); @@ -443,6 +447,11 @@ struct server_slot { return 0; } + // do not draft while injected tokens are pending + if (!inject_tokens.empty()) { + return 0; + } + // determine the max draft that fits the current slot state // note: slot.prompt is not yet expanded with the `id` token sampled above // also, need to leave space for 1 extra token to allow context shifts @@ -2404,6 +2413,24 @@ private: // act on the live slot mid generation, never defer common_sampler_reasoning_budget_force(slot->smpl.get()); res->success = true; + } else if (task.params.control_action == "inject") { + if (slot->state != SLOT_STATE_GENERATING) { + res->success = false; + res->message = "completion is still processing the prompt"; + } else if (!slot->task->params.sampling.grammar.empty() && !slot->task->params.sampling.grammar_lazy) { + // a lazy grammar (tool calls with auto choice) is fine as long as the injected text does not trigger it + res->success = false; + res->message = "cannot inject into a completion constrained by a grammar"; + } else { + const llama_tokens tokens = common_tokenize(vocab, task.params.control_text, false, true); + if (tokens.empty()) { + res->success = false; + res->message = "text produced no tokens"; + } else { + slot->inject_tokens.insert(slot->inject_tokens.end(), tokens.begin(), tokens.end()); + res->success = true; + } + } } else { res->success = false; res->message = "unknown control action"; @@ -3757,14 +3784,31 @@ private: const int tok_idx = slot.i_batch - off; llama_token id; - { + const bool injected = !slot.inject_tokens.empty(); + if (injected) { + // consume an injected token instead of sampling + id = slot.inject_tokens.front(); + slot.inject_tokens.erase(slot.inject_tokens.begin()); + } else { scoped_timer timer(t_sampl, n_sampl); id = common_sampler_sample(slot.smpl.get(), slot.ctx_tgt, tok_idx); } slot.i_batch = -1; - common_sampler_accept(slot.smpl.get(), id, true); + if (injected) { + // an injected token can violate a lazy grammar; fail only this slot instead of aborting all slots + try { + common_sampler_accept(slot.smpl.get(), id, true); + } catch (const std::exception & e) { + SLT_ERR(slot, "injected token rejected by grammar: %s\n", e.what()); + send_error(slot, "injected text conflicts with the active grammar", ERROR_TYPE_INVALID_REQUEST); + slot.release(); + return; + } + } else { + common_sampler_accept(slot.smpl.get(), id, true); + } // here we have synchronized the llama_context (due to the sampling above), so we can do time measurement const int64_t t_now = ggml_time_us(); @@ -4756,14 +4800,19 @@ void server_routes::init_routes() { const std::string cmpl_id = json_value(body, "id", std::string()); const std::string action = json_value(body, "action", std::string()); + const std::string text = json_value(body, "text", std::string()); if (cmpl_id.empty()) { res->error(format_error_response("missing completion id", ERROR_TYPE_INVALID_REQUEST)); return res; } - if (action != "reasoning_end") { + if (action != "reasoning_end" && action != "inject") { res->error(format_error_response("unknown control action", ERROR_TYPE_INVALID_REQUEST)); return res; } + if (action == "inject" && text.empty()) { + res->error(format_error_response("missing text for inject action", ERROR_TYPE_INVALID_REQUEST)); + return res; + } auto & rd = res->rd; { @@ -4771,6 +4820,7 @@ void server_routes::init_routes() { task.id = rd.get_new_id(); task.params.control_cmpl_id = cmpl_id; task.params.control_action = action; + task.params.control_text = text; rd.post_task(std::move(task)); } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index b6da4d4bd6..686aca7750 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -87,6 +87,7 @@ struct task_params { // realtime control (SERVER_TASK_TYPE_CONTROL) std::string control_action; std::string control_cmpl_id; + std::string control_text; // for "inject" action // per-request parameters for chat parsing common_chat_parser_params chat_parser_params; diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte index 2b5ca68de3..f7805120cd 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte @@ -129,9 +129,8 @@ handleSendLikeScroll(); - await chatStore.sendMessage(message, result?.extras); - - return true; + // false means a mid-generation inject failed; the form restores the input + return await chatStore.sendMessage(message, result?.extras); } let lastScrolledConversationId: string | null = null; diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte index 4119b2816d..d8c7ab2fe3 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte @@ -84,6 +84,7 @@ if (!chatFormRef?.checkModelSelected()) return; + const originalMessage = message; const messageToSend = message.trim(); const filesToSend = [...uploadedFiles]; @@ -96,7 +97,7 @@ const success = await onSend?.(messageToSend, filesToSend); if (!success) { - message = messageToSend; + message = originalMessage; uploadedFiles = filesToSend; } } diff --git a/tools/ui/src/lib/constants/control-actions.constants.ts b/tools/ui/src/lib/constants/control-actions.constants.ts index c8ebf701b1..069e98a6b2 100644 --- a/tools/ui/src/lib/constants/control-actions.constants.ts +++ b/tools/ui/src/lib/constants/control-actions.constants.ts @@ -1,5 +1,6 @@ // actions accepted by the realtime inference control endpoint (API_CHAT.CONTROL) // kept separate from the endpoint paths since these are protocol level verbs export const CONTROL_ACTION = { - END_REASONING: 'reasoning_end' + END_REASONING: 'reasoning_end', + INJECT: 'inject' } as const; diff --git a/tools/ui/src/lib/constants/settings-keys.constants.ts b/tools/ui/src/lib/constants/settings-keys.constants.ts index b53d11048d..bc159c4bae 100644 --- a/tools/ui/src/lib/constants/settings-keys.constants.ts +++ b/tools/ui/src/lib/constants/settings-keys.constants.ts @@ -6,6 +6,7 @@ */ export const SETTINGS_KEYS = { AGENTIC_MAX_TURNS: 'agenticMaxTurns', + ALLOW_INJECT_USER_MESSAGE_MID_GENERATION: 'allowInjectUserMessageMidGeneration', ALWAYS_SHOW_SIDEBAR_ON_DESKTOP: 'alwaysShowSidebarOnDesktop', ALWAYS_SHOW_TOOL_CALL_CONTENT: 'alwaysShowToolCallContent', API_KEY: 'apiKey', @@ -28,6 +29,7 @@ export const SETTINGS_KEYS = { EXCLUDE_REASONING_FROM_CONTEXT: 'excludeReasoningFromContext', FREQUENCY_PENALTY: 'frequency_penalty', FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks', + INJECTION_TEMPLATE: 'injectionTemplate', JS_SANDBOX_ENABLED: 'jsSandboxEnabled', MAX_IMAGE_RESOLUTION: 'maxImageMPixels', MAX_TOKENS: 'max_tokens', diff --git a/tools/ui/src/lib/constants/settings-registry.constants.ts b/tools/ui/src/lib/constants/settings-registry.constants.ts index bf43a26e86..4ef1131a7b 100644 --- a/tools/ui/src/lib/constants/settings-registry.constants.ts +++ b/tools/ui/src/lib/constants/settings-registry.constants.ts @@ -136,6 +136,24 @@ const SETTINGS_REGISTRY: Record = { section: SETTINGS_SECTION_SLUGS.DEVELOPER, type: SettingsFieldType.CHECKBOX }, + { + defaultValue: false, + help: 'Send a message typed during a streaming response straight into the model output via the control endpoint, instead of queueing it until the response finishes. Text only; messages with attachments are still queued. The injected text appears inside the assistant response.', + key: SETTINGS_KEYS.ALLOW_INJECT_USER_MESSAGE_MID_GENERATION, + label: 'Allow inject user message mid-generation', + section: SETTINGS_SECTION_SLUGS.DEVELOPER, + type: SettingsFieldType.CHECKBOX + }, + { + defaultValue: + '\nUser wants interrupt without stopping this session. Therefore, you are seeing this as part of your reasoning process:\n{message}\n', + dependsOn: SETTINGS_KEYS.ALLOW_INJECT_USER_MESSAGE_MID_GENERATION, + help: 'Template applied to a message injected mid-generation. {message} is replaced with the typed text.', + key: SETTINGS_KEYS.INJECTION_TEMPLATE, + label: 'Injection template', + section: SETTINGS_SECTION_SLUGS.DEVELOPER, + type: SettingsFieldType.TEXTAREA + }, { defaultValue: false, help: 'Strip thinking from previous messages before sending. When off, thinking is sent back via the reasoning_content field so the model sees its own chain-of-thought across turns.', diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 1dde2e18fe..5f1e08af29 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -508,6 +508,58 @@ export class ChatService { } } + /** + * Injects text into a running completion, targeted by its chat completion id. + * The server appends the tokens to the generated output as if the model + * produced them and streams them back. Returns true on success. + */ + static async injectText( + completionId: string, + text: string, + model?: string | null + ): Promise { + if (!completionId) { + console.error( + 'injectText: no completion id for the active message, cannot target the running completion' + ); + + return false; + } + + const body: Record = { + action: CONTROL_ACTION.INJECT, + id: completionId, + text + }; + + if (model) body.model = model; + + try { + const res = await fetch(API_CHAT.CONTROL, { + body: JSON.stringify(body), + headers: getJsonHeaders(), + method: 'POST' + }); + const data = await res.json().catch(() => null); + + if (!res.ok || data?.success !== true) { + console.error('injectText: control request failed', { + completionId, + response: data, + status: res.status + }); + + return false; + } + + return true; + } catch (error) { + console.error('injectText: control request threw', { completionId, error }); + + return false; + } + } + /** * Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache. * After a response completes, this re-submits the full conversation diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 7234d00276..df2952dc57 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -1187,23 +1187,37 @@ class ChatStore { ); } - async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise { - if (!content.trim() && (!extras || extras.length === 0)) return; + /** Returns false only when a mid-generation inject failed, so the caller can restore the input. */ + async sendMessage(content: string, extras?: DatabaseMessageExtra[]): Promise { + if (!content.trim() && (!extras || extras.length === 0)) return true; const activeConv = conversationsStore.activeConversation; + // text only for now: messages with attachments always take the steering/queue path + const preferDirectInject = + Boolean(settingsStore.config.allowInjectUserMessageMidGeneration) && + (!extras || extras.length === 0); + // If agentic loop is running, inject as a steering message instead of starting a new flow if (activeConv && agenticStore.isRunning(activeConv.id)) { + if (preferDirectInject) { + return await this.injectIntoActiveCompletion(content); + } + agenticStore.injectSteeringMessage(activeConv.id, content, extras); - return; + return true; } // If non-agentic streaming is active, queue as a pending message to send after completion if (activeConv && this.isChatLoadingInternal(activeConv.id)) { + if (preferDirectInject) { + return await this.injectIntoActiveCompletion(content); + } + this.injectPendingMessage(activeConv.id, content, extras); - return; + return true; } // Cancel any in-flight pre-encode request @@ -1222,7 +1236,7 @@ class ChatStore { const currentConv = conversationsStore.activeConversation; - if (!currentConv) return; + if (!currentConv) return true; this.showErrorDialog(null); this.setChatLoading(currentConv.id, true); @@ -1300,7 +1314,7 @@ class ChatStore { if (isAbortError(error)) { this.setChatLoading(currentConv.id, false); - return; + return true; } console.error('Failed to send message:', error); @@ -1319,6 +1333,37 @@ class ChatStore { type: dialogType }); } + + return true; + } + + /** + * Sends the text into the running completion of the active conversation via the + * control endpoint. The injected text is streamed back as part of the assistant + * response, so no message is added to the conversation. + */ + private async injectIntoActiveCompletion(content: string): Promise { + const messages = conversationsStore.activeMessages; + const activeMessage = messages[messages.length - 1]; + + if (!activeMessage?.completionId) return false; + + const template = String(settingsStore.config.injectionTemplate ?? ''); + + let rendered = content; + + if (template.includes('{message}')) { + rendered = template.replaceAll('{message}', content); + } else if (template) { + rendered = `${template}\n${content}`; + } + + // surround with newlines so the text does not glue to the tokens around it + return await ChatService.injectText( + activeMessage.completionId, + `\n${rendered}\n`, + activeMessage.model + ); } private async streamChatCompletion(