Merge branch 'upstream' into concedo_experimental

# Conflicts:
#	.devops/cann.Dockerfile
#	.devops/cpu.Dockerfile
#	.devops/cuda.Dockerfile
#	.devops/intel.Dockerfile
#	.devops/musa.Dockerfile
#	.devops/openvino.Dockerfile
#	.devops/rocm.Dockerfile
#	.devops/s390x.Dockerfile
#	.devops/vulkan.Dockerfile
#	.devops/zendnn.Dockerfile
#	.github/workflows/build-cache.yml
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/release.yml
#	app/llama.cpp
#	build-xcframework.sh
#	docs/backend/OPENVINO.md
#	ggml/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-openvino/ggml-decoder.cpp
#	ggml/src/ggml-openvino/openvino/op/add_id.cpp
#	ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp
#	ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
#	ggml/src/ggml-openvino/openvino/op/softmax.cpp
#	ggml/src/ggml-openvino/openvino/op_table.cpp
#	ggml/src/ggml-openvino/openvino/op_table.h
#	ggml/src/ggml-sycl/softmax.cpp
#	scripts/sync-ggml.last
#	tests/test-backend-ops.cpp
#	tests/test-quantize-fns.cpp
#	tools/server/CMakeLists.txt
#	tools/ui/src/lib/services/chat.service.ts
This commit is contained in:
Concedo
2026-06-27 10:33:29 +08:00
54 changed files with 2681 additions and 335 deletions
@@ -33,7 +33,7 @@
{#if !readonly && onRemove}
<div
class="absolute top-10 right-2 flex items-center justify-center opacity-0 transition-opacity group-hover:opacity-100"
class="absolute top-10 right-2 flex items-center justify-center opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100"
>
<ActionIcon icon={X} tooltip="Remove" stopPropagationOnClick onclick={() => onRemove?.()} />
</div>
@@ -56,7 +56,7 @@
<div class="relative flex h-6 items-center justify-between">
<div class="right-0 flex items-center gap-2 opacity-100 transition-opacity">
<div
class="pointer-events-auto inset-0 flex items-center gap-1 opacity-0 transition-all duration-150 group-hover:opacity-100"
class="pointer-events-auto inset-0 flex items-center gap-1 opacity-0 transition-all duration-150 group-focus-within:opacity-100 group-hover:opacity-100"
>
<ActionIcon icon={Edit} tooltip="Edit" onclick={editCtx.handleEdit} />
<ActionIcon icon={Trash2} tooltip="Delete" onclick={onDelete} />
@@ -5,6 +5,7 @@
ChatMessages,
ChatScreenDragOverlay,
ChatScreenProcessingInfo,
ChatScreenStreamResumeStatus,
ServerLoadingSplash,
ChatScreenServerError
} from '$lib/components/app';
@@ -281,6 +282,10 @@
<ChatScreenServerError />
{#if page.params.id}
<ChatScreenStreamResumeStatus />
{/if}
<div class="pointer-events-none flex flex-col gap-6 items-center w-full">
{#if (isMobile.current ? mobileScrollDownHint || isMobileUserScrolledUp : autoScroll.userScrolledUp) && page.url.hash.includes(ROUTES.CHAT) && page.params.id}
<ChatScreenActionScrollDown
@@ -0,0 +1,18 @@
<script lang="ts">
import { chatStore } from '$lib/stores/chat.svelte';
import { StreamConnectionState } from '$lib/enums';
import { Loader2 } from '@lucide/svelte';
let state = $derived(chatStore.streamConnectionState);
</script>
{#if state === StreamConnectionState.RESUMING}
<div
class="pointer-events-auto mx-auto mt-2 mb-2 flex max-w-[48rem] items-center gap-2 rounded-md border border-blue-400/40 bg-blue-50/60 px-3 py-1.5 text-sm text-blue-700 dark:bg-blue-950/40 dark:text-blue-200"
role="status"
aria-live="polite"
>
<Loader2 class="h-3.5 w-3.5 animate-spin" />
<span>Reconnecting to the stream...</span>
</div>
{/if}
@@ -683,3 +683,11 @@ export { default as ChatScreenProcessingInfo } from './ChatScreen/ChatScreenProc
* Rendered inside ChatScreen when `serverError` store has a value.
*/
export { default as ChatScreenServerError } from './ChatScreen/ChatScreenServerError.svelte';
/**
* Stream resume status indicator. Shows a small "Reconnecting to the stream..."
* banner with a spinner while `chatStore.streamConnectionState` is `resuming`,
* i.e. after a dropped connection is reattaching to the live SSE replay buffer.
* Renders nothing otherwise. Shown inside ChatScreen only on an active conversation route.
*/
export { default as ChatScreenStreamResumeStatus } from './ChatScreen/ChatScreenStreamResumeStatus.svelte';
@@ -39,7 +39,6 @@
depth = 0
}: Props = $props();
let renderActionsDropdown = $state(false);
let dropdownOpen = $state(false);
let isLoading = $derived(getAllLoadingChats().includes(conversation.id));
@@ -71,26 +70,10 @@
}
}
function handleMouseLeave() {
if (!dropdownOpen) {
renderActionsDropdown = false;
}
}
function handleMouseOver() {
renderActionsDropdown = true;
}
function handleSelect() {
onSelect?.(conversation.id);
}
$effect(() => {
if (!dropdownOpen) {
renderActionsDropdown = false;
}
});
onMount(() => {
document.addEventListener('edit-active-conversation', handleGlobalEditEvent as EventListener);
@@ -103,23 +86,19 @@
});
</script>
<!-- svelte-ignore a11y_mouse_events_have_key_events -->
<button
class="group flex min-h-9 w-full cursor-pointer items-center justify-between space-x-3 rounded-lg py-1.5 text-left transition-colors hover:bg-foreground/10 {isActive
<div
class="conversation-item group relative flex min-h-9 w-full items-center justify-between space-x-3 rounded-lg py-1.5 transition-colors hover:bg-foreground/10 {isActive
? 'bg-foreground/5 text-accent-foreground'
: ''} px-3"
onclick={handleSelect}
onmouseover={handleMouseOver}
onmouseleave={handleMouseLeave}
onfocusin={handleMouseOver}
onfocusout={(e) => {
if (!e.currentTarget.contains(e.relatedTarget as Node | null)) {
handleMouseLeave();
}
}}
>
<button
class="absolute inset-0 z-0 cursor-pointer rounded-lg focus:outline-none focus-visible:ring-2 focus-visible:ring-ring"
onclick={handleSelect}
aria-label={conversation.name}
>
</button>
<div
class="flex min-w-0 flex-1 items-center gap-2"
class="pointer-events-none relative z-10 flex min-w-0 flex-1 items-center gap-2"
style:padding-left="{depth * FORK_TREE_DEPTH_PADDING}px"
>
{#if depth > 0}
@@ -130,7 +109,7 @@
<a
{...props}
href={RouterService.chat(conversation.forkedFromConversationId)}
class="flex shrink-0 items-center text-muted-foreground transition-colors hover:text-foreground"
class="pointer-events-auto flex shrink-0 items-center text-muted-foreground transition-colors hover:text-foreground"
>
<GitBranch class="h-3.5 w-3.5" />
</a>
@@ -146,18 +125,15 @@
{#if isLoading}
<Tooltip.Root>
<Tooltip.Trigger>
<div
class="stop-button flex h-4 w-4 shrink-0 cursor-pointer items-center justify-center rounded text-muted-foreground transition-colors hover:text-foreground"
<button
class="stop-button pointer-events-auto flex h-4 w-4 shrink-0 cursor-pointer items-center justify-center rounded text-muted-foreground transition-colors hover:text-foreground"
onclick={handleStop}
onkeydown={(e) => e.key === 'Enter' && handleStop(e)}
role="button"
tabindex="0"
aria-label="Stop generation"
>
<Loader2 class="loading-icon h-3.5 w-3.5 animate-spin" />
<Square class="stop-icon hidden h-3 w-3 fill-current text-destructive" />
</div>
</button>
</Tooltip.Trigger>
<Tooltip.Content>
@@ -169,52 +145,50 @@
<TruncatedText text={conversation.name} class="text-sm font-medium" showTooltip={false} />
</div>
{#if renderActionsDropdown}
<div class="actions flex items-center">
<DropdownMenuActions
triggerIcon={MoreHorizontal}
triggerTooltip="More actions"
bind:open={dropdownOpen}
actions={[
{
icon: conversation.pinned ? PinOff : Pin,
label: conversation.pinned ? 'Unpin' : 'Pin',
onclick: (e: Event) => {
e.stopPropagation();
handleTogglePin();
}
},
{
icon: Pencil,
label: 'Edit',
onclick: handleEdit,
shortcut: ['shift', 'cmd', 'e']
},
{
icon: Download,
label: 'Export',
onclick: (e: Event) => {
e.stopPropagation();
conversationsStore.downloadConversation(conversation.id);
},
shortcut: ['shift', 'cmd', 's']
},
{
icon: Trash2,
label: 'Delete',
onclick: handleDelete,
variant: 'destructive',
shortcut: ['shift', 'cmd', 'd'],
separator: true
<div class="actions pointer-events-auto relative z-20 flex items-center">
<DropdownMenuActions
triggerIcon={MoreHorizontal}
triggerTooltip="More actions"
bind:open={dropdownOpen}
actions={[
{
icon: conversation.pinned ? PinOff : Pin,
label: conversation.pinned ? 'Unpin' : 'Pin',
onclick: (e: Event) => {
e.stopPropagation();
handleTogglePin();
}
]}
/>
</div>
{/if}
</button>
},
{
icon: Pencil,
label: 'Edit',
onclick: handleEdit,
shortcut: ['shift', 'cmd', 'e']
},
{
icon: Download,
label: 'Export',
onclick: (e: Event) => {
e.stopPropagation();
conversationsStore.downloadConversation(conversation.id);
},
shortcut: ['shift', 'cmd', 's']
},
{
icon: Trash2,
label: 'Delete',
onclick: handleDelete,
variant: 'destructive',
shortcut: ['shift', 'cmd', 'd'],
separator: true
}
]}
/>
</div>
</div>
<style>
button {
.conversation-item {
:global([data-slot='dropdown-menu-trigger']:not([data-state='open'])) {
opacity: 0;
}
@@ -239,7 +213,8 @@
}
}
&:is(:hover) .stop-button {
&:is(:hover) .stop-button,
&:focus-within .stop-button {
:global(.stop-icon) {
display: block;
}
@@ -21,5 +21,11 @@ export const API_TOOLS = {
EXECUTE: '/tools'
};
// resumable stream routes, the conv::model identity is appended as a path segment
export const API_STREAM = {
BASE: './v1/stream',
LOOKUP: './v1/streams/lookup'
};
/** CORS proxy endpoint path */
export const CORS_PROXY_ENDPOINT = '/cors-proxy';
+1
View File
@@ -46,6 +46,7 @@ export * from './routes';
export * from './sandbox';
export * from './settings-keys';
export * from './settings-registry';
export * from './stream';
export * from './supported-file-types';
export * from './table-html-restorer';
export * from './title-generation';
+3
View File
@@ -26,6 +26,9 @@ export const THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.th
export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`;
export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`;
/** Key prefix for per-conversation resumable stream state, conversationId is appended */
export const STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX = `${STORAGE_APP_NAME}.streamResume.`;
// Deprecated old key names (kept for backward compat while users migrate)
/** @deprecated Use {@link ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY} instead */
export const DEPRECATED_ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME_DEPRECATED}.alwaysAllowedTools`;
+3
View File
@@ -0,0 +1,3 @@
// grace window after a visibilitychange before we kick a reader whose socket likely died
// while the tab was hidden. covers brief background pauses without thrashing live streams
export const STREAM_VISIBILITY_KICK_MS = 1000;
+9
View File
@@ -5,6 +5,15 @@ export enum ChatMessageStatsView {
SUMMARY = 'summary'
}
/**
* Connection state of a streamed completion, drives the resume status indicator.
*/
export enum StreamConnectionState {
STREAMING = 'streaming',
RESUMING = 'resuming',
LOST = 'lost'
}
/**
* Reasoning format options for API requests.
*/
+1
View File
@@ -10,6 +10,7 @@ export { AgenticSectionType, ContinueIntentKind, ToolCallType } from './agentic.
export {
ChatMessageStatsView,
StreamConnectionState,
ContentPartType,
ConversationSelectionMode,
ErrorDialogType,
+348 -83
View File
@@ -1,6 +1,7 @@
import { getJsonHeaders } from '$lib/utils/api-headers';
import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers';
import { formatAttachmentText } from '$lib/utils/formatters';
import { isAbortError } from '$lib/utils/abort';
import { streamIdentity } from '$lib/utils/stream-identity';
import {
ATTACHMENT_LABEL_PDF_FILE,
ATTACHMENT_LABEL_MCP_PROMPT,
@@ -13,7 +14,10 @@ import {
CONTROL_ACTION,
SSE_LINE_SEPARATOR,
SSE_DATA_PREFIX,
SSE_DONE_MARKER
SSE_DONE_MARKER,
STREAM_VISIBILITY_KICK_MS,
STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX,
API_STREAM
} from '$lib/constants';
import {
AttachmentType,
@@ -21,12 +25,14 @@ import {
FileTypeAudio,
MessageRole,
MimeTypeAudio,
ReasoningFormat
ReasoningFormat,
StreamConnectionState
} from '$lib/enums';
import type {
ApiChatMessageContentPart,
ApiChatMessageData,
ApiChatCompletionToolCall
ApiChatCompletionToolCall,
ApiStreamSession
} from '$lib/types/api';
import type {
AudioInputFormat,
@@ -54,6 +60,19 @@ function getAudioInputFormat(mimeType: string): AudioInputFormat {
return FileTypeAudio.MP3;
}
interface ResumableStreamState {
bytesReceived: number;
updatedAt: number;
// model frozen at POST time, lets a reload rebuild the exact conv::model identity the
// server keyed the session under. null when the POST carried no explicit model
model?: string | null;
}
function streamStorageKey(conversationId: string): string {
return STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX + conversationId;
}
export class ChatService {
/**
*
@@ -128,6 +147,7 @@ export class ChatService {
onChunk,
onComplete,
onError,
onConnectionState,
onReasoningChunk,
onToolCallChunk,
onModel,
@@ -312,9 +332,16 @@ export class ChatService {
}
try {
const headers: Record<string, string> = { ...getJsonHeaders() };
// tag streaming requests with the conversation id, this single header is the opt in for the
// server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit
// model the ::model suffix keeps the per model session distinct
if (stream && conversationId) {
headers['X-Conversation-Id'] = streamIdentity(conversationId, options.model);
}
const response = await fetch(API_CHAT.COMPLETIONS, {
method: 'POST',
headers: getJsonHeaders(),
headers,
body: JSON.stringify(requestBody),
signal
});
@@ -341,7 +368,9 @@ export class ChatService {
onCompletionId,
onTimings,
conversationId,
signal
signal,
onConnectionState,
options.model
);
return;
@@ -473,6 +502,116 @@ export class ChatService {
* @param excludeReasoning - Whether to strip reasoning content (should match excludeReasoningFromContext setting)
* @param signal - Optional AbortSignal to cancel the pre-encode request
*/
static async cancelServerStream(conversationId: string, model?: string | null): Promise<void> {
if (!conversationId) return;
try {
const id = streamIdentity(conversationId, model);
await fetch(`${API_STREAM.BASE}/${encodeURIComponent(id)}`, {
method: 'DELETE',
headers: getAuthHeaders()
});
} catch (e) {
console.warn('cancelServerStream failed:', e);
}
}
/**
* Pick the running session to splice into when discoverActiveStream lists candidates for a
* conversation. Finalized sessions are not candidates: their final content was already written
* to the DB by the original onComplete handler, so attaching to them would replay a buffer that
* may not match what the DB holds. A continue session's buffer holds only the appended deltas,
* not the pre continue prefix, so replaying it as a fresh generation would erase the original.
*
* Among running sessions we tie break on the most recent started_at, which covers the case of
* multiple inferences left running on the same conversation.
*/
static selectActiveStream(
sessions: ApiStreamSession[] | null | undefined
): ApiStreamSession | null {
if (!Array.isArray(sessions) || sessions.length === 0) {
return null;
}
const running = sessions.filter((s) => !s.is_done);
if (running.length === 0) {
return null;
}
return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best));
}
// persist the running byte count and the frozen model for a conversation, a later visit
// resumes the SSE replay at the right offset under the same conv::model identity
static saveStreamState(
conversationId: string,
bytesReceived: number,
model?: string | null
): void {
if (!conversationId) return;
try {
const state: ResumableStreamState = {
bytesReceived,
updatedAt: Date.now(),
model: model ?? null
};
localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state));
} catch {
// localStorage may be full or disabled, silently ignore
}
}
static getStreamState(conversationId: string): ResumableStreamState | null {
if (!conversationId) return null;
try {
const raw = localStorage.getItem(streamStorageKey(conversationId));
if (!raw) return null;
const parsed = JSON.parse(raw) as ResumableStreamState;
if (!parsed || typeof parsed.bytesReceived !== 'number') return null;
return parsed;
} catch {
return null;
}
}
static clearStreamState(conversationId: string): void {
if (!conversationId) return;
try {
localStorage.removeItem(streamStorageKey(conversationId));
} catch {
// nothing to do
}
}
/**
* Rebuild the stream identity for a resume. The model persisted at POST time wins, including a
* stored null which means the POST carried no explicit model so the identity stays the bare conv
* id. Only fall back to the caller supplied current model when nothing was persisted.
*/
static resumeStreamIdentity(
conversationId: string,
state: ResumableStreamState | null,
fallbackModel: string | null
): string {
const model = state && state.model !== undefined ? state.model : fallbackModel;
return streamIdentity(conversationId, model);
}
/**
* Reconnect to an interrupted stream for this conversation. Returns the fetch Response so the
* existing SSE parser drains it like a fresh stream. The server returns 200 on success, 404 if
* no session exists for the conv_id, and 400 if the offset is below the dropped prefix.
*/
static async resumeStream(
conversationId: string,
signal?: AbortSignal,
model?: string | null
): Promise<Response | null> {
if (!conversationId) return null;
const state = ChatService.getStreamState(conversationId);
const from = state?.bytesReceived ?? 0;
const id = streamIdentity(conversationId, model);
const url = `${API_STREAM.BASE}/${encodeURIComponent(id)}?from=${from}`;
return await fetch(url, { method: 'GET', signal, headers: getAuthHeaders() });
}
static async preEncode(
messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[],
model?: string | null,
@@ -557,7 +696,7 @@ export class ChatService {
* @returns {Promise<void>} Promise that resolves when streaming is complete
* @throws {Error} if the stream cannot be read or parsed
*/
private static async handleStreamResponse(
static async handleStreamResponse(
response: Response,
onChunk?: (chunk: string) => void,
onComplete?: (
@@ -573,15 +712,34 @@ export class ChatService {
onCompletionId?: (id: string) => void,
onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void,
conversationId?: string,
abortSignal?: AbortSignal
abortSignal?: AbortSignal,
onConnectionState?: (state: StreamConnectionState) => void,
streamModel?: string | null
): Promise<void> {
const reader = response.body?.getReader();
let reader = response.body?.getReader();
if (!reader) {
throw new Error('No response body');
}
const decoder = new TextDecoder();
// bytesParsed is the absolute server side buffer offset of the next byte to parse
// segmentStartOffset is the absolute offset where the current reader started, reset on resume
// segmentBytesRead is wire bytes read by the current reader
let bytesParsed = 0;
let segmentStartOffset = 0;
let segmentBytesRead = 0;
let lastByteAt = Date.now();
// each resume must produce at least one byte to be retried again
// if a resume returns 200 but yields nothing, we abandon
// since the session has a bounded size, the total number of retries is bounded by construction
let madeProgress = true;
const encoder = new TextEncoder();
if (conversationId) {
ChatService.saveStreamState(conversationId, 0, streamModel);
}
onConnectionState?.(StreamConnectionState.STREAMING);
let decoder = new TextDecoder();
let aggregatedContent = '';
let fullReasoningContent = '';
let aggregatedToolCalls: ApiChatCompletionToolCall[] = [];
@@ -633,88 +791,184 @@ export class ChatService {
}
};
const onVisibilityChange = () => {
if (typeof document === 'undefined') return;
if (document.visibilityState !== 'visible') return;
if (streamFinished) return;
if (!conversationId) return;
// the bytes have been quiet for too long, the OS likely killed the socket
// kicking the reader unblocks reader.read with done=true so the outer loop can resume
if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) {
reader!.cancel().catch(() => {});
}
};
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', onVisibilityChange);
}
try {
let chunk = '';
// outer loop drives the resume cycle, swaps reader on premature end of stream
while (true) {
if (abortSignal?.aborted) break;
while (true) {
if (abortSignal?.aborted) break;
const { done, value } = await reader.read();
if (done)
{
streamFinished = true;
let done: boolean;
let value: Uint8Array | undefined;
try {
const r = await reader.read();
done = r.done;
value = r.value;
} catch (readErr) {
// reader.read() rejects with TypeError when the underlying connection drops
// instead of just resolving with done=true. treat it like done so the outer
// loop swaps reader via the resume path
if (isAbortError(readErr)) {
throw readErr;
}
console.warn('reader.read() rejected, treating as premature end:', readErr);
done = true;
value = undefined;
}
if (done)
{
streamFinished = true;
break;
}
if (abortSignal?.aborted) break;
if (value && value.byteLength > 0) {
segmentBytesRead += value.byteLength;
lastByteAt = Date.now();
if (!madeProgress) {
madeProgress = true;
onConnectionState?.(StreamConnectionState.STREAMING);
}
}
chunk += decoder.decode(value, { stream: true });
const lines = chunk.split(SSE_LINE_SEPARATOR);
chunk = lines.pop() || '';
// the persisted offset must point right after the last fully parsed line,
// the trailing `chunk` is partial bytes still waiting for a newline
if (conversationId) {
const tailBytes = encoder.encode(chunk).byteLength;
bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes;
ChatService.saveStreamState(conversationId, bytesParsed, streamModel);
}
for (const line of lines) {
if (abortSignal?.aborted) break;
if (line.startsWith(SSE_DATA_PREFIX)) {
const data = line.slice(SSE_DATA_PREFIX.length).trim();
if (data === SSE_DONE_MARKER) {
streamFinished = true;
continue;
}
try {
const parsed: ApiChatCompletionStreamChunk = JSON.parse(data);
const choice = parsed.choices?.[0];
const content = choice?.delta?.content;
const reasoningContent = choice?.delta?.reasoning_content;
const toolCalls = choice?.delta?.tool_calls;
const timings = parsed.timings;
const promptProgress = parsed.prompt_progress;
const chunkModel = ChatService.extractModelName(parsed);
if (chunkModel && !modelEmitted) {
modelEmitted = true;
onModel?.(chunkModel);
}
if (parsed.id && !idEmitted) {
idEmitted = true;
onCompletionId?.(parsed.id);
}
if (promptProgress) {
ChatService.notifyTimings(undefined, promptProgress, onTimings);
}
if (timings) {
ChatService.notifyTimings(timings, promptProgress, onTimings);
lastTimings = timings;
}
if (content) {
finalizeOpenToolCallBatch();
aggregatedContent += content;
if (!abortSignal?.aborted) {
onChunk?.(content);
}
}
if (reasoningContent) {
finalizeOpenToolCallBatch();
fullReasoningContent += reasoningContent;
if (!abortSignal?.aborted) {
onReasoningChunk?.(reasoningContent);
}
}
processToolCallDelta(toolCalls);
} catch (e) {
console.error('Error parsing JSON chunk:', e);
}
}
}
if (abortSignal?.aborted) break;
if (streamFinished) break;
}
// inner reader done, decide whether to try a resume
if (abortSignal?.aborted) break;
if (streamFinished) break;
if (!conversationId) break;
if (!madeProgress) {
onConnectionState?.(StreamConnectionState.LOST);
onError?.(new Error('Stream resume produced no new bytes, giving up'));
break;
}
onConnectionState?.(StreamConnectionState.RESUMING);
madeProgress = false;
// the server resends starting at bytesParsed, discard any partial line we held, it
// will be retransmitted from a clean line boundary. reuse the frozen model, not the
// live dropdown
const resumeResp = await ChatService.resumeStream(
conversationId,
abortSignal,
streamModel
).catch(() => null);
// an abort landing during the resume request is intentional, not a lost connection
if (abortSignal?.aborted) break;
chunk += decoder.decode(value, { stream: true });
const lines = chunk.split(SSE_LINE_SEPARATOR);
chunk = lines.pop() || '';
for (const line of lines) {
if (abortSignal?.aborted) break;
if (line.startsWith(SSE_DATA_PREFIX)) {
const data = line.slice(SSE_DATA_PREFIX.length).trim();
if (data === SSE_DONE_MARKER) {
streamFinished = true;
continue;
}
try {
const parsed: ApiChatCompletionStreamChunk = JSON.parse(data);
const choice = parsed.choices?.[0];
const content = choice?.delta?.content;
const reasoningContent = choice?.delta?.reasoning_content;
const toolCalls = choice?.delta?.tool_calls;
const timings = parsed.timings;
const promptProgress = parsed.prompt_progress;
const chunkModel = ChatService.extractModelName(parsed);
if (chunkModel && !modelEmitted) {
modelEmitted = true;
onModel?.(chunkModel);
}
if (parsed.id && !idEmitted) {
idEmitted = true;
onCompletionId?.(parsed.id);
}
if (promptProgress) {
ChatService.notifyTimings(undefined, promptProgress, onTimings);
}
if (timings) {
ChatService.notifyTimings(timings, promptProgress, onTimings);
lastTimings = timings;
}
if (content) {
finalizeOpenToolCallBatch();
aggregatedContent += content;
if (!abortSignal?.aborted) {
onChunk?.(content);
}
}
if (reasoningContent) {
finalizeOpenToolCallBatch();
fullReasoningContent += reasoningContent;
if (!abortSignal?.aborted) {
onReasoningChunk?.(reasoningContent);
}
}
processToolCallDelta(toolCalls);
} catch (e) {
console.error('Error parsing JSON chunk:', e);
}
}
if (!resumeResp || resumeResp.status !== 200) {
onConnectionState?.(StreamConnectionState.LOST);
onError?.(new Error('Stream connection lost and could not be resumed'));
break;
}
const newReader = resumeResp.body?.getReader();
if (!newReader) break;
if (abortSignal?.aborted) break;
try {
reader.releaseLock();
} catch {
/* ignore */
}
reader = newReader;
decoder = new TextDecoder();
chunk = '';
segmentStartOffset = bytesParsed;
segmentBytesRead = 0;
lastByteAt = Date.now();
}
if (abortSignal?.aborted) return;
@@ -722,6 +976,10 @@ export class ChatService {
if (streamFinished) {
finalizeOpenToolCallBatch();
if (conversationId) {
ChatService.clearStreamState(conversationId);
}
const finalToolCalls =
aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined;
@@ -739,7 +997,14 @@ export class ChatService {
throw err;
} finally {
reader.releaseLock();
if (typeof document !== 'undefined') {
document.removeEventListener('visibilitychange', onVisibilityChange);
}
try {
reader.releaseLock();
} catch {
/* ignore */
}
}
}
+13 -12
View File
@@ -628,19 +628,20 @@ export class MCPService {
);
const runtimeErrorHandler = (error: Error) => {
// Ignore errors that are expected when the SDK's transport is closed,
// or when connecting to servers that don't support SSE (stateless-only
// endpoints returning 405). The SDK wraps the original AbortError in
// a new Error with the message "SSE stream disconnected: AbortError",
// and also produces "Cannot cancel a stream locked by a reader".
// DOMException is thrown by the browser when aborting fetch requests.
const msg = error.message || String(error);
// the SDK reports any post initialize error here, including the abort we trigger
// ourselves on the next health check cycle, on tab unload, or on server teardown.
// these are lifecycle aborts, not actionable errors, so we keep them out of the red console.
// the SDK wraps the original AbortError in a generic Error like
// "SSE stream disconnected: AbortError: The operation was aborted."
// which isAbortError cannot recognize by name alone, so we also pattern match on the message
if (isAbortError(error)) {
return;
}
const msg = error?.message ?? '';
if (
error.name === 'AbortError' ||
error instanceof DOMException ||
msg.includes('SSE stream disconnected') ||
msg.includes('stream locked by a reader') ||
msg.includes('The operation was aborted')
/SSE stream disconnected:.*AbortError/i.test(msg) ||
/AbortError: .*aborted/i.test(msg) ||
/stream locked by a reader/i.test(msg)
) {
return;
}
+1 -1
View File
@@ -614,7 +614,7 @@ class AgenticStore {
throw error;
}
},
undefined,
conversationId,
signal
);
+455 -15
View File
@@ -11,9 +11,11 @@
* @see ChatService in services/chat.service.ts for API operations
*/
import { SvelteMap } from 'svelte/reactivity';
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
import { DatabaseService } from '$lib/services/database.service';
import { ChatService } from '$lib/services/chat.service';
import { streamIdentity } from '$lib/utils/stream-identity';
import { getAuthHeaders } from '$lib/utils/api-headers';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { config } from '$lib/stores/settings.svelte';
import { agenticStore } from '$lib/stores/agentic.svelte';
@@ -49,10 +51,17 @@ import type {
import type {
ApiChatMessageData,
ApiProcessingState,
ApiStreamSession,
DatabaseMessage,
DatabaseMessageExtra
} from '$lib/types';
import { ContinueIntentKind, ErrorDialogType, MessageRole, MessageType } from '$lib/enums';
import {
ContinueIntentKind,
ErrorDialogType,
MessageRole,
MessageType,
StreamConnectionState
} from '$lib/enums';
interface ConversationStateEntry {
lastAccessed: number;
@@ -65,9 +74,25 @@ class ChatStore {
isLoading = $state(false);
// true while the active conversation streams reasoning content but no visible content yet
isReasoning = $state(false);
// resumable stream connection state for the active conversation
// streaming -> bytes flowing normally, resuming -> waiting on /v1/stream/:id reconnect, lost -> unrecoverable
streamConnectionState = $state<StreamConnectionState>(StreamConnectionState.STREAMING);
chatLoadingStates = new SvelteMap<string, boolean>();
chatReasoningStates = new SvelteMap<string, boolean>();
chatStreamingStates = new SvelteMap<string, { response: string; messageId: string }>();
chatStreamingStates = new SvelteMap<
string,
{ response: string; messageId: string; model?: string | null }
>();
// convs that the backend reports as having a running session, populated by the global sync
// at app mount and on visibilitychange. it does not overlap with chatLoadingStates which
// tracks inferences driven by this browser, both are unioned to feed the sidebar spinners
private remoteRunningConvs = new SvelteSet<string>();
// per conv attach lifecycle, used to derive the global streaming flag without flipping it
// off when one conv finishes while another is still streaming. mirrors chatLoadingStates
// in scope but tracks the attach + tee replay path specifically
private attachingConvs = new SvelteSet<string>();
// in-flight discoverActiveStream guard, keyed by conv id
private discoveringConvs = new SvelteSet<string>();
private abortControllers = new SvelteMap<string, AbortController>();
private preEncodeAbortController: AbortController | null = null;
private processingStates = new SvelteMap<string, ApiProcessingState | null>();
@@ -98,6 +123,11 @@ class ChatStore {
this.chatLoadingStates.delete(convId);
if (convId === conversationsStore.activeConversation?.id) this.isLoading = false;
this.setChatReasoning(convId, false);
// the local pipe is the authoritative observer of session end: when it finishes (clean
// onComplete or explicit Stop), the backend session is finalized too, so we drop the
// sidebar hint for this conv right away instead of waiting for the next visibilitychange
// snapshot. without this the spinner ghosts until the user toggles the tab
this.remoteRunningConvs.delete(convId);
}
}
@@ -110,9 +140,18 @@ class ChatStore {
if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false;
}
}
private setChatStreaming(convId: string, response: string, messageId: string): void {
private setChatStreaming(
convId: string,
response: string,
messageId: string,
model?: string | null
): void {
this.touchConversationState(convId);
this.chatStreamingStates.set(convId, { response, messageId });
this.chatStreamingStates.set(convId, {
response,
messageId,
model: model ?? this.chatStreamingStates.get(convId)?.model
});
if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response;
}
private clearChatStreaming(convId: string): void {
@@ -137,6 +176,314 @@ class ChatStore {
}
}
}
/**
* Server side stream discovery, split in three pieces:
*
* probeServerStream(convId) -> hits POST /v1/streams/lookup with the conv id, returns the session to attach
* to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything.
*
* attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream
* from byte 0, finds the assistant slot to splice into (creates a placeholder if the conv has
* no assistant message yet, for cross device or fresh local DB cases), and pipes the SSE bytes
* into the message via handleStreamResponse.
*
* discoverActiveStream(convId) -> probe + attach in one call. Used by callers that do not need
* to overlap the probe with other async work.
*
* The mount of the chat page in +page.svelte calls probeServerStream in parallel with
* loadConversation, then attachServerStream once both have settled. This gives the earliest
* possible time to spinner and avoids racing against an empty activeMessages array.
*/
async probeServerStream(convId: string): Promise<ApiStreamSession | null> {
if (!convId) return null;
let listResp: Response;
try {
// POST the one conv id we are probing
listResp = await fetch(`./v1/streams/lookup`, {
method: 'POST',
headers: { ...getAuthHeaders(), 'Content-Type': 'application/json' },
body: JSON.stringify({ conversation_ids: [convId] })
});
} catch (e) {
console.warn('probeServerStream fetch failed:', e);
return null;
}
if (!listResp.ok) {
console.warn(`probeServerStream got HTTP ${listResp.status} for conv ${convId}`);
return null;
}
let sessions: ApiStreamSession[];
try {
sessions = (await listResp.json()) as ApiStreamSession[];
} catch (e) {
console.warn('probeServerStream JSON parse failed:', e);
return null;
}
return ChatService.selectActiveStream(sessions);
}
async attachServerStream(convId: string, streamId?: string): Promise<void> {
if (!convId) return;
if (this.chatStreamingStates.has(convId)) return;
// flip the spinner immediately, the user sees activity as soon as the conv becomes active.
// the global isStreamingActive flag is derived from attachingConvs.size, so adding here
// turns it on, and removing in unlock only turns it off when this is the last attach
this.setChatLoading(convId, true);
this.attachingConvs.add(convId);
this.setStreamingActive(true);
// only set the active processing conv if we are looking at it, otherwise a background
// attach would steal the indicator from the conv the user is currently viewing
if (convId === conversationsStore.activeConversation?.id) {
this.setActiveProcessingConversation(convId);
}
const unlock = () => {
this.attachingConvs.delete(convId);
// flip the global flag off only when no other conv is still attaching
if (this.attachingConvs.size === 0) {
this.setStreamingActive(false);
}
this.setChatLoading(convId, false);
this.clearChatStreaming(convId);
};
// fetch the replay stream from byte 0, rebuild the assistant message from scratch.
// resolve the server side identity, fall back to streamIdentity when the caller does not
// pass a streamId. probeServerStream returns the full id (with ::model suffix when present)
const id = streamId || streamIdentity(convId, selectedModelName());
let response: Response;
try {
response = await fetch(`./v1/stream/${encodeURIComponent(id)}?from=0`, {
headers: getAuthHeaders()
});
} catch (e) {
console.error('attachServerStream replay fetch failed:', e);
unlock();
return;
}
if (!response.ok) {
console.warn(`attachServerStream replay got HTTP ${response.status} for conv ${convId}`);
unlock();
return;
}
// load the target conversation messages by id, not via the active store. when multiple
// attaches run in parallel the active store may reflect another conv and writing through
// its index mixes content across convs (CoT flicker, message bleed). by going through the
// DB we stay isolated, and only mirror into the active store when the attached conv is
// the one currently displayed
let messages: DatabaseMessage[];
try {
messages = await DatabaseService.getConversationMessages(convId);
} catch (e) {
console.error('attachServerStream load messages failed:', e);
unlock();
return;
}
// locate the slot to splice into, create a placeholder assistant message if there is none.
// we use the conv-scoped findLastAssistantIdx helpers, they only depend on the array
let targetIdx = this.findLastAssistantIdx(messages);
if (targetIdx === -1) {
const lastUserIdx = this.findLastUserIdx(messages);
if (lastUserIdx === -1) {
console.warn(
`attachServerStream: conv ${convId} has no user or assistant message, cannot splice`
);
unlock();
return;
}
try {
const placeholder = await DatabaseService.createMessageBranch(
{
convId,
role: MessageRole.ASSISTANT,
content: '',
type: MessageType.TEXT,
timestamp: Date.now(),
parent: messages[lastUserIdx].id,
children: [],
toolCalls: ''
} as Omit<DatabaseMessage, 'id'>,
messages[lastUserIdx].id
);
messages = [...messages, placeholder];
targetIdx = messages.length - 1;
// only push into the active store when this conv is the one displayed right now
if (convId === conversationsStore.activeConversation?.id) {
conversationsStore.addMessageToActive(placeholder);
}
} catch (e) {
console.error('attachServerStream placeholder creation failed:', e);
unlock();
return;
}
}
if (targetIdx === -1) {
unlock();
return;
}
const targetMessage = messages[targetIdx];
const targetMessageId = targetMessage.id;
// when the assistant slot already has content, the running session is a continue or
// another append flow and its buffer holds only the appended deltas. preserve the prefix
// and let the replay add to it. when the slot is empty the session buffer holds the whole
// message so we wipe and rebuild from byte 0
const existingContent = targetMessage.content ?? '';
const existingReasoning = targetMessage.reasoningContent ?? '';
const isAppendMode = existingContent.length > 0;
// helper: write to the active store only when the attached conv is currently displayed.
// the lookup by message id is robust to reordering of activeMessages, two parallel attaches
// can no longer step on each other's indices
const writeActive = (updates: Partial<DatabaseMessage>) => {
if (convId !== conversationsStore.activeConversation?.id) {
return;
}
const liveIdx = conversationsStore.findMessageIndex(targetMessageId);
if (liveIdx === -1) return;
conversationsStore.updateMessageAtIndex(liveIdx, updates);
};
if (!isAppendMode) {
writeActive({ content: '', reasoningContent: undefined });
}
// extract the model suffix, the resume calls in handleStreamResponse must reuse the model
// the session was tagged with, not the live dropdown
const sepIdx = id.indexOf('::');
const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2);
this.setChatStreaming(convId, existingContent, targetMessageId, attachedModel);
const abortController = this.getOrCreateAbortController(convId);
let streamedContent = '';
let streamedReasoningContent = '';
const cleanup = () => {
unlock();
this.setProcessingState(convId, null);
};
try {
await ChatService.handleStreamResponse(
response,
(chunk: string) => {
streamedContent += chunk;
const displayed = isAppendMode ? existingContent + streamedContent : streamedContent;
writeActive({ content: displayed });
this.setChatStreaming(convId, displayed, targetMessageId);
},
async (
finalContent?: string,
reasoningContent?: string,
timings?: ChatMessageTimings,
toolCalls?: string
) => {
const streamed = streamedContent || finalContent || '';
const streamedR = streamedReasoningContent || reasoningContent || '';
const content = isAppendMode ? existingContent + streamed : streamed;
const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR;
// the DB write is the source of truth, mirror to the active store only when
// the conv is currently displayed
await DatabaseService.updateMessage(targetMessageId, {
content,
reasoningContent: reasoning || undefined,
toolCalls: toolCalls || '',
timings
});
writeActive({
content,
reasoningContent: reasoning || undefined,
timings
});
cleanup();
},
(err: Error) => {
console.error('attachServerStream pipe error:', err);
cleanup();
},
(chunk: string) => {
streamedReasoningContent += chunk;
const displayed = isAppendMode
? existingReasoning + streamedReasoningContent
: streamedReasoningContent;
writeActive({ reasoningContent: displayed });
},
undefined,
undefined,
undefined,
undefined,
convId,
abortController.signal,
(connState: StreamConnectionState) => {
if (convId === conversationsStore.activeConversation?.id) {
this.streamConnectionState = connState;
}
},
attachedModel
);
} catch (e) {
console.error('attachServerStream pipe crashed:', e);
cleanup();
}
}
async discoverActiveStream(convId: string): Promise<void> {
if (!convId) return;
if (this.chatStreamingStates.has(convId)) return;
if (this.chatLoadingStates.get(convId)) return;
// concurrency guard: another discover may already be running for this conv (typical race
// between mount and visibilitychange on tab switch). a second concurrent fetch on the same
// /v1/stream/<id> would duplicate every byte into the DB message, this guard bounces it
if (this.discoveringConvs.has(convId)) return;
this.discoveringConvs.add(convId);
try {
// the model is frozen at POST time, rebuild the exact conv::model identity from the
// persisted state so the lookup key matches what the server stored. null means a single
// model conv with no ::suffix, only guess from the dropdown with no persisted state
const localState = ChatService.getStreamState(convId);
const streamId = ChatService.resumeStreamIdentity(convId, localState, selectedModelName());
// primary path: ask the server which sessions exist for this identity
const serverTarget = await this.probeServerStream(streamId);
if (serverTarget) {
// pass the full server side identity (may carry a ::model suffix) so the GET routes
// straight to the owning session, no probe or fan out
await this.attachServerStream(convId, serverTarget.conversation_id);
return;
}
// fallback: local state remembers an interrupted byte offset for this conv, the server may
// still have a live session matching that identity (we just lost the bytes mid stream). retry
// with the frozen identity, the server probe inside attachServerStream tells us if it exists
if (!localState) {
return;
}
await this.attachServerStream(convId, streamId);
// if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever
if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) {
ChatService.clearStreamState(convId);
}
} finally {
this.discoveringConvs.delete(convId);
}
}
private findLastAssistantIdx(messages: DatabaseMessage[]): number {
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i].role === MessageRole.ASSISTANT) return i;
}
return -1;
}
private findLastUserIdx(messages: DatabaseMessage[]): number {
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i].role === MessageRole.USER) return i;
}
return -1;
}
clearUIState(): void {
this.isLoading = false;
@@ -265,13 +612,83 @@ class ChatStore {
}
getAllLoadingChats(): string[] {
return Array.from(this.chatLoadingStates.keys());
// union of local (this browser is piping) and remote (backend reports a running session
// for this conv but no local pipe yet) sources. the sidebar shows one spinner per entry
const out = new SvelteSet<string>(this.chatLoadingStates.keys());
for (const id of this.remoteRunningConvs) {
out.add(id);
}
return Array.from(out);
}
getAllStreamingChats(): string[] {
return Array.from(this.chatStreamingStates.keys());
}
/**
* Resync the remote running convs set from the backend. Called by the layout at mount and on
* visibilitychange, no polling. A snapshot semantic: the set is replaced wholesale, stale entries
* for sessions that finalized while the browser was elsewhere are dropped naturally.
*/
async syncRemoteRunningStreams(): Promise<void> {
// the conversations store loads from IndexedDB asynchronously, the +layout onMount caller
// fires before that finishes. read ids straight from the DB so the result does not depend
// on the store init race, and the sidebar spinners light up at first paint for every conv
// the user owns even if it has not been hydrated into the store yet
let ids: string[];
try {
const all = await DatabaseService.getAllConversations();
ids = all.map((c) => c.id).filter((id) => !!id);
} catch (e) {
console.warn('syncRemoteRunningStreams DB read failed:', e);
return;
}
// only ask about conv ids the user already owns
if (ids.length === 0) {
for (const id of Array.from(this.remoteRunningConvs)) {
this.remoteRunningConvs.delete(id);
}
return;
}
// rebuild the frozen conv::model identity per conv so a session started with a model still
// matches. the server response is mapped back to the bare id below for the sidebar set
const lookupIds = ids.map((id) =>
ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null)
);
let sessions: ApiStreamSession[];
try {
const resp = await fetch('./v1/streams/lookup', {
method: 'POST',
headers: { ...getAuthHeaders(), 'Content-Type': 'application/json' },
body: JSON.stringify({ conversation_ids: lookupIds })
});
if (!resp.ok) return;
const body = (await resp.json()) as unknown;
if (!Array.isArray(body)) return;
sessions = body as ApiStreamSession[];
} catch (e) {
console.warn('syncRemoteRunningStreams fetch failed:', e);
return;
}
const running = new SvelteSet<string>();
for (const s of sessions) {
if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) {
// strip the optional ::model suffix, the sidebar set is keyed by the bare conv id
const sepIdx = s.conversation_id.indexOf('::');
const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx);
running.add(bareId);
}
}
for (const id of Array.from(this.remoteRunningConvs)) {
if (!running.has(id)) {
this.remoteRunningConvs.delete(id);
}
}
for (const id of running) {
this.remoteRunningConvs.add(id);
}
}
getChatStreamingPublic(convId: string): { response: string; messageId: string } | undefined {
return this.getChatStreaming(convId);
}
@@ -922,6 +1339,11 @@ class ChatStore {
onModel: streamCallbacks.onModel,
onCompletionId: streamCallbacks.onCompletionId,
onTimings: streamCallbacks.onTimings,
onConnectionState: (state: StreamConnectionState) => {
if (convId === conversationsStore.activeConversation?.id) {
this.streamConnectionState = state;
}
},
onComplete: async (
finalContent?: string,
reasoningContent?: string,
@@ -979,6 +1401,12 @@ class ChatStore {
async stopGenerationForChat(convId: string): Promise<void> {
await this.savePartialResponseIfNeeded(convId);
this.setStreamingActive(false);
// tell the server to stop the generation, not just drop the HTTP socket. without this the
// detached drain keeps producing tokens until eos or max_tokens. use the frozen identity
// captured when the session started, not the live dropdown
const streamStateForStop = this.chatStreamingStates.get(convId);
const modelForStop = streamStateForStop?.model ?? selectedModelName();
void ChatService.cancelServerStream(convId, modelForStop);
this.abortRequest(convId);
this.setChatLoading(convId, false);
this.clearChatStreaming(convId);
@@ -1393,7 +1821,11 @@ class ChatStore {
const updateStreamingContent = (fullContent: string) => {
this.setChatStreaming(msg.convId, fullContent, msg.id);
conversationsStore.updateMessageAtIndex(idx, { content: fullContent });
// resolve the row by id on every write, switching to another conv mid continue makes
// this a no op instead of writing positionally into the now displayed conversation
conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), {
content: fullContent
});
};
const abortController = this.getOrCreateAbortController(msg.convId);
@@ -1403,6 +1835,11 @@ class ChatStore {
{
...this.getApiOptions(),
continueFinalMessage: true,
onConnectionState: (state: StreamConnectionState) => {
if (msg.convId === conversationsStore.activeConversation?.id) {
this.streamConnectionState = state;
}
},
onChunk: (chunk: string) => {
appendedContent += chunk;
hasReceivedContent = true;
@@ -1414,7 +1851,7 @@ class ChatStore {
hasReceivedContent = true;
// mark streaming state so a stop mid-thinking can persist the partial reasoning
this.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id);
conversationsStore.updateMessageAtIndex(idx, {
conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), {
reasoningContent: originalReasoning + appendedReasoning
});
this.setChatReasoning(msg.convId, true);
@@ -1455,7 +1892,7 @@ class ChatStore {
timings
});
conversationsStore.updateMessageAtIndex(idx, {
conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), {
content: fullContent,
reasoningContent: fullReasoning,
timestamp: Date.now(),
@@ -1477,11 +1914,14 @@ class ChatStore {
timestamp: Date.now()
});
conversationsStore.updateMessageAtIndex(idx, {
content: originalContent + appendedContent,
reasoningContent: originalReasoning + appendedReasoning || undefined,
timestamp: Date.now()
});
conversationsStore.updateMessageAtIndex(
conversationsStore.findMessageIndex(msg.id),
{
content: originalContent + appendedContent,
reasoningContent: originalReasoning + appendedReasoning || undefined,
timestamp: Date.now()
}
);
}
this.setChatLoading(msg.convId, false);
@@ -1498,7 +1938,7 @@ class ChatStore {
reasoningContent: originalReasoning + appendedReasoning || undefined,
timestamp: Date.now()
});
conversationsStore.updateMessageAtIndex(idx, {
conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), {
content: originalContent + appendedContent,
reasoningContent: originalReasoning + appendedReasoning || undefined,
timestamp: Date.now()
+15
View File
@@ -512,3 +512,18 @@ export interface ApiRouterModelsUnloadResponse {
success: boolean;
error?: string;
}
/**
* Entry returned by POST /v1/streams/lookup. The client passes the conv ids it owns in the body
* and the server returns one entry per matching live or recently completed background streaming
* session, keyed by conversation_id. The WebUI uses this at mount and on visibilitychange to
* populate sidebar spinners and to reattach to an ongoing inference for the active conversation.
* The server never lists ids the client did not ask about, so foreign random UUIDs stay private.
*/
export interface ApiStreamSession {
conversation_id: string;
is_done: boolean;
total_bytes: number;
started_at: number;
completed_at: number;
}
+2 -1
View File
@@ -34,7 +34,8 @@ export type {
ApiRouterModelsListResponse,
ApiRouterModelsUnloadRequest,
ApiRouterModelsUnloadResponse,
AudioInputFormat
AudioInputFormat,
ApiStreamSession
} from './api';
// Chat types
+4 -2
View File
@@ -4,9 +4,10 @@ import type { OpenAIToolDefinition } from './mcp';
import type { DatabaseMessageExtra } from './database';
import type {
ParameterSource,
ReasoningEffort,
SyncableParameterType,
SettingsFieldType
SettingsFieldType,
StreamConnectionState,
ReasoningEffort
} from '$lib/enums';
import type { Icon } from '@lucide/svelte';
import type { Component } from 'svelte';
@@ -119,6 +120,7 @@ export interface SettingsChatServiceOptions {
toolCalls?: string
) => void;
onError?: (error: Error) => void;
onConnectionState?: (state: StreamConnectionState) => void;
}
export type SettingsConfigType = typeof SETTING_CONFIG_DEFAULT & {
+23 -5
View File
@@ -6,6 +6,17 @@
* when needed (e.g., user stops generation, navigates away, etc.).
*/
// the standard DOMException name for a cancelled operation
const ABORT_ERROR_NAME = 'AbortError';
// browser specific TypeError messages emitted when a fetch reader is cut by page unload,
// navigation, or a transient network drop. functionally aborts, not actionable errors
const ABORT_LIKE_MESSAGE_PATTERNS = [
/input stream/i, // Firefox: stream cut at unload
/network connection was lost/i, // Safari: transient network drop
/load failed/i // Safari: page navigation during fetch
];
/**
* Throws an AbortError if the signal is aborted.
* Use this at the start of async operations to fail fast.
@@ -23,7 +34,7 @@
*/
export function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) {
throw new DOMException('Operation was aborted', 'AbortError');
throw new DOMException('Operation was aborted', ABORT_ERROR_NAME);
}
}
@@ -48,11 +59,18 @@ export function throwIfAborted(signal?: AbortSignal): void {
* ```
*/
export function isAbortError(error: unknown): boolean {
if (error instanceof DOMException && error.name === 'AbortError') {
if (error instanceof DOMException && error.name === ABORT_ERROR_NAME) {
return true;
}
if (error instanceof Error && error.name === 'AbortError') {
return true;
if (error instanceof Error) {
if (error.name === ABORT_ERROR_NAME) {
return true;
}
// these patterns are functionally aborts, keep them out of the red console
if (error instanceof TypeError) {
const msg = error.message ?? '';
if (ABORT_LIKE_MESSAGE_PATTERNS.some((re) => re.test(msg))) return true;
}
}
return false;
}
@@ -133,7 +151,7 @@ export async function withAbortSignal<T>(promise: Promise<T>, signal?: AbortSign
return new Promise<T>((resolve, reject) => {
const abortHandler = () => {
reject(new DOMException('Operation was aborted', 'AbortError'));
reject(new DOMException('Operation was aborted', ABORT_ERROR_NAME));
};
signal.addEventListener('abort', abortHandler, { once: true });
+13
View File
@@ -0,0 +1,13 @@
/**
* Build the conversation identity used by the server side replay buffer.
*
* The server identifies a stream session by a conversation id sent in the
* X-Conversation-Id header. When the user has explicitly picked a model the
* client appends ::modelName, so a per model session stays distinct and the
* router resolves the owning child through its conv_id -> model map.
*/
export function streamIdentity(conversationId: string, model?: string | null): string {
if (!conversationId) return '';
if (!model) return conversationId;
return `${conversationId}::${model}`;
}
@@ -4,7 +4,7 @@
import { afterNavigate } from '$app/navigation';
import { DialogModelNotAvailable } from '$lib/components/app';
import { APP_NAME, ROUTES } from '$lib/constants';
import { chatStore, isLoading } from '$lib/stores/chat.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import { conversationsStore, activeConversation } from '$lib/stores/conversations.svelte';
import { modelsStore, modelOptions } from '$lib/stores/models.svelte';
@@ -83,7 +83,7 @@
// Skip loading if this conversation is already active (e.g., just created)
if (activeConversation()?.id === chatId) {
// Still handle URL params even if conversation is active
void chatStore.discoverActiveStream(chatId);
if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) {
handleUrlParams();
}
@@ -92,35 +92,33 @@
(async () => {
const success = await conversationsStore.loadConversation(chatId);
if (success) {
chatStore.syncLoadingStateForChat(chatId);
// Handle URL params after conversation is loaded
if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) {
await handleUrlParams();
}
} else {
if (!success) {
await goto(ROUTES.START);
return;
}
chatStore.syncLoadingStateForChat(chatId);
// server probe (with localStorage fallback) and attach
await chatStore.discoverActiveStream(chatId);
if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) {
await handleUrlParams();
}
})();
}
});
$effect(() => {
if (typeof window !== 'undefined') {
const handleBeforeUnload = () => {
if (isLoading()) {
console.log('Page unload detected while streaming - aborting stream');
chatStore.stopGeneration();
}
};
if (typeof window === 'undefined' || typeof document === 'undefined') return;
window.addEventListener('beforeunload', handleBeforeUnload);
return () => {
window.removeEventListener('beforeunload', handleBeforeUnload);
};
}
// when the tab comes back to the foreground, re-run discovery to catch any race
// where the initial mount probe missed an active session
const onVisibility = () => {
if (document.visibilityState !== 'visible') return;
if (!chatId) return;
void chatStore.discoverActiveStream(chatId);
};
document.addEventListener('visibilitychange', onVisibility);
return () => document.removeEventListener('visibilitychange', onVisibility);
});
</script>
+12
View File
@@ -11,6 +11,7 @@
import { PwaMetaTags, PwaRefreshAlert } from '$lib/components/pwa';
import { pwaAssetsHead } from 'virtual:pwa-assets/head';
import { chatStore } from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import * as Tooltip from '$lib/components/ui/tooltip';
import { isRouterMode, serverStore } from '$lib/stores/server.svelte';
@@ -154,8 +155,18 @@
onMount(() => {
updateFavicon();
// snapshot of every backend running stream on first load, populates the sidebar spinners
// so the user sees each conv that has a live inference, even ones not opened yet
void chatStore.syncRemoteRunningStreams();
});
// refresh that snapshot when the tab returns to the foreground, a stream may have advanced
// or ended while it was hidden. snapshot only, no polling
function handleVisibilityChange() {
if (document.visibilityState !== 'visible') return;
void chatStore.syncRemoteRunningStreams();
}
$effect(() => {
void theme.isSystemDark;
@@ -280,6 +291,7 @@
</svelte:head>
<svelte:window onkeydown={handleKeydown} bind:innerHeight bind:innerWidth />
<svelte:document onvisibilitychange={handleVisibilityChange} />
<Tooltip.Provider delayDuration={TOOLTIP_DELAY_DURATION}>
<div class="flex flex-col md:flex-row">
+56
View File
@@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest';
import { isAbortError } from '$lib/utils/abort';
describe('isAbortError', () => {
it('returns false for null, undefined and non-error values', () => {
expect(isAbortError(null)).toBe(false);
expect(isAbortError(undefined)).toBe(false);
expect(isAbortError('string error')).toBe(false);
expect(isAbortError({ name: 'AbortError' })).toBe(false);
expect(isAbortError(42)).toBe(false);
});
it('returns true for DOMException with AbortError name', () => {
const err = new DOMException('Operation was aborted', 'AbortError');
expect(isAbortError(err)).toBe(true);
});
it('returns true for plain Error with AbortError name', () => {
const err = new Error('aborted');
err.name = 'AbortError';
expect(isAbortError(err)).toBe(true);
});
it('returns false for unrelated Error instances', () => {
expect(isAbortError(new Error('something failed'))).toBe(false);
expect(isAbortError(new TypeError('not related'))).toBe(false);
expect(isAbortError(new RangeError('out of range'))).toBe(false);
});
it('recognizes Firefox TypeError "Error in input stream" emitted at page unload', () => {
expect(isAbortError(new TypeError('Error in input stream'))).toBe(true);
expect(isAbortError(new TypeError('TypeError: Error in input stream'))).toBe(true);
});
it('recognizes Safari "The network connection was lost" during transient drop', () => {
expect(isAbortError(new TypeError('The network connection was lost.'))).toBe(true);
});
it('recognizes Safari "Load failed" during page navigation', () => {
expect(isAbortError(new TypeError('Load failed'))).toBe(true);
});
it('does NOT recognize generic TypeError messages as aborts', () => {
// matching too broadly would hide real bugs, the predicate must stay conservative
expect(isAbortError(new TypeError('Failed to fetch'))).toBe(false);
expect(isAbortError(new TypeError('Cannot read property of undefined'))).toBe(false);
expect(isAbortError(new TypeError('NetworkError when attempting to fetch resource'))).toBe(
false
);
});
it('is case insensitive on the matched substrings', () => {
expect(isAbortError(new TypeError('error in INPUT STREAM'))).toBe(true);
expect(isAbortError(new TypeError('the network connection WAS LOST'))).toBe(true);
});
});
@@ -0,0 +1,74 @@
import { describe, expect, it } from 'vitest';
import { ChatService } from '$lib/services/chat.service';
import type { ApiStreamSession } from '$lib/types';
function makeSession(overrides: Partial<ApiStreamSession>): ApiStreamSession {
return {
conversation_id: 'conv',
is_done: true,
total_bytes: 0,
started_at: 0,
completed_at: 0,
...overrides
};
}
describe('selectActiveStream', () => {
it('returns null on empty input', () => {
expect(ChatService.selectActiveStream([])).toBeNull();
});
it('returns null on null or undefined input', () => {
expect(ChatService.selectActiveStream(null)).toBeNull();
expect(ChatService.selectActiveStream(undefined)).toBeNull();
});
it('returns the single session when it is running', () => {
const s = makeSession({ conversation_id: 'only', is_done: false, started_at: 42 });
expect(ChatService.selectActiveStream([s])).toBe(s);
});
it('returns null when the single session is finalized', () => {
const s = makeSession({ conversation_id: 'only', is_done: true, started_at: 42 });
expect(ChatService.selectActiveStream([s])).toBeNull();
});
it('prefers a still running session over a finalized one regardless of started_at', () => {
const finalized = makeSession({ conversation_id: 'old', is_done: true, started_at: 1000 });
const running = makeSession({ conversation_id: 'new', is_done: false, started_at: 10 });
expect(ChatService.selectActiveStream([finalized, running])?.conversation_id).toBe('new');
expect(ChatService.selectActiveStream([running, finalized])?.conversation_id).toBe('new');
});
it('among running sessions, picks the most recently started one', () => {
const a = makeSession({ conversation_id: 'a', is_done: false, started_at: 100 });
const b = makeSession({ conversation_id: 'b', is_done: false, started_at: 200 });
const c = makeSession({ conversation_id: 'c', is_done: false, started_at: 150 });
expect(ChatService.selectActiveStream([a, b, c])?.conversation_id).toBe('b');
expect(ChatService.selectActiveStream([c, a, b])?.conversation_id).toBe('b');
});
it('returns null when all sessions are finalized, the DB already holds the content', () => {
const a = makeSession({ conversation_id: 'a', is_done: true, started_at: 10 });
const b = makeSession({ conversation_id: 'b', is_done: true, started_at: 30 });
const c = makeSession({ conversation_id: 'c', is_done: true, started_at: 20 });
expect(ChatService.selectActiveStream([a, b, c])).toBeNull();
});
it('keeps the first match on ties when both are running with identical started_at', () => {
// reduce visits left to right, the initial accumulator stays unless a strictly greater value appears
const a = makeSession({ conversation_id: 'first', is_done: false, started_at: 50 });
const b = makeSession({ conversation_id: 'second', is_done: false, started_at: 50 });
expect(ChatService.selectActiveStream([a, b])?.conversation_id).toBe('first');
});
it('handles a typical realistic mix: two finalized old, one freshly running, one freshly finalized', () => {
const old1 = makeSession({ conversation_id: 'old1', is_done: true, started_at: 100 });
const old2 = makeSession({ conversation_id: 'old2', is_done: true, started_at: 200 });
const freshFin = makeSession({ conversation_id: 'freshFin', is_done: true, started_at: 500 });
const running = makeSession({ conversation_id: 'running', is_done: false, started_at: 400 });
expect(ChatService.selectActiveStream([old1, old2, freshFin, running])?.conversation_id).toBe(
'running'
);
});
});
+128
View File
@@ -0,0 +1,128 @@
import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
// node env unit project has no DOM, install a minimal localStorage backed by a Map
beforeAll(() => {
const store = new Map<string, string>();
const polyfill: Storage = {
get length() {
return store.size;
},
clear: () => store.clear(),
getItem: (k) => (store.has(k) ? store.get(k)! : null),
key: (i) => Array.from(store.keys())[i] ?? null,
removeItem: (k) => {
store.delete(k);
},
setItem: (k, v) => {
store.set(k, String(v));
}
};
(globalThis as unknown as { localStorage: Storage }).localStorage = polyfill;
});
import { ChatService } from '$lib/services/chat.service';
import { STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX } from '$lib/constants';
describe('ChatService stream resume', () => {
beforeEach(() => {
localStorage.clear();
});
afterEach(() => {
localStorage.clear();
});
it('returns null when no state exists for the conversation', () => {
expect(ChatService.getStreamState('conv-a')).toBeNull();
});
it('saves and reads back the byte count', () => {
ChatService.saveStreamState('conv-a', 4242);
const got = ChatService.getStreamState('conv-a');
expect(got).not.toBeNull();
expect(got!.bytesReceived).toBe(4242);
expect(typeof got!.updatedAt).toBe('number');
});
it('overwrites the previous byte count on a new save for the same conversation', () => {
ChatService.saveStreamState('conv-a', 100);
ChatService.saveStreamState('conv-a', 200);
const got = ChatService.getStreamState('conv-a');
expect(got!.bytesReceived).toBe(200);
});
it('keeps states for distinct conversations isolated', () => {
ChatService.saveStreamState('conv-a', 10);
ChatService.saveStreamState('conv-b', 20);
expect(ChatService.getStreamState('conv-a')!.bytesReceived).toBe(10);
expect(ChatService.getStreamState('conv-b')!.bytesReceived).toBe(20);
});
it('clears the state for a given conversation', () => {
ChatService.saveStreamState('conv-a', 10);
ChatService.clearStreamState('conv-a');
expect(ChatService.getStreamState('conv-a')).toBeNull();
});
it('ignores empty conversation id on save', () => {
ChatService.saveStreamState('', 1);
expect(ChatService.getStreamState('')).toBeNull();
});
it('returns null on corrupted storage payload', () => {
localStorage.setItem(`${STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX}conv-a`, '{not-json');
expect(ChatService.getStreamState('conv-a')).toBeNull();
});
it('persists the model alongside the byte count', () => {
ChatService.saveStreamState('conv-a', 10, 'model-x');
expect(ChatService.getStreamState('conv-a')!.model).toBe('model-x');
});
it('stores a null model when none is provided', () => {
ChatService.saveStreamState('conv-a', 10);
expect(ChatService.getStreamState('conv-a')!.model).toBeNull();
});
it('overwrites the model on a new save for the same conversation', () => {
ChatService.saveStreamState('conv-a', 10, 'model-x');
ChatService.saveStreamState('conv-a', 20, 'model-y');
expect(ChatService.getStreamState('conv-a')!.model).toBe('model-y');
});
describe('resumeStreamIdentity', () => {
it('appends the persisted model so the resume key matches the frozen POST identity', () => {
ChatService.saveStreamState('conv-a', 10, 'model-x');
expect(
ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown')
).toBe('conv-a::model-x');
});
it('keeps the bare conv id when the persisted model is null', () => {
ChatService.saveStreamState('conv-a', 10);
expect(
ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown')
).toBe('conv-a');
});
it('falls back to the current model only when no state is persisted', () => {
expect(ChatService.resumeStreamIdentity('conv-a', null, 'dropdown')).toBe('conv-a::dropdown');
});
it('ignores the fallback when a state exists, the persisted value is authoritative', () => {
ChatService.saveStreamState('conv-a', 10, 'model-x');
expect(
ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown')
).toBe('conv-a::model-x');
});
it('falls back when a legacy state has no model field', () => {
localStorage.setItem(
`${STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX}conv-a`,
JSON.stringify({ bytesReceived: 10, updatedAt: 1 })
);
expect(
ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown')
).toBe('conv-a::dropdown');
});
});
});