Merge branch 'upstream' into concedo_experimental

# Conflicts:
#	.github/workflows/build-webgpu.yml
#	CMakeLists.txt
#	common/CMakeLists.txt
#	docs/development/HOWTO-add-model.md
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-sycl/CMakeLists.txt
#	tests/test-arg-parser.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tests/test-save-load-state.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/README.md
This commit is contained in:
Concedo
2026-07-27 22:28:38 +08:00
74 changed files with 2516 additions and 361 deletions
@@ -13,6 +13,13 @@
const gauge = useContextGauge();
// The gauge hook wraps a processing state instance that only follows the
// live stream while its own monitoring flag is set, so the card instance
// starts monitoring like the dial does.
$effect(() => {
gauge.startMonitoring();
});
let cardEl = $state<HTMLElement | null>(null);
// Any press outside the card and outside the dial closes the card.
@@ -44,7 +51,7 @@
<div
role="status"
bind:this={cardEl}
class="absolute z-50 w-64 -translate-x-1/2 rounded-lg border border-border/50 bg-popover p-3 text-popover-foreground shadow-lg"
class="absolute z-50 w-64 -translate-x-1/2 rounded-lg border border-border/50 bg-popover p-3 text-sm text-popover-foreground shadow-lg ring-1 ring-foreground/10"
style="left: {gaugePopup.centerX}px; bottom: {gaugePopup.bottom}px"
onpointerenter={gaugeCardEnter}
onpointerleave={gaugeCardLeave}
@@ -10,7 +10,7 @@
} from '$lib/components/app';
import { getMessageEditContext } from '$lib/contexts';
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { isLoading, isChatStreaming } from '$lib/stores/chat.svelte';
import { chatStore, isLoading, isChatStreaming } from '$lib/stores/chat.svelte';
import { modelLoadProgressText } from '$lib/utils';
import { MessageRole } from '$lib/enums';
import { config } from '$lib/stores/settings.svelte';
@@ -82,8 +82,11 @@
let hasNoContent = $derived(!message?.content?.trim());
let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming);
// during a router auto-load the message has no model yet, so target the selected one
let loadTargetModel = $derived(message.model ?? modelsStore.selectedModelName);
// during a router auto-load the message has no model yet: target the model frozen in the
// persisted stream state (survives a reload), then fall back to the dropdown selection
let loadTargetModel = $derived(
message.model ?? chatStore.getResumeModel(message.convId) ?? modelsStore.selectedModelName
);
let modelLoadProgress = $derived(
isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null
);
@@ -7,7 +7,7 @@
import { getMessageEditContext } from '$lib/contexts';
import { KeyboardKey, MessageRole } from '$lib/enums';
import { config } from '$lib/stores/settings.svelte';
import { isIMEComposing } from '$lib/utils';
import { autoResizeTextarea, isIMEComposing } from '$lib/utils';
interface Props {
class?: string;
@@ -91,6 +91,11 @@
resizeObserver.disconnect();
};
});
$effect(() => {
if (editCtx.isEditing && textareaElement) {
autoResizeTextarea(textareaElement);
}
});
function toggleExpand() {
isExpanded = !isExpanded;
@@ -105,11 +110,15 @@
{#if editCtx.isEditing}
<div class="w-full max-w-[80%]">
<textarea
style="max-height: var(--max-message-height);"
bind:this={textareaElement}
value={editCtx.editedContent}
class="min-h-[60px] w-full resize-none rounded-2xl px-3 py-2 text-sm {INPUT_CLASSES}"
onkeydown={handleEditKeydown}
oninput={(e) => editCtx.setContent(e.currentTarget.value)}
oninput={(e) => {
autoResizeTextarea(e.currentTarget);
editCtx.setContent(e.currentTarget.value);
}}
placeholder="Edit system message..."
></textarea>
@@ -31,11 +31,11 @@
import { config } from '$lib/stores/settings.svelte';
import { serverLoading, serverError } from '$lib/stores/server.svelte';
import { parseFilesToMessageExtras } from '$lib/utils/browser-only';
import { onDestroy, onMount } from 'svelte';
import { onDestroy, onMount, tick } from 'svelte';
import ChatScreenGreeting from './ChatScreenGreeting.svelte';
import ChatScreenActionScrollDown from './ChatScreenActionScrollDown.svelte';
import ChatScreenDialogsAndAlerts from './ChatScreenDialogsAndAlerts.svelte';
import { ROUTES } from '$lib/constants';
import { LANDING_SETTLE_MAX_MS, LANDING_STABLE_FRAMES, ROUTES } from '$lib/constants';
let { showCenteredEmpty = false } = $props();
@@ -128,6 +128,41 @@
return true;
}
let lastScrolledConversationId: string | null = null;
// Lands at the bottom of a conversation the first time its messages
// render, whether the route comes from another conversation or from a
// non-conversation route. The page keeps growing after the first pin
// without DOM mutations (content-visibility size realizations, syntax
// highlight passes), so the instant pin repeats every frame until the
// height settles, bailing out on user scroll or conversation change.
async function handleMessagesReady(messageCount: number) {
if (messageCount === 0) return;
const id = activeConversation()?.id ?? null;
if (!id || id === lastScrolledConversationId) return;
lastScrolledConversationId = id;
await tick();
autoScroll.scrollToBottom();
const container = scroll.chatScrollContainer;
if (!container) return;
const started = performance.now();
let stableFrames = 0;
let lastHeight = container.scrollHeight;
const settle = () => {
if (autoScroll.userScrolledUp) return;
if (activeConversation()?.id !== id) return;
autoScroll.scrollToBottom();
const height = container.scrollHeight;
stableFrames = height === lastHeight ? stableFrames + 1 : 0;
lastHeight = height;
if (stableFrames >= LANDING_STABLE_FRAMES) return;
if (performance.now() - started > LANDING_SETTLE_MAX_MS) return;
requestAnimationFrame(settle);
};
requestAnimationFrame(settle);
}
function handleSendLikeScroll() {
if (!isMobile.current) {
autoScroll.enable();
@@ -246,6 +281,7 @@
{#if !isEmpty}
<ChatMessages
messages={activeMessages()}
onMessagesReady={handleMessagesReady}
onUserAction={() => {
handleSendLikeScroll();
}}
@@ -159,8 +159,10 @@
try {
const input = document.createElement('input');
// No `accept` filter: iOS resolves each entry to a UTI and has none for
// `.jsonl`, which greys out exported conversations in the file picker.
// `parseImportFile` detects the format from the file contents instead.
input.type = HtmlInputType.FILE;
input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;
input.onchange = async (e) => {
const file = (e.target as HTMLInputElement)?.files?.[0];
@@ -199,9 +201,17 @@
.snapshot(fullImportData)
.filter((item) => selectedIds.has(item.conv.id));
await conversationsStore.importConversationsData(selectedData);
const { imported, skipped } = await conversationsStore.importConversationsData(selectedData);
importedConversations = selectedConversations;
// A conversation already in the database is left untouched, so the summary
// lists what was written and the toast accounts for the rest.
if (skipped.length > 0) {
toast.info(
`Skipped ${skipped.length} conversation${skipped.length === 1 ? '' : 's'} already in your library`
);
}
importedConversations = imported;
showImportSummary = true;
showExportSummary = false;
showImportDialog = false;
+5 -1
View File
@@ -21,7 +21,11 @@ export const API_TOOLS = {
EXECUTE: '/tools'
};
// resumable stream routes, the conv::model identity is appended as a path segment
// resumable stream routes, the conv::model identity travels as the conv_id query param
// because model names can contain slashes that a path segment cannot carry
// resume retry cadence while the owning model is still loading (server answers 503)
export const STREAM_RESUME_RETRY_MS = 2000;
export const API_STREAM = {
BASE: './v1/stream',
LOOKUP: './v1/streams/lookup'
@@ -1,4 +1,10 @@
export const AUTO_SCROLL_INTERVAL = 100;
// Conversation landing: the page keeps growing after the first bottom pin
// without DOM mutations (content-visibility size realizations, syntax
// highlight passes), so the pin repeats every frame until the height holds
// for this many consecutive frames, bounded by the time cap below.
export const LANDING_STABLE_FRAMES = 10;
export const LANDING_SETTLE_MAX_MS = 1000;
// Chat main view: tight threshold because scroll-here events come from
// discrete assistant-message appends.
export const AUTO_SCROLL_AT_BOTTOM_THRESHOLD = 10;
@@ -0,0 +1,3 @@
// First bytes of every ZIP local file header ("PK"). Import detects an archive
// from these bytes rather than from the filename, which the OS may not preserve.
export const ZIP_MAGIC = [0x50, 0x4b];
+1
View File
@@ -13,6 +13,7 @@ export * from './storage';
export * from './attachment-menu';
export * from './auto-scroll';
export * from './context-gauge-popup';
export * from './conversation-import';
export * from './binary-detection';
export * from './built-in-tools';
export * from './cache';
@@ -7,6 +7,9 @@ export const EXPORT_CONV_NAME_SUFFIX_MAX_LENGTH = 20;
// Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00
export const ISO_TIMESTAMP_SLICE_LENGTH = 19;
// Producer marker carried by the session record of a JSONL export
export const SESSION_HARNESS = 'llama.app';
// Replacements for making the conversation title filename-friendly
export const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi;
export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_';
+9 -6
View File
@@ -14,12 +14,15 @@ export const SANDBOX_EMPTY_OUTPUT = '(no output)';
export const SANDBOX_TRUNCATION_NOTICE = '[output truncated]';
const NERDAMER_DESCRIPTION = `
Symbolic/numeric math via \`nerdamer\` (pre-loaded, do not require, use it directly).
nerdamer('diff(sin(x)/x,x)') or nerdamer.diff('sin(x)/x','x') → Expression; convert with .toString()/.text()/.toTeX(), or .evaluate() (→ still Expression, then .toString()).
nerdamer(expr,{x:2}) substitutes only; chain .evaluate() or pass 'numer' for numeric result.
solve(expr,var)→Symbol[]; solveEquations([eq1,..])→[[var,val],..] pairs.
Functions: simplify/expand/factor(expr), diff(expr,var[,n]), integrate(expr,var), defint(expr,from,to,var), limit(expr,var,to), laplace(expr,t,s), ilt(expr,s,t), gcd/lcm(a,b), roots/coeffs/partfrac(expr,var), pfactor(n), numer/decimals/erf(expr), product/sum(expr,var,from,to), mean/median/stdev/variance(...vals).
Object.keys(nerdamer).filter(k=>typeof nerdamer[k]==='function') lists all available functions. If you need a function not documented above, list them first — do not guess function names.`;
Symbolic/numeric math via \`nerdamer\`
nerdamer(expr,subs?,opts?)/nerdamer.func(...)→Expression Format via .text(fmt?) (fmt: 'decimals'|'fractions'|'scientific') eval via .evaluate(subs?)
nerdamer(expr,{x:2}) substitutes numeric via opts 'numer' or .evaluate()
simplify/expand/factor(expr) div/gcd/lcm(...) coeffs/partfrac(expr,var)
diff/integrate(expr,var) defint(expr,lo,hi,var?) sum/product(expr,var,lo,hi) limit(expr,var,pt)
solve(expr,var) solveEquations([eq1,eq2],[var1,var2])
polarform/rectform/arg/realpart/imagpart(z)
set/get Var/Constant(name,val?) setFunction(name,[params],body)
IMPORTANT:Identifier 'nerdamer' has already been declared, use it directly`;
/**
* Build the sandbox tool definition. When `includeSymbolicMath` is true,
@@ -0,0 +1,9 @@
/**
* Discriminator of a record line in the JSONL conversation format. A session
* record opens a conversation and carries its properties; every following
* message record belongs to it.
*/
export enum SessionRecordType {
SESSION = 'session',
MESSAGE = 'message'
}
+2
View File
@@ -27,6 +27,8 @@ export {
ReasoningFormat
} from './chat.enums';
export { SessionRecordType } from './conversation-import.enums';
export { ReasoningEffort } from './reasoning-effort.enums';
export {
+30 -2
View File
@@ -343,6 +343,9 @@ export class ChatService {
// model the ::model suffix keeps the per model session distinct
if (stream && conversationId) {
headers['X-Conversation-Id'] = streamIdentity(conversationId, options.model);
// persist the pending stream before the fetch: a reload during the model load or
// the prompt processing must still find its way back to the session once it exists
ChatService.saveStreamState(conversationId, 0, options.model ?? null);
}
const response = await fetch(API_CHAT.COMPLETIONS, {
@@ -353,6 +356,11 @@ export class ChatService {
});
if (!response.ok) {
// a rejected request (including one cancelled by a stop during the model load)
// leaves nothing to resume
if (conversationId) {
ChatService.clearStreamState(conversationId);
}
const error = await ChatService.parseErrorResponse(response);
if (onError) {
@@ -512,7 +520,7 @@ export class ChatService {
if (!conversationId) return;
try {
const id = streamIdentity(conversationId, model);
await fetch(`${API_STREAM.BASE}/${encodeURIComponent(id)}`, {
await fetch(`${API_STREAM.BASE}?conv_id=${encodeURIComponent(id)}`, {
method: 'DELETE',
headers: getAuthHeaders()
});
@@ -605,6 +613,26 @@ export class ChatService {
* existing SSE parser drains it like a fresh stream. The server returns 200 on success, 404 if
* no session exists for the conv_id, and 400 if the offset is below the dropped prefix.
*/
// probe the resume route status without consuming the stream: the SSE route has no HEAD,
// so issue the GET and abort it right after the status line. 0 on network error
static async probeResumeStatus(streamId: string): Promise<number> {
if (!streamId) return 0;
const ac = new AbortController();
try {
const resp = await fetch(
`${API_STREAM.BASE}?conv_id=${encodeURIComponent(streamId)}&from=0`,
{
headers: getAuthHeaders(),
signal: ac.signal
}
);
ac.abort();
return resp.status;
} catch {
return 0;
}
}
static async resumeStream(
conversationId: string,
signal?: AbortSignal,
@@ -614,7 +642,7 @@ export class ChatService {
const state = ChatService.getStreamState(conversationId);
const from = state?.bytesReceived ?? 0;
const id = streamIdentity(conversationId, model);
const url = `${API_STREAM.BASE}/${encodeURIComponent(id)}?from=${from}`;
const url = `${API_STREAM.BASE}?conv_id=${encodeURIComponent(id)}&from=${from}`;
return await fetch(url, { method: 'GET', signal, headers: getAuthHeaders() });
}
@@ -554,12 +554,13 @@ export class DatabaseService {
* Skips conversations that already exist.
*
* @param data - Array of { conv, messages } objects
* @returns The conversations written to the database and the ones skipped
*/
static async importConversations(
data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]
): Promise<{ imported: number; skipped: number }> {
let importedCount = 0;
let skippedCount = 0;
): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
const imported: DatabaseConversation[] = [];
const skipped: DatabaseConversation[] = [];
return await db.transaction(
'rw',
@@ -570,8 +571,7 @@ export class DatabaseService {
const existing = await db[IDXDB_TABLES.conversations].get(conv.id);
if (existing) {
console.warn(`Conversation "${conv.name}" already exists, skipping...`);
skippedCount++;
skipped.push(conv);
continue;
}
@@ -580,10 +580,10 @@ export class DatabaseService {
await db[IDXDB_TABLES.messages].put(msg);
}
importedCount++;
imported.push(conv);
}
return { imported: importedCount, skipped: skippedCount };
return { imported, skipped };
}
);
}
+60 -5
View File
@@ -14,6 +14,7 @@
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
import { DatabaseService } from '$lib/services/database.service';
import { ChatService } from '$lib/services/chat.service';
import { STREAM_RESUME_RETRY_MS } from '$lib/constants/api-endpoints';
import { streamIdentity } from '$lib/utils/stream-identity';
import { getAuthHeaders } from '$lib/utils/api-headers';
import { CONTENT_TYPE_HEADER } from '$lib/constants';
@@ -78,7 +79,7 @@ class ChatStore {
// true while the active conversation streams reasoning content but no visible content yet
isReasoning = $state(false);
// resumable stream connection state for the active conversation
// streaming -> bytes flowing normally, resuming -> waiting on /v1/stream/:id reconnect, lost -> unrecoverable
// streaming -> bytes flowing normally, resuming -> waiting on /v1/stream reconnect, lost -> unrecoverable
streamConnectionState = $state<StreamConnectionState>(StreamConnectionState.STREAMING);
chatLoadingStates = new SvelteMap<string, boolean>();
chatReasoningStates = new SvelteMap<string, boolean>();
@@ -94,6 +95,11 @@ class ChatStore {
// off when one conv finishes while another is still streaming. mirrors chatLoadingStates
// in scope but tracks the attach + tee replay path specifically
private attachingConvs = new SvelteSet<string>();
// pending resume retry timers while an owning model loads, one per conv
private resumeRetryTimers = new SvelteMap<string, ReturnType<typeof setTimeout>>();
// convs whose resume waits on a model load: their loading state belongs to the retry loop,
// so discoverActiveStream must not treat it as a live send and bail
private resumePendingConvs = new SvelteSet<string>();
// in-flight discoverActiveStream guard, keyed by conv id
private discoveringConvs = new SvelteSet<string>();
private abortControllers = new SvelteMap<string, AbortController>();
@@ -263,7 +269,7 @@ class ChatStore {
const id = streamId || streamIdentity(convId, selectedModelName());
let response: Response;
try {
response = await fetch(`./v1/stream/${encodeURIComponent(id)}?from=0`, {
response = await fetch(`./v1/stream?conv_id=${encodeURIComponent(id)}&from=0`, {
headers: getAuthHeaders()
});
} catch (e) {
@@ -438,13 +444,22 @@ class ChatStore {
}
}
/**
* Model frozen at send time for a stream awaiting resume, from the persisted stream state.
* The load progress indicator targets it after a reload, when the message row has no model
* yet and the dropdown selection may not be restored.
*/
getResumeModel(convId: string): string | null {
return ChatService.getStreamState(convId)?.model ?? null;
}
async discoverActiveStream(convId: string): Promise<void> {
if (!convId) return;
if (this.chatStreamingStates.has(convId)) return;
if (this.chatLoadingStates.get(convId)) return;
if (this.chatLoadingStates.get(convId) && !this.resumePendingConvs.has(convId)) return;
// concurrency guard: another discover may already be running for this conv (typical race
// between mount and visibilitychange on tab switch). a second concurrent fetch on the same
// /v1/stream/<id> would duplicate every byte into the DB message, this guard bounces it
// /v1/stream would duplicate every byte into the DB message, this guard bounces it
if (this.discoveringConvs.has(convId)) return;
this.discoveringConvs.add(convId);
@@ -470,6 +485,38 @@ class ChatStore {
if (!localState) {
return;
}
// quiet status probe first: a full attach flips the loading UI on every try, probing
// keeps the retry loop invisible while the owning model is still loading (503)
const status = await ChatService.probeResumeStatus(streamId);
if (status === 503) {
// make the wait visible: the empty assistant row persisted at send time renders
// the processing info, whose model load percentage flows from the models feed
this.resumePendingConvs.add(convId);
this.setChatLoading(convId, true);
if (!this.resumeRetryTimers.has(convId)) {
this.resumeRetryTimers.set(
convId,
setTimeout(() => {
this.resumeRetryTimers.delete(convId);
void this.discoverActiveStream(convId);
}, STREAM_RESUME_RETRY_MS)
);
}
return;
}
if (this.resumePendingConvs.delete(convId) && status !== 200) {
// the wait is over without a session to attach, drop the visible loading state
this.setChatLoading(convId, false);
}
if (status === 0) {
// transient network failure, the next mount or visibility change retries
return;
}
if (status !== 200) {
// the session is gone (stopped, TTL expired), nothing to resume anymore
ChatService.clearStreamState(convId);
return;
}
await this.attachServerStream(convId, streamId);
// if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever
if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) {
@@ -1469,8 +1516,16 @@ class ChatStore {
// detached drain keeps producing tokens until eos or max_tokens. use the frozen identity
// captured when the session started, not the live dropdown
const streamStateForStop = this.chatStreamingStates.get(convId);
const modelForStop = streamStateForStop?.model;
const modelForStop = streamStateForStop?.model ?? ChatService.getStreamState(convId)?.model;
void ChatService.cancelServerStream(convId, modelForStop);
// an explicit stop leaves nothing to resume and kills a pending resume retry
ChatService.clearStreamState(convId);
const retryTimer = this.resumeRetryTimers.get(convId);
if (retryTimer !== undefined) {
clearTimeout(retryTimer);
this.resumeRetryTimers.delete(convId);
}
this.resumePendingConvs.delete(convId);
this.abortRequest(convId);
this.setChatLoading(convId, false);
this.clearChatStreaming(convId);
+52 -83
View File
@@ -30,11 +30,11 @@ import type { McpServerOverride } from '$lib/types/database';
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
import {
MessageRole,
HtmlInputType,
FileExtensionText,
MimeTypeText,
MimeTypeApplication,
ReasoningEffort
ReasoningEffort,
SessionRecordType
} from '$lib/enums';
import {
ISO_DATE_TIME_SEPARATOR,
@@ -47,7 +47,10 @@ import {
ISO_TIME_SEPARATOR_REPLACEMENT,
NON_ALPHANUMERIC_REGEX,
MULTIPLE_UNDERSCORE_REGEX,
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY,
NEWLINE,
SESSION_HARNESS,
ZIP_MAGIC
} from '$lib/constants';
import { ROUTES } from '$lib/constants/routes';
@@ -914,30 +917,35 @@ class ConversationsStore {
/**
* Serializes a session (a conversation with its messages) as JSONL.
* The first line is the session header (a `type: 'session'` record carrying the
* conversation properties); each subsequent line is a single message.
* The first line is the session header (a `SessionRecordType.SESSION` record
* carrying the conversation properties); each subsequent line is a single message.
* @param data - The exported conversation payload
* @returns The JSONL string (one record per line)
*/
serializeSessionToJsonl(data: ExportedConversation): string {
const { conv, messages } = data;
const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv });
const sessionLine = JSON.stringify({
type: SessionRecordType.SESSION,
harness: SESSION_HARNESS,
...conv
});
const messageLines = messages.map((message: DatabaseMessage) => {
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
const { toolCalls, ...rest } = message;
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
return JSON.stringify({ type: 'message', message: normalized });
return JSON.stringify({ type: SessionRecordType.MESSAGE, message: normalized });
});
return [sessionLine, ...messageLines].join('\n');
return [sessionLine, ...messageLines].join(NEWLINE);
}
/**
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
* A `type: 'session'` line starts a new session; following `type: 'message'`
* lines are appended to it. Supports multiple sessions in a single file.
* A `SessionRecordType.SESSION` line starts a new session; following
* `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple
* sessions in a single file.
* @param text - The JSONL file contents
* @returns The parsed conversations with their messages
*/
@@ -945,20 +953,20 @@ class ConversationsStore {
const sessions: ExportedConversation[] = [];
let current: ExportedConversation | null = null;
for (const line of text.split('\n')) {
for (const line of text.split(NEWLINE)) {
const trimmed = line.trim();
if (!trimmed) continue;
const record = JSON.parse(trimmed);
if (record.type === 'session') {
if (record.type === SessionRecordType.SESSION) {
// Drop the discriminator and harness marker; the rest is the conversation.
const conv = { ...record };
delete conv.type;
delete conv.harness;
current = { conv: conv as DatabaseConversation, messages: [] };
sessions.push(current);
} else if (record.type === 'message') {
} else if (record.type === SessionRecordType.MESSAGE) {
if (!current) {
throw new Error('Invalid JSONL: message record before any session record');
}
@@ -977,27 +985,47 @@ class ConversationsStore {
}
/**
* Parses an import file into conversations, accepting the current `.jsonl` and
* `.zip` formats as well as the legacy `.json` format.
* Reports whether the text is the JSONL session format, whose first non-empty
* line is a `SessionRecordType.SESSION` record. A legacy JSON export starts
* with an array or an object that has no such discriminator.
* @param text - The file contents
*/
private isSessionsJsonl(text: string): boolean {
const trimmed = text.trimStart();
const lineEnd = trimmed.indexOf(NEWLINE);
const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd);
try {
return JSON.parse(firstLine).type === SessionRecordType.SESSION;
} catch {
// Not a standalone JSON record, so not the JSONL format.
return false;
}
}
/**
* Parses an import file into conversations, accepting the current JSONL and
* ZIP formats as well as the legacy JSON format. The format comes from the
* contents, so an import works whatever the file is named.
* @param file - The user-selected file
* @returns The parsed conversations with their messages
*/
async parseImportFile(file: File): Promise<ExportedConversation[]> {
const name = file.name.toLowerCase();
const bytes = new Uint8Array(await file.arrayBuffer());
if (name.endsWith(FileExtensionText.ZIP)) {
const entries = unzipSync(new Uint8Array(await file.arrayBuffer()));
if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) {
const entries = unzipSync(bytes);
const sessions: ExportedConversation[] = [];
for (const [entryName, bytes] of Object.entries(entries)) {
for (const [entryName, entryBytes] of Object.entries(entries)) {
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
sessions.push(...this.parseSessionsJsonl(strFromU8(bytes)));
sessions.push(...this.parseSessionsJsonl(strFromU8(entryBytes)));
}
return sessions;
}
const text = await file.text();
const text = strFromU8(bytes);
if (name.endsWith(FileExtensionText.JSONL)) {
if (this.isSessionsJsonl(text)) {
return this.parseSessionsJsonl(text);
}
@@ -1103,73 +1131,14 @@ class ConversationsStore {
this.downloadConversationFile({ conv: conversation, messages });
}
/**
* Imports conversations from a JSON file
* Opens file picker and processes the selected file
* @returns The list of imported conversations
*/
async importConversations(): Promise<DatabaseConversation[]> {
return new Promise((resolve, reject) => {
const input = document.createElement('input');
input.type = HtmlInputType.FILE;
input.accept = FileExtensionText.JSON;
input.onchange = async (e) => {
const file = (e.target as HTMLInputElement)?.files?.[0];
if (!file) {
reject(new Error('No file selected'));
return;
}
try {
const text = await file.text();
const parsedData = JSON.parse(text);
let importedData: ExportedConversations;
if (Array.isArray(parsedData)) {
importedData = parsedData;
} else if (
parsedData &&
typeof parsedData === 'object' &&
'conv' in parsedData &&
'messages' in parsedData
) {
importedData = [parsedData];
} else {
throw new Error('Invalid file format');
}
const result = await DatabaseService.importConversations(importedData);
toast.success(`Imported ${result.imported} conversation(s), skipped ${result.skipped}`);
await this.loadConversations();
const importedConversations = (
Array.isArray(importedData) ? importedData : [importedData]
).map((item) => item.conv);
resolve(importedConversations);
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Unknown error';
console.error('Failed to import conversations:', err);
toast.error('Import failed', { description: message });
reject(new Error(`Import failed: ${message}`));
}
};
input.click();
});
}
/**
* Imports conversations from provided data (without file picker)
* @param data - Array of conversation data with messages
* @returns Import result with counts
* @returns The conversations written to the database and the ones skipped
*/
async importConversationsData(
data: ExportedConversations
): Promise<{ imported: number; skipped: number }> {
): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
const result = await DatabaseService.importConversations(data);
await this.loadConversations();
return result;
@@ -161,9 +161,3 @@ export function generateModalityErrorMessage(
return message;
}
/**
* Generate file input accept string based on model modalities
* @param capabilities - The modality capabilities to check against
* @returns Accept string for HTML file input element
*/
@@ -0,0 +1,68 @@
import { afterEach, describe, expect, it } from 'vitest';
import { DatabaseService } from '$lib/services/database.service';
import { MessageRole, MessageType } from '$lib/enums';
import type { ExportedConversation } from '$lib/types/database';
function makeSession(id: string): ExportedConversation {
return {
conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` },
messages: [
{
id: `${id}-msg`,
convId: id,
type: MessageType.TEXT,
timestamp: 0,
role: MessageRole.USER,
content: `hello from ${id}`,
parent: null,
children: []
}
]
} as unknown as ExportedConversation;
}
afterEach(async () => {
const conversations = await DatabaseService.getAllConversations();
await DatabaseService.bulkDeleteConversations(conversations.map((conv) => conv.id));
});
/**
* An import leaves a conversation already in the database untouched, so the
* caller needs to know what was written to report it instead of echoing the
* selection back at the user.
*/
describe('DatabaseService.importConversations', () => {
it('reports the conversations it wrote', async () => {
const { imported, skipped } = await DatabaseService.importConversations([
makeSession('a'),
makeSession('b')
]);
expect(imported.map((conv) => conv.id)).toEqual(['a', 'b']);
expect(skipped).toEqual([]);
expect(await DatabaseService.getConversationMessages('a')).toHaveLength(1);
});
it('reports an existing conversation as skipped and leaves it untouched', async () => {
await DatabaseService.importConversations([makeSession('a')]);
await DatabaseService.updateConversation('a', { name: 'Renamed locally' });
const { imported, skipped } = await DatabaseService.importConversations([makeSession('a')]);
expect(imported).toEqual([]);
expect(skipped.map((conv) => conv.id)).toEqual(['a']);
expect((await DatabaseService.getConversation('a'))?.name).toBe('Renamed locally');
});
it('imports the new conversations of a partially known selection', async () => {
await DatabaseService.importConversations([makeSession('a')]);
const { imported, skipped } = await DatabaseService.importConversations([
makeSession('a'),
makeSession('b')
]);
expect(imported.map((conv) => conv.id)).toEqual(['b']);
expect(skipped.map((conv) => conv.id)).toEqual(['a']);
});
});
@@ -0,0 +1,112 @@
import { beforeAll, describe, expect, it } from 'vitest';
import { zipSync, strToU8 } from 'fflate';
import { MessageRole, MessageType } from '$lib/enums';
import { NEWLINE } from '$lib/constants';
import type { ExportedConversation } from '$lib/types/database';
let conversationsStore: typeof import('$lib/stores/conversations.svelte').conversationsStore;
// node env unit project has no DOM, install a minimal localStorage backed by a
// Map before the store module reads it. Transforming the store takes seconds,
// so import it once for the whole file.
beforeAll(async () => {
const store = new Map<string, string>();
const polyfill: Storage = {
get length() {
return store.size;
},
clear: () => store.clear(),
getItem: (k) => (store.has(k) ? store.get(k)! : null),
key: (i) => Array.from(store.keys())[i] ?? null,
removeItem: (k) => {
store.delete(k);
},
setItem: (k, v) => {
store.set(k, String(v));
}
};
(globalThis as unknown as { localStorage: Storage }).localStorage = polyfill;
({ conversationsStore } = await import('$lib/stores/conversations.svelte'));
}, 30000);
function makeSession(id: string): ExportedConversation {
return {
conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` },
messages: [
{
id: `${id}-msg`,
convId: id,
type: MessageType.TEXT,
timestamp: 0,
role: MessageRole.USER,
content: `hello from ${id}`,
parent: null,
children: []
}
]
} as unknown as ExportedConversation;
}
/**
* `parseImportFile` detects the format from the file contents. iOS has no UTI
* for `.jsonl`, so the picker cannot filter on it and the filename carries no
* guarantee: a JSONL export must import under any name.
*/
describe('conversationsStore.parseImportFile', () => {
it('imports a JSONL export whose name has no meaningful extension', async () => {
const jsonl = conversationsStore.serializeSessionToJsonl(makeSession('a'));
const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export'));
expect(sessions).toHaveLength(1);
expect(sessions[0].conv.id).toBe('a');
expect(sessions[0].messages[0].content).toBe('hello from a');
});
it('imports several sessions from one JSONL file', async () => {
const jsonl = [makeSession('a'), makeSession('b')]
.map((session) => conversationsStore.serializeSessionToJsonl(session))
.join(NEWLINE);
const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export.txt'));
expect(sessions.map((session) => session.conv.id)).toEqual(['a', 'b']);
});
it('imports a ZIP archive whose name has no meaningful extension', async () => {
const zipped = zipSync({
'a.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('a'))),
'b.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('b'))),
'notes.txt': strToU8('ignored')
});
const sessions = await conversationsStore.parseImportFile(new File([zipped], 'archive'));
expect(sessions.map((session) => session.conv.id).sort()).toEqual(['a', 'b']);
});
it('imports the legacy JSON array format', async () => {
const json = JSON.stringify([makeSession('a')], null, 2);
const sessions = await conversationsStore.parseImportFile(new File([json], 'export.jsonl'));
expect(sessions).toHaveLength(1);
expect(sessions[0].conv.id).toBe('a');
});
it('imports the legacy JSON single object format', async () => {
const json = JSON.stringify(makeSession('a'));
const sessions = await conversationsStore.parseImportFile(new File([json], 'export'));
expect(sessions).toHaveLength(1);
expect(sessions[0].conv.id).toBe('a');
});
it('rejects a file that holds neither format', async () => {
await expect(
conversationsStore.parseImportFile(new File(['not an export'], 'export.jsonl'))
).rejects.toThrow();
});
});