Merge commit 'f14f4e421b2177fadcf9d15ebccb0492e5464d86' into concedo_experimental

# Conflicts:
#	.github/workflows/docker.yml
#	AGENTS.md
#	CONTRIBUTING.md
#	docs/build.md
#	examples/llama.android/app/build.gradle.kts
#	examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt
#	examples/llama.android/app/src/main/res/layout/activity_main.xml
#	examples/llama.android/gradle/libs.versions.toml
#	examples/llama.android/lib/src/main/cpp/ai_chat.cpp
#	examples/llama.android/lib/src/main/java/com/arm/aichat/InferenceEngine.kt
#	examples/llama.android/lib/src/main/java/com/arm/aichat/internal/InferenceEngineImpl.kt
#	examples/model-conversion/scripts/causal/compare-embeddings-logits.sh
#	examples/model-conversion/scripts/embedding/run-original-model.py
#	examples/retrieval/retrieval.cpp
#	ggml/src/CMakeLists.txt
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-cpu/kleidiai/kernels.cpp
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-cuda/CMakeLists.txt
#	ggml/src/ggml-cuda/mmq.cu
#	ggml/src/ggml-cuda/mmq.cuh
#	src/CMakeLists.txt
#	tools/llama-bench/llama-bench.cpp
#	tools/server/CMakeLists.txt

tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte

@@ -89,6 +89,7 @@
 	const fallbackToolCalls = $derived(typeof toolCallContent === 'string' ? toolCallContent : null);
 
 	const processingState = useProcessingState();
+
 	let currentConfig = $derived(config());
 	let isRouter = $derived(isRouterMode());
 	let displayedModel = $derived((): string | null => {
@@ -116,6 +117,12 @@
 		}
 	});
 
+	$effect(() => {
+		if (isLoading() && !message?.content?.trim()) {
+			processingState.startMonitoring();
+		}
+	});
+
 	function formatToolCallBadge(toolCall: ApiChatCompletionToolCall, index: number) {
 		const callNumber = index + 1;
 		const functionName = toolCall.function?.name?.trim();
@@ -186,7 +193,7 @@
 		<div class="mt-6 w-full max-w-[48rem]" in:fade>
 			<div class="processing-container">
 				<span class="processing-text">
-					{processingState.getProcessingMessage()}
+					{processingState.getPromptProgressText() ?? processingState.getProcessingMessage()}
 				</span>
 			</div>
 		</div>
@@ -263,6 +270,23 @@
 						predictedTokens={message.timings.predicted_n}
 						predictedMs={message.timings.predicted_ms}
 					/>
+				{:else if isLoading() && currentConfig.showMessageStats}
+					{@const liveStats = processingState.getLiveProcessingStats()}
+					{@const genStats = processingState.getLiveGenerationStats()}
+					{@const promptProgress = processingState.processingState?.promptProgress}
+					{@const isStillProcessingPrompt =
+						promptProgress && promptProgress.processed < promptProgress.total}
+
+					{#if liveStats || genStats}
+						<ChatMessageStatistics
+							isLive={true}
+							isProcessingPrompt={!!isStillProcessingPrompt}
+							promptTokens={liveStats?.tokensProcessed}
+							promptMs={liveStats?.timeMs}
+							predictedTokens={genStats?.tokensGenerated}
+							predictedMs={genStats?.timeMs}
+						/>
+					{/if}
 				{/if}
 			</div>
 		{/if}

tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics.svelte

@@ -5,21 +5,64 @@
 	import { ChatMessageStatsView } from '$lib/enums';
 
 	interface Props {
-		predictedTokens: number;
-		predictedMs: number;
+		predictedTokens?: number;
+		predictedMs?: number;
 		promptTokens?: number;
 		promptMs?: number;
+		// Live mode: when true, shows stats during streaming
+		isLive?: boolean;
+		// Whether prompt processing is still in progress
+		isProcessingPrompt?: boolean;
+		// Initial view to show (defaults to READING in live mode)
+		initialView?: ChatMessageStatsView;
 	}
 
-	let { predictedTokens, predictedMs, promptTokens, promptMs }: Props = $props();
+	let {
+		predictedTokens,
+		predictedMs,
+		promptTokens,
+		promptMs,
+		isLive = false,
+		isProcessingPrompt = false,
+		initialView = ChatMessageStatsView.GENERATION
+	}: Props = $props();
 
-	let activeView: ChatMessageStatsView = $state(ChatMessageStatsView.GENERATION);
+	let activeView: ChatMessageStatsView = $state(initialView);
+	let hasAutoSwitchedToGeneration = $state(false);
 
-	let tokensPerSecond = $derived((predictedTokens / predictedMs) * 1000);
-	let timeInSeconds = $derived((predictedMs / 1000).toFixed(2));
+	// In live mode: auto-switch to GENERATION tab when prompt processing completes
+	$effect(() => {
+		if (isLive) {
+			// Auto-switch to generation tab only when prompt processing is done (once)
+			if (
+				!hasAutoSwitchedToGeneration &&
+				!isProcessingPrompt &&
+				predictedTokens &&
+				predictedTokens > 0
+			) {
+				activeView = ChatMessageStatsView.GENERATION;
+				hasAutoSwitchedToGeneration = true;
+			} else if (!hasAutoSwitchedToGeneration) {
+				// Stay on READING while prompt is still being processed
+				activeView = ChatMessageStatsView.READING;
+			}
+		}
+	});
+
+	let hasGenerationStats = $derived(
+		predictedTokens !== undefined &&
+			predictedTokens > 0 &&
+			predictedMs !== undefined &&
+			predictedMs > 0
+	);
+
+	let tokensPerSecond = $derived(hasGenerationStats ? (predictedTokens! / predictedMs!) * 1000 : 0);
+	let timeInSeconds = $derived(
+		predictedMs !== undefined ? (predictedMs / 1000).toFixed(2) : '0.00'
+	);
 
 	let promptTokensPerSecond = $derived(
-		promptTokens !== undefined && promptMs !== undefined
+		promptTokens !== undefined && promptMs !== undefined && promptMs > 0
 			? (promptTokens / promptMs) * 1000
 			: undefined
 	);
@@ -34,11 +77,14 @@
 			promptTokensPerSecond !== undefined &&
 			promptTimeInSeconds !== undefined
 	);
+
+	// In live mode, generation tab is disabled until we have generation stats
+	let isGenerationDisabled = $derived(isLive && !hasGenerationStats);
 </script>
 
 <div class="inline-flex items-center text-xs text-muted-foreground">
 	<div class="inline-flex items-center rounded-sm bg-muted-foreground/15 p-0.5">
-		{#if hasPromptStats}
+		{#if hasPromptStats || isLive}
 			<Tooltip.Root>
 				<Tooltip.Trigger>
 					<button
@@ -65,25 +111,32 @@
 					class="inline-flex h-5 w-5 items-center justify-center rounded-sm transition-colors {activeView ===
 					ChatMessageStatsView.GENERATION
 						? 'bg-background text-foreground shadow-sm'
-						: 'hover:text-foreground'}"
-					onclick={() => (activeView = ChatMessageStatsView.GENERATION)}
+						: isGenerationDisabled
+							? 'cursor-not-allowed opacity-40'
+							: 'hover:text-foreground'}"
+					onclick={() => !isGenerationDisabled && (activeView = ChatMessageStatsView.GENERATION)}
+					disabled={isGenerationDisabled}
 				>
 					<Sparkles class="h-3 w-3" />
 					<span class="sr-only">Generation</span>
 				</button>
 			</Tooltip.Trigger>
 			<Tooltip.Content>
-				<p>Generation (token output)</p>
+				<p>
+					{isGenerationDisabled
+						? 'Generation (waiting for tokens...)'
+						: 'Generation (token output)'}
+				</p>
 			</Tooltip.Content>
 		</Tooltip.Root>
 	</div>
 
 	<div class="flex items-center gap-1 px-2">
-		{#if activeView === ChatMessageStatsView.GENERATION}
+		{#if activeView === ChatMessageStatsView.GENERATION && hasGenerationStats}
 			<BadgeChatStatistic
 				class="bg-transparent"
 				icon={WholeWord}
-				value="{predictedTokens} tokens"
+				value="{predictedTokens?.toLocaleString()} tokens"
 				tooltipLabel="Generated tokens"
 			/>
 			<BadgeChatStatistic

tools/server/webui/src/lib/hooks/use-processing-state.svelte.ts

@@ -1,10 +1,27 @@
 import { activeProcessingState } from '$lib/stores/chat.svelte';
 import { config } from '$lib/stores/settings.svelte';
 
+export interface LiveProcessingStats {
+	tokensProcessed: number;
+	totalTokens: number;
+	timeMs: number;
+	tokensPerSecond: number;
+	etaSecs?: number;
+}
+
+export interface LiveGenerationStats {
+	tokensGenerated: number;
+	timeMs: number;
+	tokensPerSecond: number;
+}
+
 export interface UseProcessingStateReturn {
 	readonly processingState: ApiProcessingState | null;
 	getProcessingDetails(): string[];
 	getProcessingMessage(): string;
+	getPromptProgressText(): string | null;
+	getLiveProcessingStats(): LiveProcessingStats | null;
+	getLiveGenerationStats(): LiveGenerationStats | null;
 	shouldShowDetails(): boolean;
 	startMonitoring(): void;
 	stopMonitoring(): void;
@@ -29,6 +46,7 @@ export interface UseProcessingStateReturn {
 export function useProcessingState(): UseProcessingStateReturn {
 	let isMonitoring = $state(false);
 	let lastKnownState = $state<ApiProcessingState | null>(null);
+	let lastKnownProcessingStats = $state<LiveProcessingStats | null>(null);
 
 	// Derive processing state reactively from chatStore's direct state
 	const processingState = $derived.by(() => {
@@ -46,6 +64,34 @@ export function useProcessingState(): UseProcessingStateReturn {
 		}
 	});
 
+	// Track last known processing stats for when promptProgress disappears
+	$effect(() => {
+		if (processingState?.promptProgress) {
+			const { processed, total, time_ms, cache } = processingState.promptProgress;
+			const actualProcessed = processed - cache;
+			const actualTotal = total - cache;
+
+			if (actualProcessed > 0 && time_ms > 0) {
+				const tokensPerSecond = actualProcessed / (time_ms / 1000);
+				lastKnownProcessingStats = {
+					tokensProcessed: actualProcessed,
+					totalTokens: actualTotal,
+					timeMs: time_ms,
+					tokensPerSecond
+				};
+			}
+		}
+	});
+
+	function getETASecs(done: number, total: number, elapsedMs: number): number | undefined {
+		const elapsedSecs = elapsedMs / 1000;
+		const progressETASecs =
+			done === 0 || elapsedSecs < 0.5
+				? undefined // can be the case for the 0% progress report
+				: elapsedSecs * (total / done - 1);
+		return progressETASecs;
+	}
+
 	function startMonitoring(): void {
 		if (isMonitoring) return;
 		isMonitoring = true;
@@ -59,28 +105,25 @@ export function useProcessingState(): UseProcessingStateReturn {
 		const currentConfig = config();
 		if (!currentConfig.keepStatsVisible) {
 			lastKnownState = null;
+			lastKnownProcessingStats = null;
 		}
 	}
 
 	function getProcessingMessage(): string {
-		const state = processingState;
-		if (!state) {
+		if (!processingState) {
 			return 'Processing...';
 		}
 
-		switch (state.status) {
+		switch (processingState.status) {
 			case 'initializing':
 				return 'Initializing...';
 			case 'preparing':
-				if (state.progressPercent !== undefined) {
-					return `Processing (${state.progressPercent}%)`;
+				if (processingState.progressPercent !== undefined) {
+					return `Processing (${processingState.progressPercent}%)`;
 				}
 				return 'Preparing response...';
 			case 'generating':
-				if (state.tokensDecoded > 0) {
-					return `Generating... (${state.tokensDecoded} tokens)`;
-				}
-				return 'Generating...';
+				return '';
 			default:
 				return 'Processing...';
 		}
@@ -131,8 +174,76 @@ export function useProcessingState(): UseProcessingStateReturn {
 	}
 
 	function shouldShowDetails(): boolean {
-		const state = processingState;
-		return state !== null && state.status !== 'idle';
+		return processingState !== null && processingState.status !== 'idle';
+	}
+
+	/**
+	 * Returns a short progress message with percent
+	 */
+	function getPromptProgressText(): string | null {
+		if (!processingState?.promptProgress) return null;
+
+		const { processed, total, cache } = processingState.promptProgress;
+
+		const actualProcessed = processed - cache;
+		const actualTotal = total - cache;
+		const percent = Math.round((actualProcessed / actualTotal) * 100);
+		const eta = getETASecs(actualProcessed, actualTotal, processingState.promptProgress.time_ms);
+
+		if (eta !== undefined) {
+			const etaSecs = Math.ceil(eta);
+			return `Processing ${percent}% (ETA: ${etaSecs}s)`;
+		}
+
+		return `Processing ${percent}%`;
+	}
+
+	/**
+	 * Returns live processing statistics for display (prompt processing phase)
+	 * Returns last known stats when promptProgress becomes unavailable
+	 */
+	function getLiveProcessingStats(): LiveProcessingStats | null {
+		if (processingState?.promptProgress) {
+			const { processed, total, time_ms, cache } = processingState.promptProgress;
+
+			const actualProcessed = processed - cache;
+			const actualTotal = total - cache;
+
+			if (actualProcessed > 0 && time_ms > 0) {
+				const tokensPerSecond = actualProcessed / (time_ms / 1000);
+
+				return {
+					tokensProcessed: actualProcessed,
+					totalTokens: actualTotal,
+					timeMs: time_ms,
+					tokensPerSecond
+				};
+			}
+		}
+
+		// Return last known stats if promptProgress is no longer available
+		return lastKnownProcessingStats;
+	}
+
+	/**
+	 * Returns live generation statistics for display (token generation phase)
+	 */
+	function getLiveGenerationStats(): LiveGenerationStats | null {
+		if (!processingState) return null;
+
+		const { tokensDecoded, tokensPerSecond } = processingState;
+
+		if (tokensDecoded <= 0) return null;
+
+		// Calculate time from tokens and speed
+		const timeMs =
+			tokensPerSecond && tokensPerSecond > 0 ? (tokensDecoded / tokensPerSecond) * 1000 : 0;
+
+		return {
+			tokensGenerated: tokensDecoded,
+			timeMs,
+			tokensPerSecond: tokensPerSecond || 0
+		};
 	}
 
 	return {
@@ -141,6 +252,9 @@ export function useProcessingState(): UseProcessingStateReturn {
 		},
 		getProcessingDetails,
 		getProcessingMessage,
+		getPromptProgressText,
+		getLiveProcessingStats,
+		getLiveGenerationStats,
 		shouldShowDetails,
 		startMonitoring,
 		stopMonitoring

tools/server/webui/src/lib/services/chat.ts

@@ -117,7 +117,8 @@ export class ChatService {
 				role: msg.role,
 				content: msg.content
 			})),
-			stream
+			stream,
+			return_progress: stream ? true : undefined
 		};
 
 		// Include model in request if provided (required in ROUTER mode)
@@ -271,7 +272,7 @@ export class ChatService {
 		onReasoningChunk?: (chunk: string) => void,
 		onToolCallChunk?: (chunk: string) => void,
 		onModel?: (model: string) => void,
-		onTimings?: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void,
+		onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void,
 		conversationId?: string,
 		abortSignal?: AbortSignal
 	): Promise<void> {
@@ -370,11 +371,13 @@ export class ChatService {
 								onModel?.(chunkModel);
 							}
 
-							if (timings || promptProgress) {
+							if (promptProgress) {
+								ChatService.notifyTimings(undefined, promptProgress, onTimings);
+							}
+
+							if (timings) {
 								ChatService.notifyTimings(timings, promptProgress, onTimings);
-								if (timings) {
-									lastTimings = timings;
-								}
+								lastTimings = timings;
 							}
 
 							if (content) {
@@ -772,10 +775,11 @@ export class ChatService {
 		timings: ChatMessageTimings | undefined,
 		promptProgress: ChatMessagePromptProgress | undefined,
 		onTimingsCallback:
-			| ((timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void)
+			| ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void)
 			| undefined
 	): void {
-		if (!timings || !onTimingsCallback) return;
+		if (!onTimingsCallback || (!timings && !promptProgress)) return;
+
 		onTimingsCallback(timings, promptProgress);
 	}
 }

tools/server/webui/src/lib/stores/chat.svelte.ts

@@ -303,11 +303,17 @@ class ChatStore {
 		const currentConfig = config();
 		const outputTokensMax = currentConfig.max_tokens || -1;
 
+		// Note: for timings data, the n_prompt does NOT include cache tokens
 		const contextUsed = promptTokens + cacheTokens + predictedTokens;
 		const outputTokensUsed = predictedTokens;
 
+		// Note: for prompt progress, the "processed" DOES include cache tokens
+		// we need to exclude them to get the real prompt tokens processed count
+		const progressCache = promptProgress?.cache || 0;
+		const progressActualDone = (promptProgress?.processed ?? 0) - progressCache;
+		const progressActualTotal = (promptProgress?.total ?? 0) - progressCache;
 		const progressPercent = promptProgress
-			? Math.round((promptProgress.processed / promptProgress.total) * 100)
+			? Math.round((progressActualDone / progressActualTotal) * 100)
 			: undefined;
 
 		return {
@@ -324,6 +330,7 @@ class ChatStore {
 			topP: currentConfig.top_p ?? 0.95,
 			speculative: false,
 			progressPercent,
+			promptProgress,
 			promptTokens,
 			promptMs,
 			cacheTokens
@@ -534,7 +541,7 @@ class ChatStore {
 					conversationsStore.updateMessageAtIndex(idx, { toolCalls: streamedToolCallContent });
 				},
 				onModel: (modelName: string) => recordModel(modelName),
-				onTimings: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => {
+				onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => {
 					const tokensPerSecond =
 						timings?.predicted_ms && timings?.predicted_n
 							? (timings.predicted_n / timings.predicted_ms) * 1000
@@ -1032,7 +1039,7 @@ class ChatStore {
 						});
 					},
 
-					onTimings: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => {
+					onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => {
 						const tokensPerSecond =
 							timings?.predicted_ms && timings?.predicted_n
 								? (timings.predicted_n / timings.predicted_ms) * 1000

tools/server/webui/src/lib/types/api.d.ts

@@ -186,6 +186,7 @@ export interface ApiChatCompletionRequest {
 	}>;
 	stream?: boolean;
 	model?: string;
+	return_progress?: boolean;
 	// Reasoning parameters
 	reasoning_format?: string;
 	// Generation parameters
@@ -341,6 +342,7 @@ export interface ApiProcessingState {
 	tokensPerSecond?: number;
 	// Progress information from prompt_progress
 	progressPercent?: number;
+	promptProgress?: ChatMessagePromptProgress;
 	promptTokens?: number;
 	promptMs?: number;
 	cacheTokens?: number;

tools/server/webui/src/lib/types/settings.d.ts

@@ -51,7 +51,7 @@ export interface SettingsChatServiceOptions {
 	onReasoningChunk?: (chunk: string) => void;
 	onToolCallChunk?: (chunk: string) => void;
 	onModel?: (model: string) => void;
-	onTimings?: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void;
+	onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void;
 	onComplete?: (
 		response: string,
 		reasoningContent?: string,