Merge branch 'upstream' into concedo_experimental

# Conflicts:
#	CMakeLists.txt
#	docs/speculative.md
#	ggml/src/ggml-cuda/CMakeLists.txt
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c
#	ggml/src/ggml-hexagon/htp/hmx-ops.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-hexagon/htp/rope-ops.c
#	ggml/src/ggml-hexagon/htp/ssm-conv.c
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	scripts/snapdragon/adb/run-bench.sh
#	scripts/snapdragon/adb/run-cli.sh
#	scripts/snapdragon/adb/run-completion.sh
#	scripts/snapdragon/adb/run-mtmd.sh
#	scripts/snapdragon/windows/run-bench.ps1
#	scripts/snapdragon/windows/run-cli.ps1
#	scripts/snapdragon/windows/run-completion.ps1
#	scripts/snapdragon/windows/run-mtmd.ps1
#	src/llama-vocab.cpp
#	tests/test-backend-ops.cpp
#	tools/batched-bench/CMakeLists.txt
#	tools/batched-bench/batched-bench.cpp
#	tools/cli/CMakeLists.txt
#	tools/cli/README.md
#	tools/cli/cli.cpp
#	tools/completion/CMakeLists.txt
#	tools/completion/README.md
#	tools/llama-bench/CMakeLists.txt
#	tools/llama-bench/llama-bench.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/mtmd/tests/test-deepseek-ocr.py
#	tools/mtmd/tests/tests-requirements.txt
#	tools/perplexity/CMakeLists.txt
#	tools/perplexity/perplexity.cpp
#	tools/quantize/CMakeLists.txt
#	tools/server/CMakeLists.txt
#	tools/server/README.md
#	ty.toml

tools/ui/src/lib/constants/image-size.ts

@@ -0,0 +1 @@
+export const MEGAPIXELS_TO_PIXELS = 1_000_000;

tools/ui/src/lib/constants/settings-keys.ts

@@ -18,6 +18,7 @@ export const SETTINGS_KEYS = {
 	TITLE_GENERATION_USE_FIRST_LINE: 'titleGenerationUseFirstLine',
 	TITLE_GENERATION_USE_LLM: 'titleGenerationUseLLM',
 	TITLE_GENERATION_PROMPT: 'titleGenerationPrompt',
+	MAX_IMAGE_RESOLUTION: 'maxImageMPixels',
 	// Display
 	SHOW_MESSAGE_STATS: 'showMessageStats',
 	SHOW_THOUGHT_IN_PROGRESS: 'showThoughtInProgress',

tools/ui/src/lib/constants/settings-registry.ts

@@ -193,6 +193,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 				defaultValue: TITLE_GENERATION.DEFAULT_PROMPT,
 				type: SettingsFieldType.TEXTAREA,
 				section: SETTINGS_SECTION_SLUGS.GENERAL
+			},
+			{
+				key: SETTINGS_KEYS.MAX_IMAGE_RESOLUTION,
+				label: 'Maximum image resolution (megapixels)',
+				help: 'Images larger than this will be resized before sending to server. Set to 0 to disable.',
+				defaultValue: 0,
+				type: SettingsFieldType.INPUT,
+				section: SETTINGS_SECTION_SLUGS.GENERAL
 			}
 		]
 	},

tools/ui/src/lib/constants/uri-template.ts

@@ -55,3 +55,6 @@ export const VARIABLE_PREFIX_MODIFIER_REGEX = /:[\d]+$/;
 
 /** Regex to strip one or more leading slashes */
 export const LEADING_SLASHES_REGEX = /^\/+/;
+
+/** Regex to match base64-encoded image URIs (format: "data:image/[media type];base64,[data]")*/
+export const BASE64_IMAGE_URI_REGEX = /^data:(image\/[a-z0-9.\-+]+);base64/;

tools/ui/src/lib/enums/files.enums.ts

@@ -183,6 +183,10 @@ export enum MimeTypeAudio {
 	MP3 = 'audio/mp3',
 	MP4 = 'audio/mp4',
 	WAV = 'audio/wav',
+	WAVE = 'audio/wave',
+	X_WAV = 'audio/x-wav',
+	X_WAVE = 'audio/x-wave',
+	X_PN_WAV = 'audio/x-pn-wav',
 	WEBM = 'audio/webm',
 	WEBM_OPUS = 'audio/webm;codecs=opus'
 }

tools/ui/src/lib/services/chat.service.ts

@@ -5,12 +5,15 @@ import {
 	ATTACHMENT_LABEL_PDF_FILE,
 	ATTACHMENT_LABEL_MCP_PROMPT,
 	ATTACHMENT_LABEL_MCP_RESOURCE,
-	LEGACY_AGENTIC_REGEX
+	LEGACY_AGENTIC_REGEX,
+	SETTINGS_KEYS
 } from '$lib/constants';
 import {
 	AttachmentType,
 	ContentPartType,
+	FileTypeAudio,
 	MessageRole,
+	MimeTypeAudio,
 	ReasoningFormat,
 	UrlProtocol
 } from '$lib/enums';
@@ -19,8 +22,31 @@ import type {
 	ApiChatMessageData,
 	ApiChatCompletionToolCall
 } from '$lib/types/api';
-import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types';
+import type {
+	AudioInputFormat,
+	DatabaseMessageExtraMcpPrompt,
+	DatabaseMessageExtraMcpResource
+} from '$lib/types';
 import { modelsStore } from '$lib/stores/models.svelte';
+import { settingsStore } from '../stores/settings.svelte';
+import { capImageDataURLSize } from '../utils/cap-img-size';
+import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
+
+function getAudioInputFormat(mimeType: string): AudioInputFormat {
+	const normalizedMimeType = mimeType.trim().toLowerCase();
+
+	if (
+		normalizedMimeType === MimeTypeAudio.WAV ||
+		normalizedMimeType === MimeTypeAudio.WAVE ||
+		normalizedMimeType === MimeTypeAudio.X_WAV ||
+		normalizedMimeType === MimeTypeAudio.X_WAVE ||
+		normalizedMimeType === MimeTypeAudio.X_PN_WAV
+	) {
+		return FileTypeAudio.WAV;
+	}
+
+	return FileTypeAudio.MP3;
+}
 
 export class ChatService {
 	/**
@@ -134,26 +160,28 @@ export class ChatService {
 			continueFinalMessage
 		} = options;
 
-		const normalizedMessages: ApiChatMessageData[] = messages
-			.map((msg) => {
-				if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
-					const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] };
+		const normalizedMessages: ApiChatMessageData[] = (
+			await Promise.all(
+				messages.map((msg) => {
+					if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
+						const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] };
 
-					return ChatService.convertDbMessageToApiChatMessageData(dbMsg);
-				} else {
-					return msg as ApiChatMessageData;
-				}
-			})
-			.filter((msg) => {
-				// Filter out empty system messages
-				if (msg.role === MessageRole.SYSTEM) {
-					const content = typeof msg.content === 'string' ? msg.content : '';
+						return ChatService.convertDbMessageToApiChatMessageData(dbMsg);
+					} else {
+						return msg as ApiChatMessageData;
+					}
+				})
+			)
+		).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
+			// Filter out empty system messages
+			if (msg.role === MessageRole.SYSTEM) {
+				const content = typeof msg.content === 'string' ? msg.content : '';
 
-					return content.trim().length > 0;
-				}
+				return content.trim().length > 0;
+			}
 
-				return true;
-			});
+			return true;
+		});
 
 		// Filter out image attachments if the model doesn't support vision
 		if (options.model && !modelsStore.modelSupportsVision(options.model)) {
@@ -382,25 +410,27 @@ export class ChatService {
 		excludeReasoning?: boolean,
 		signal?: AbortSignal
 	): Promise<void> {
-		const normalizedMessages: ApiChatMessageData[] = messages
-			.map((msg) => {
-				if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
-					return ChatService.convertDbMessageToApiChatMessageData(
-						msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
-					);
-				}
+		const normalizedMessages: ApiChatMessageData[] = (
+			await Promise.all(
+				messages.map((msg) => {
+					if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
+						return ChatService.convertDbMessageToApiChatMessageData(
+							msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
+						);
+					}
 
-				return msg as ApiChatMessageData;
-			})
-			.filter((msg) => {
-				if (msg.role === MessageRole.SYSTEM) {
-					const content = typeof msg.content === 'string' ? msg.content : '';
+					return msg as ApiChatMessageData;
+				})
+			)
+		).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
+			if (msg.role === MessageRole.SYSTEM) {
+				const content = typeof msg.content === 'string' ? msg.content : '';
 
-					return content.trim().length > 0;
-				}
+				return content.trim().length > 0;
+			}
 
-				return true;
-			});
+			return true;
+		});
 
 		const requestBody: Record<string, unknown> = {
 			messages: normalizedMessages.map((msg: ApiChatMessageData) => {
@@ -787,9 +817,9 @@ export class ChatService {
 	 * @returns {ApiChatMessageData} object formatted for the chat completion API
 	 * @static
 	 */
-	static convertDbMessageToApiChatMessageData(
+	static async convertDbMessageToApiChatMessageData(
 		message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }
-	): ApiChatMessageData {
+	): Promise<ApiChatMessageData> {
 		// Handle tool result messages (role: 'tool')
 		if (message.role === MessageRole.TOOL && message.toolCallId) {
 			return {
@@ -867,9 +897,14 @@ export class ChatService {
 		);
 
 		for (const image of imageFiles) {
+			const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION);
+			let base64Url = image.base64Url;
+			if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) {
+				base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
+			}
 			contentParts.push({
 				type: ContentPartType.IMAGE_URL,
-				image_url: { url: image.base64Url }
+				image_url: { url: base64Url }
 			});
 		}
 
@@ -883,7 +918,7 @@ export class ChatService {
 				type: ContentPartType.INPUT_AUDIO,
 				input_audio: {
 					data: audio.base64Data,
-					format: audio.mimeType.includes('wav') ? 'wav' : 'mp3'
+					format: getAudioInputFormat(audio.mimeType)
 				}
 			});
 		}

tools/ui/src/lib/stores/agentic.svelte.ts

@@ -416,21 +416,23 @@ class AgenticStore {
 
 		console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`);
 
-		const normalizedMessages: ApiChatMessageData[] = messages
-			.map((msg) => {
-				if ('id' in msg && 'convId' in msg && 'timestamp' in msg)
-					return ChatService.convertDbMessageToApiChatMessageData(
-						msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
-					);
-				return msg as ApiChatMessageData;
-			})
-			.filter((msg) => {
-				if (msg.role === MessageRole.SYSTEM) {
-					const content = typeof msg.content === 'string' ? msg.content : '';
-					return content.trim().length > 0;
-				}
-				return true;
-			});
+		const normalizedMessages: ApiChatMessageData[] = (
+			await Promise.all(
+				messages.map((msg) => {
+					if ('id' in msg && 'convId' in msg && 'timestamp' in msg)
+						return ChatService.convertDbMessageToApiChatMessageData(
+							msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
+						);
+					return msg as ApiChatMessageData;
+				})
+			)
+		).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
+			if (msg.role === MessageRole.SYSTEM) {
+				const content = typeof msg.content === 'string' ? msg.content : '';
+				return content.trim().length > 0;
+			}
+			return true;
+		});
 
 		this.updateSession(conversationId, {
 			isRunning: true,

tools/ui/src/lib/types/api.d.ts

@@ -1,6 +1,8 @@
-import type { ContentPartType, ServerModelStatus, ServerRole } from '$lib/enums';
+import type { ContentPartType, FileTypeAudio, ServerModelStatus, ServerRole } from '$lib/enums';
 import type { ChatMessagePromptProgress, ChatRole } from './chat';
 
+export type AudioInputFormat = FileTypeAudio.WAV | FileTypeAudio.MP3;
+
 export interface ApiChatCompletionToolFunction {
 	name: string;
 	description?: string;
@@ -20,7 +22,7 @@ export interface ApiChatMessageContentPart {
 	};
 	input_audio?: {
 		data: string;
-		format: 'wav' | 'mp3';
+		format: AudioInputFormat;
 	};
 	input_video?: {
 		data: string;

tools/ui/src/lib/types/index.ts

@@ -29,7 +29,8 @@ export type {
 	ApiRouterModelsStatusResponse,
 	ApiRouterModelsListResponse,
 	ApiRouterModelsUnloadRequest,
-	ApiRouterModelsUnloadResponse
+	ApiRouterModelsUnloadResponse,
+	AudioInputFormat
 } from './api';
 
 // Chat types

tools/ui/src/lib/utils/cap-img-size.ts

@@ -0,0 +1,73 @@
+import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
+import { BASE64_IMAGE_URI_REGEX } from '$lib/constants/uri-template';
+import { MimeTypeImage } from '$lib/enums';
+
+/**
+ * Converts an Image base64 data URL to another Image data URL with capped dimensions to reduce file size.
+ * @param base64UrlImage - The Image base64 data URL to convert
+ * @param maxMegapixels - The maximum image size in megapixels for the output Image
+ * @returns Promise resolving to Image data URL
+ */
+export function capImageDataURLSize(
+	base64UrlImage: string,
+	maxMegapixels: number
+): Promise<string> {
+	return new Promise((resolve, reject) => {
+		try {
+			
+			const mimeMatch = base64UrlImage.match(BASE64_IMAGE_URI_REGEX);
+			
+			if (!mimeMatch) {
+				return reject(new Error('Invalid data URL format.'));
+			}
+
+			const mimeType = mimeMatch[1] as MimeTypeImage;
+
+			if (!Object.values(MimeTypeImage).includes(mimeType)) {
+				return reject(new Error(`Unsupported image MIME type: ${mimeType}`));
+			}
+
+			const img = new Image();
+
+			img.onload = () => {
+				try {
+					const canvas = document.createElement('canvas');
+					const ctx = canvas.getContext('2d');
+
+					if (!ctx) {
+						throw new Error('Failed to get 2D canvas context.');
+					}
+
+					const targetWidth = img.naturalWidth;
+					const targetHeight = img.naturalHeight;
+					const totalPixels = targetWidth * targetHeight;
+					const maxPixels = Math.floor(maxMegapixels * MEGAPIXELS_TO_PIXELS);
+
+					if (maxPixels > 0 && totalPixels > maxPixels) {
+						const scaleFactor = Math.sqrt(maxPixels / totalPixels);
+						canvas.width = Math.floor(targetWidth * scaleFactor);
+						canvas.height = Math.floor(targetHeight * scaleFactor);
+					} else {
+						return resolve(base64UrlImage);
+					}
+
+					ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+					resolve(canvas.toDataURL(mimeType));
+				} catch (err) {
+					reject(err instanceof Error ? err : new Error(String(err)));
+				}
+			};
+
+			img.onerror = () => {
+				reject(new Error('Failed to load image.'));
+			};
+
+			img.src = base64UrlImage;
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			const errorMessage = `Error resizing image: ${message}`;
+			console.error(errorMessage, error);
+			reject(new Error(errorMessage));
+		}
+	});
+}

tools/ui/src/lib/utils/file-type.ts

@@ -18,8 +18,12 @@ import {
 	MimeTypeText
 } from '$lib/enums';
 
+function normalizeMimeType(mimeType: string): string {
+	return mimeType.trim().toLowerCase();
+}
+
 export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {
-	switch (mimeType) {
+	switch (normalizeMimeType(mimeType)) {
 		// Images
 		case MimeTypeImage.JPEG:
 		case MimeTypeImage.PNG:
@@ -33,6 +37,10 @@ export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {
 		case MimeTypeAudio.MP3:
 		case MimeTypeAudio.MP4:
 		case MimeTypeAudio.WAV:
+		case MimeTypeAudio.WAVE:
+		case MimeTypeAudio.X_WAV:
+		case MimeTypeAudio.X_WAVE:
+		case MimeTypeAudio.X_PN_WAV:
 		case MimeTypeAudio.WEBM:
 		case MimeTypeAudio.WEBM_OPUS:
 			return FileTypeCategory.AUDIO;