From f5acccad63e3698ca5091038bae72686b4212b9f Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Tue, 9 Jun 2026 21:13:48 +0800
Subject: [PATCH] cleanup round 1

---
 CMakeLists.txt        |   1 +
 Makefile              |  54 ++---
 embd_res/klite.embd   | 183 ++++++++-------
 gpttype_adapter.cpp   |  43 +---
 koboldcpp.py          |   2 +-
 otherarch/otherarch.h |   1 -
 tools/mtmd/clip.cpp   | 517 ++----------------------------------------
 tools/mtmd/clip.h     |  14 +-
 tools/mtmd/llava.cpp  | 171 --------------
 tools/quantclip.cpp   |   1 -
 10 files changed, 168 insertions(+), 819 deletions(-)
 delete mode 100644 tools/mtmd/llava.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 14960f9d0..469367299 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -466,6 +466,7 @@ add_library(common2
             tools/mtmd/mtmd.cpp
             tools/mtmd/mtmd.h
             tools/mtmd/mtmd-helper.cpp
+            tools/mtmd/mtmd-image.cpp
             tools/mtmd/mtmd-helper.h
             tools/mtmd/clip.cpp
             tools/mtmd/clip.h
diff --git a/Makefile b/Makefile
index dbea676cf..46776f103 100644
--- a/Makefile
+++ b/Makefile
@@ -594,6 +594,8 @@ mtmd.o: tools/mtmd/mtmd.cpp tools/mtmd/mtmd.h
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 mtmd-helper.o: tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-helper.h
 	$(CXX) $(CXXFLAGS) -c $< -o $@
+mtmd-image.o: tools/mtmd/mtmd-image.cpp tools/mtmd/mtmd-image.h
+	$(CXX) $(CXXFLAGS) -c $< -o $@
 unicode-common.o: common/unicode.cpp common/unicode.h
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 unicode.o: src/unicode.cpp src/unicode.h
@@ -768,31 +770,31 @@ clean:
 	rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o
 
 # useful tools
-main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
+mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
 	$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
+fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
 	$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h mtmd.o mtmd-helper.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
+embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
 	$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
+rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
 	$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 
 ggml/src/ggml-vulkan-shaders.cpp:
@@ -893,11 +895,11 @@ else
 endif
 
 #generated libraries
-koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(DEFAULT_BUILD)
 
 ifdef FAILSAFE_BUILD
-koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS)
+koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS)
 	$(FAILSAFE_BUILD)
 else
 koboldcpp_failsafe:
@@ -905,7 +907,7 @@ koboldcpp_failsafe:
 endif
 
 ifdef NOAVX2_BUILD
-koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS)
+koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS)
 	$(NOAVX2_BUILD)
 else
 koboldcpp_noavx2:
@@ -913,7 +915,7 @@ koboldcpp_noavx2:
 endif
 
 ifdef CUBLAS_BUILD
-koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS)
+koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS)
 	$(CUBLAS_BUILD)
 else
 koboldcpp_cublas:
@@ -921,7 +923,7 @@ koboldcpp_cublas:
 endif
 
 ifdef HIPBLAS_BUILD
-koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS)
+koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS)
 	$(HIPBLAS_BUILD)
 else
 koboldcpp_hipblas:
@@ -929,12 +931,12 @@ koboldcpp_hipblas:
 endif
 
 ifdef VULKAN_BUILD
-koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(VULKAN_BUILD)
 ifdef NOAVX2_BUILD
-koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS)
+koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS)
 	$(VULKAN_BUILD)
-koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS)
+koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS)
 	$(VULKAN_BUILD)
 else
 koboldcpp_vulkan_noavx2:
@@ -952,15 +954,15 @@ koboldcpp_vulkan_failsafe:
 endif
 
 # tools
-quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
-quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
-quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
-quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
-quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
+quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
 quantize_clip: tools/mtmd/clip.cpp tools/quantclip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
 	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
diff --git a/embd_res/klite.embd b/embd_res/klite.embd
index 7e0221d2a..79a9a5b9f 100644
--- a/embd_res/klite.embd
+++ b/embd_res/klite.embd
@@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
 -->
 <head>
 	<script id="init-config">
-	const LITEVER = 335;
+	const LITEVER = 336;
 	const urlParams = new URLSearchParams(window.location.search);
 	var localflag = urlParams.get('local'); //this will be replaced automatically in embedded kcpp
 	const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@@ -13708,6 +13708,18 @@ Current version indicated by LITEVER below.
 				data = { "data": data }; //fix for their bad format
 			}
 
+			//fix for gemini fetch models, truncate name
+			if(data && data.data && desired_oai_ep.toLowerCase().includes("googleapis.com"))
+			{
+				for(let x=0;x<data.data.length;++x)
+				{
+					if(data.data[x].id && data.data[x].id.toLowerCase().startsWith("models/"))
+					{
+						data.data[x].id = data.data[x].id.substring(7);
+					}
+				}
+			}
+
 			if (!data.error && data.data && data.data.length > 0)
 			{
 				update_oai_model_list(data,dropdown,epchoice);
@@ -20930,38 +20942,95 @@ Current version indicated by LITEVER below.
 		return submit_payload;
 	}
 
-	function oai_append_media(mainoaibody)
-	{
-		for(let i=0;i<insertAIVisionImages.length;++i)
-		{
-			let oaiimg = {
-				"type": "image_url",
-				"image_url": {
-					"url": insertAIVisionImages[i]
-				}
-			};
-			mainoaibody.push(oaiimg);
+	//inject actual media into turns that contain media placeholders
+	function oai_append_media(oai_messages) {
+		var insert_image_idx = 0;
+		var insert_audio_idx = 0;
+		var regex = /\n?\(Attached (Image|Audio) \d+\)\n?/g;
+		function makeImagePart(url) {
+			return { type: "image_url", image_url: { url: url } };
 		}
-		for(let i=0;i<insertAIAudioSounds.length;++i)
-		{
-			let curr = insertAIAudioSounds[i];
-			let format = "wav";
-			if(curr.includes("base64,"))
-			{
-				let parts = curr.split("base64,");
-				curr = parts[1];
-				format = (parts[0].includes("mp3")||parts[0].includes("mpeg"))?"mp3":"wav";
+
+		function makeAudioPart(src) {
+			var curr = String(src || "");
+			var format = "wav";
+			if (curr.indexOf("base64,") !== -1) {
+				var parts = curr.split("base64,");
+				var header = parts[0];
+				curr = parts.slice(1).join("base64,");
+				format = header.indexOf("mp3") !== -1 || header.indexOf("mpeg") !== -1 ? "mp3" : "wav";
 			}
-			let oaiimg = {
-				"type": "input_audio",
-				"input_audio": {
-					"data": curr,
-					"format": format
-				}
-			};
-			mainoaibody.push(oaiimg);
+			return { type: "input_audio", input_audio: { data: curr, format: format } };
 		}
-		return mainoaibody;
+
+		function appendMediaFromText(text, out) {
+			var matches = text.match(regex);
+			if (!matches) return text;
+
+			for (var i = 0; i < matches.length; ++i) {
+				var m = matches[i];
+				if (m.indexOf("Image") !== -1) {
+					if (insert_image_idx < insertAIVisionImages.length) {
+						out.push(makeImagePart(insertAIVisionImages[insert_image_idx]));
+						insert_image_idx++;
+					}
+				}
+				else if (m.indexOf("Audio") !== -1) {
+					if (insert_audio_idx < insertAIAudioSounds.length) {
+						out.push(makeAudioPart(insertAIAudioSounds[insert_audio_idx]));
+						insert_audio_idx++;
+					}
+				}
+			}
+
+			return text.replace(regex, "");
+		}
+
+		for (var i = 0; i < oai_messages.length; ++i) {
+			var msg = oai_messages[i];
+			if (!msg || typeof msg !== "object") continue;
+
+			var content = msg.content;
+
+			if (typeof content === "string") {
+				var mediaParts = [];
+				var cleanedText = appendMediaFromText(content, mediaParts);
+
+				if (mediaParts.length > 0) {
+					msg.content = mediaParts;
+					if (cleanedText.length > 0) {
+						msg.content.push({
+							type: "text",
+							text: cleanedText
+						});
+					}
+				} else {
+					msg.content = cleanedText;
+				}
+			}
+			else if (Array.isArray(content)) {
+				var newContent = [];
+				var combinedText = "";
+
+				for (var j = 0; j < content.length; ++j) {
+					var part = content[j];
+
+					if (part && part.type === "text" && typeof part.text === "string") {
+						combinedText += appendMediaFromText(part.text, newContent);
+					} else {
+						newContent.push(part);
+					}
+				}
+
+				if (combinedText.length > 0) {
+					newContent.push({ type: "text", text: combinedText });
+				}
+
+				msg.content = newContent;
+			}
+		}
+
+		return oai_messages;
 	}
 
 	function dispatch_submit_generation(submit_payload, input_was_empty) //if input is not empty, always unban eos
@@ -21011,53 +21080,7 @@ Current version indicated by LITEVER below.
 					submit_payload.messages = [];
 					submit_payload.continue_assistant_turn = true;
 					let mhistory = repack_instruct_history(combinedprompt);
-					let oaiimgs = [];
-					oaiimgs = oai_append_media(oaiimgs);
-					if(mhistory.length>0 && oaiimgs.length>0)
-					{
-						//do we have the same number of images as placeholders? if so, assign each image to the right turns
-						let placeholder_count = 0;
-						const regex = /\n\(Attached (Image|Audio) \d+\)\n/g;
-						for(let i=0;i<mhistory.length;++i)
-						{
-							const matches = mhistory[i].msg.match(regex);
-							const count = matches ? matches.length : 0;
-							placeholder_count += count;
-						}
-						if(placeholder_count==oaiimgs.length)
-						{
-							//exactly same number, so we can put images into matching turns
-							let mediaidx = 0;//for each occurance of the regex in this turn, insert mediaidx from oaiimgs and increment it
-							for (let i = 0; i < mhistory.length; ++i)
-							{
-								let msg = mhistory[i].msg;
-								const matches = msg.match(regex);
-								const count = matches ? matches.length : 0;
-								if(count>0)
-								{
-									msg = msg.replace(regex, '');
-									let newMsg = [{ type: 'text', text: msg }];
-									for (let j = 0; j < count; ++j)
-									{
-										if (mediaidx < oaiimgs.length)
-										{
-											newMsg.push(oaiimgs[mediaidx]);
-											mediaidx++;
-										}
-									}
-									mhistory[i].msg = newMsg;
-								}
-							}
-						}
-						else
-						{
-							mhistory[0].msg = [{type: 'text', text: mhistory[0].msg}];
-							for(let i=0;i<oaiimgs.length;++i)
-							{
-								mhistory[0].msg.push(oaiimgs[i]);
-							}
-						}
-					}
+
 					for(let i=0;i<mhistory.length;++i)
 					{
 						let cturn = { "role": (mhistory[i].myturn?"user":(mhistory[i].issystem?"system":"assistant")), "content": mhistory[i].msg };
@@ -21072,6 +21095,9 @@ Current version indicated by LITEVER below.
 						}
 						submit_payload.messages.push(cturn);
 					}
+
+					submit_payload.messages = oai_append_media(submit_payload.messages);
+
 					if(determine_if_mcp_active())
 					{
 						let senttools = MCPGetAllowedTools();
@@ -21216,7 +21242,6 @@ Current version indicated by LITEVER below.
 								"text": mainoaibody
 							}
 						];
-						mainoaibody = oai_append_media(mainoaibody);
 					}
 
 					oai_payload.messages = [];
@@ -21291,6 +21316,8 @@ Current version indicated by LITEVER below.
 						}
 					}
 
+					oai_payload.messages = oai_append_media(oai_payload.messages);
+
 					if(determine_if_mcp_active())
 					{
 						let senttools = MCPGetAllowedTools();
diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp
index 030dc4dd4..e76ecd537 100644
--- a/gpttype_adapter.cpp
+++ b/gpttype_adapter.cpp
@@ -1997,18 +1997,12 @@ static void load_grammar(const std::string & gammarstr)
     }
 }
 
-static bool kcpp_eval_media(llama_context * ctx_llama, const media_chunk & mediachunk, int n_batch, int * n_past, bool is2d) {
+static bool kcpp_eval_media(llama_context * ctx_llama, const media_chunk & mediachunk, int n_batch, int * n_past) {
     if (mtmd_ctx && mediachunk.mtmd_chunk) {
         llama_pos new_n_past = *n_past;
-        int32_t result = mtmd_helper_eval_chunk_single(
-            mtmd_ctx,
-            ctx_llama,
-            static_cast<const mtmd_input_chunk *>(mediachunk.mtmd_chunk),
-            *n_past,
-            0,
-            n_batch,
-            false,
-            &new_n_past);
+        int32_t   result     = mtmd_helper_eval_chunk_single(mtmd_ctx, ctx_llama,
+                                                             static_cast<const mtmd_input_chunk *>(mediachunk.mtmd_chunk),
+                                                             *n_past, 0, n_batch, false, &new_n_past);
         if (result != 0) {
             fprintf(stderr, "\n%s : failed to eval mtmd media chunk, status %d\n", __func__, result);
             return false;
@@ -2016,25 +2010,8 @@ static bool kcpp_eval_media(llama_context * ctx_llama, const media_chunk & media
         *n_past = new_n_past;
         return true;
     }
-
-    float * img_embd = mediachunk.clp_img_embd;
-    int num_img_tokens = mediachunk.clp_image_tokens;
-    int img_nx = mediachunk.nx;
-    int img_ny = mediachunk.ny;
-    int n_embd_mmproj  = llama_model_n_embd_inp(llama_get_model(ctx_llama));
-    const int image_n_past = *n_past;
-
-    kcpp_embd_batch media_batch = kcpp_embd_batch(img_embd, num_img_tokens, image_n_past, use_mrope, is2d, img_nx, img_ny);
-    for (int i = 0; i < num_img_tokens; i += n_batch) {
-        const int n_eval = std::min(n_batch, num_img_tokens - i);
-        llama_batch batch_embd_view = media_batch.get_view(i, n_eval, n_embd_mmproj);
-        if (llama_decode(ctx_llama, batch_embd_view)) {
-            fprintf(stderr, "\n%s : failed to eval image\n", __func__);
-            return false;
-        }
-    }
-    *n_past += num_img_tokens;
-    return true;
+    fprintf(stderr, "\n%s : Error, MTMD or media chunk is not initialized!\n", __func__);
+    return false;
 }
 
 //given an old GGUF context and a new context that has some middle portion removed,
@@ -4763,11 +4740,6 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
         {
             for(int j=0;j<media_objects[i].mediachunks.size();++j)
             {
-                if(media_objects[i].mediachunks[j].clp_img_embd!=nullptr)
-                {
-                    free(media_objects[i].mediachunks[j].clp_img_embd);
-                    media_objects[i].mediachunks[j].clp_img_embd = nullptr;
-                }
                 if(media_objects[i].mediachunks[j].mtmd_chunk!=nullptr)
                 {
                     mtmd_input_chunk_free(static_cast<mtmd_input_chunk *>(media_objects[i].mediachunks[j].mtmd_chunk));
@@ -6180,8 +6152,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
                                 {
                                     printf("\rProcessing Media Embedding %d (%d tokens)",(i+1), chunk.clp_image_tokens);
                                 }
-                                bool is2d = (media_objects[i].is_audio?false:true);
-                                bool err = kcpp_eval_media(llama_ctx_v4,chunk,kcpp_data->n_batch,&n_past,is2d);
+                                bool err = kcpp_eval_media(llama_ctx_v4,chunk,kcpp_data->n_batch,&n_past);
                                 mediatokensevaled += chunk.clp_image_tokens;
                                 if(!err)
                                 {
diff --git a/koboldcpp.py b/koboldcpp.py
index a616a0a43..ab0cce8ec 100644
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -12036,7 +12036,7 @@ if __name__ == '__main__':
     advparser.add_argument("--maingpu","--main-gpu","-mg", help="Only used in a multi-gpu setup. Sets the index of the main GPU that will be used.",metavar=('[Device ID]'), type=int, default=-1)
     advparser.add_argument("--maxrequestsize", metavar=('[size in MB]'), help="Specify a max request payload size. Any requests to the server larger than this size will be dropped. Do not change if unsure.", type=int, default=32)
     advparser.add_argument("--mcpfile", metavar=('[mcp json file]'), help="Specify path to mcp.json which contains the Cladue Desktop compatible MCP server config.", default="")
-    advparser.add_argument("--mmproj", metavar=('[filename]'), help="Select a multimodal projector file for vision models like LLaVA.", default="")
+    advparser.add_argument("--mmproj", metavar=('[filename]'), help="Select a multimodal projector file for vision models.", default="")
     advparser.add_argument("--mmprojcpu","--no-mmproj-offload", help="Force CLIP for Vision mmproj always on CPU.", action='store_true')
     advparser.add_argument("--moecpu","--n-cpu-moe", "-ncmoe", metavar=('[layers affected]'), help="Keep the Mixture of Experts (MoE) weights of the first N layers in the CPU. If no value is provided, applies to all layers.", nargs='?', const=999, type=int, default=0)
     advparser.add_argument("--moeexperts", metavar=('[num of experts]'), help="How many experts to use for MoE models (default=follow gguf)", type=int, default=-1)
diff --git a/otherarch/otherarch.h b/otherarch/otherarch.h
index 351a3991f..39742510d 100644
--- a/otherarch/otherarch.h
+++ b/otherarch/otherarch.h
@@ -515,7 +515,6 @@ struct media_chunk
     void * mtmd_chunk = nullptr; // mtmd_input_chunk, owned by this chunk
     int32_t clp_image_tokens = 0; //holds number of tokens used in this chunk
     int32_t clp_image_positions = 0; //holds number of KV positions used in this chunk
-    float * clp_img_embd = nullptr; //legacy embedding memory, must be freed each use
     int32_t nx = 0; //only used for 2d roped images
     int32_t ny = 0;
 };
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 8d66b0d24..4207fe4fe 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -6,20 +6,10 @@
 
 #include "ggml.h"
 #include "ggml-cpp.h"
-#include "ggml-cpu.h"
 #include "ggml-alloc.h"
 #include "ggml-backend.h"
 #include "gguf.h"
 
-#ifdef GGML_USE_CUDA
-#include "ggml-cuda.h"
-#endif
-#ifdef GGML_USE_METAL
-#include "ggml-metal.h"
-#endif
-#ifdef GGML_USE_VULKAN
-#include "ggml-vulkan.h"
-#endif
 
 #define STB_IMAGE_IMPLEMENTATION
 #include "stb_image.h"
@@ -34,18 +24,17 @@
 #include <cstring>
 #include <fstream>
 #include <map>
-#include <regex>
 #include <stdexcept>
 #include <unordered_set>
 #include <vector>
-#include <sstream>
 #include <cinttypes>
 #include <limits>
-#include <numeric>
 #include <array>
-#include <numeric>
 #include <functional>
+#include <sstream>
 #include <filesystem>
+#include <numeric>
+#include <regex>
 #include <float.h>
 
 #include "clip-model.h"
@@ -198,12 +187,6 @@ static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u
 #endif
 
 
-bool enable_gpu_clip = true;
-void set_clip_uses_gpu(bool usegpu)
-{
-    enable_gpu_clip = usegpu;
-}
-
 struct clip_ctx {
     clip_model model;
 
@@ -235,22 +218,22 @@ struct clip_ctx {
     clip_ctx(clip_context_params & ctx_params) {
         flash_attn_type = ctx_params.flash_attn_type;
         no_alloc = ctx_params.no_alloc;
-	backend_cpu = ggml_backend_cpu_init(); //always has CPU backend
+        backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
         if (!backend_cpu) {
             throw std::runtime_error("failed to initialize CPU backend");
         }
-
-        if(enable_gpu_clip)
-        {
-        #ifdef GGML_USE_CUDA
-            backend = ggml_backend_cuda_init(0);
-        #endif
-        #ifdef GGML_USE_METAL
-            backend = ggml_backend_metal_init();
-        #endif
-        #ifdef GGML_USE_VULKAN
-            backend = ggml_backend_vk_init(0);
-        #endif
+        if (ctx_params.use_gpu) {
+            auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE");
+            if (backend_name != nullptr) {
+                backend = ggml_backend_init_by_name(backend_name, nullptr);
+                if (!backend) {
+                    LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name);
+                }
+            }
+            if (!backend) {
+                backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);
+                backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);
+            }
         }
 
         if (backend) {
@@ -3258,439 +3241,6 @@ void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny
     memcpy(img->buf.data(), rgb_pixels, img->buf.size());
 }
 
-//note that the memory here must be subsequently freed!
-uint8_t* make_new_letterbox_img(uint8_t* input_image, int nx, int ny, int nc, int target_width, int target_height) {
-    int new_image_size = (target_width * target_height * nc) + 512; //add some padding
-    if (target_width < nx || target_height < ny) {
-        printf("\nERROR: Target size smaller than input image\n");
-        return nullptr;
-    }
-    uint8_t* letterboxed_image = (uint8_t*)malloc(new_image_size);
-    if(letterboxed_image==nullptr)
-    {
-        printf("\nWARNING: make_new_letterbox_img MALLOC FAILED\n");
-        return nullptr;
-    }
-    memset(letterboxed_image, 0, new_image_size);  // fill with black (0) or any color you prefer
-    int offset_x = (target_width - nx) / 2;
-    int offset_y = (target_height - ny) / 2;
-    for (int y = 0; y < ny; ++y) {
-        memcpy(
-            letterboxed_image + ((y + offset_y) * target_width + offset_x) * nc,
-            input_image + (y * nx * nc),
-            nx * nc
-        );
-    }
-    return letterboxed_image;
-}
-uint8_t* scale_down_image(uint8_t* input_image, int& nx, int& ny, int nc, int max_width, int max_height) {
-    float aspect_ratio = static_cast<float>(nx) / ny;
-    int new_width = nx;
-    int new_height = ny;
-    if (nx > max_width || ny > max_height) {
-        if (aspect_ratio > 1.0f) { // wider than tall
-            new_width = max_width;
-            new_height = static_cast<int>(max_width / aspect_ratio);
-        } else { // taller than wide
-            new_height = max_height;
-            new_width = static_cast<int>(max_height * aspect_ratio);
-        }
-    }
-    uint8_t* resized_image = (uint8_t*)malloc(new_width * new_height * nc);
-    int resok = stbir_resize_uint8(input_image, nx, ny, 0, resized_image, new_width, new_height, 0, nc);
-    if (!resok) {
-        printf("\nKCPP SD: clip resize image failed!\n");
-        free(resized_image);
-        return nullptr;
-    }
-    nx = new_width;
-    ny = new_height;
-    return resized_image;
-}
-
-bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img, const int maxdims) {
-    int nx, ny, nc;
-    auto * data = stbi_load_from_memory(bytes, bytes_length, &nx, &ny, &nc, 3);
-    if (!data) {
-        LOG_ERR("%s: failed to decode image bytes\n", __func__);
-        return false;
-    }
-
-    float maxaspect = 4.0f;
-
-    //check if image needs downscaling
-    if (nx > maxdims || ny > maxdims) {
-        printf("\nImage requires resizing: original size %d x %d scaling to max %d px\n",nx,ny,maxdims);
-        uint8_t* resized_image = scale_down_image(data, nx, ny, 3, maxdims, maxdims);
-        if(resized_image!=nullptr)
-        {
-            stbi_image_free(data); // Free the original image buffer and assign the new one
-            data = resized_image;
-            printf("Resized to clamped to %d x %d\n",nx,ny);
-        }
-    }
-
-    float aspect_ratio = static_cast<float>(nx) / ny;
-    int new_width = nx;
-    int new_height = ny;
-    bool need_letterbox = false;
-    // Check if the image exceeds the aspect ratio limits
-    if (aspect_ratio > maxaspect) {
-        new_height = (int)(nx / maxaspect);
-        need_letterbox = true;
-    } else if (aspect_ratio < 1.0f / maxaspect) {
-        new_width = (int)(ny / maxaspect);
-        need_letterbox = true;
-    }
-
-    if (need_letterbox) {
-        printf("\nImage requires letterboxing: %d x %d changed to %d x %d\n",nx,ny,new_width, new_height);
-        uint8_t* letterboxed_image = make_new_letterbox_img(data, nx, ny, nc, new_width, new_height);
-        if(letterboxed_image!=nullptr)
-        {
-            clip_build_img_from_pixels(letterboxed_image, new_width, new_height, img);
-            free(letterboxed_image);
-            letterboxed_image = nullptr;
-        }
-        else
-        {
-            //letterboxing failed. just use original image
-            clip_build_img_from_pixels(data, nx, ny, img);
-        }
-    }
-    else
-    {
-        clip_build_img_from_pixels(data, nx, ny, img);
-    }
-    stbi_image_free(data);
-    return true;
-}
-
-//kcpp: ported wholesale from mtmd.cpp init_vision() function for easy maintenance.
-//the idea is we just copy init_vision's contents over whenever it updates, what we want is the image_preproc
-#include "mtmd-image.cpp"
-std::unique_ptr<mtmd_image_preprocessor> image_preproc; //only valuable thing we need
-void setup_init_vision_shim_kcpp(struct clip_ctx * ctx_v) {
-
-    //useless vars that do nothing but allow compile
-    enum fake_mtmd_slice_tmpl {
-        MTMD_SLICE_TMPL_NONE,
-        MTMD_SLICE_TMPL_MINICPMV_2_5,
-        MTMD_SLICE_TMPL_MINICPMV_2_6,
-        MTMD_SLICE_TMPL_LLAMA4,
-        MTMD_SLICE_TMPL_IDEFICS3,
-        MTMD_SLICE_TMPL_LFM2,
-        MTMD_SLICE_TMPL_STEP3VL,
-    };
-    auto lookup_token = [](const std::string & token_text) -> llama_token {
-        return -1;
-    };
-    fake_mtmd_slice_tmpl slice_tmpl = MTMD_SLICE_TMPL_NONE;
-    std::vector<llama_token> tok_ov_img_start;  // overview image
-    std::vector<llama_token> tok_ov_img_end;    // overview image
-    std::vector<llama_token> tok_slices_start;  // start of all slices
-    std::vector<llama_token> tok_slices_end;    // end of all slices
-    std::vector<llama_token> tok_sli_img_start; // single slice start
-    std::vector<llama_token> tok_sli_img_end;   // single slice end
-    std::vector<llama_token> tok_sli_img_mid;   // between 2 slices
-    std::vector<llama_token> tok_row_end;       // end of row
-    bool tok_row_end_trail = false;
-    bool ov_img_first      = false;
-    std::string img_beg;
-    std::string img_end;
-    std::string sli_img_start_tmpl;
-    //end of useless block
-
-    // =====
-    // actual llama.cpp transplanted code from init_vision, starts below here!!! do not modify!
-        GGML_ASSERT(ctx_v != nullptr);
-        image_preproc.reset();
-
-        projector_type proj = clip_get_projector_type(ctx_v);
-
-        switch (proj) {
-            case PROJECTOR_TYPE_MLP:
-            case PROJECTOR_TYPE_MLP_NORM:
-            case PROJECTOR_TYPE_LDP:
-            case PROJECTOR_TYPE_LDPV2:
-            case PROJECTOR_TYPE_COGVLM:
-            case PROJECTOR_TYPE_JANUS_PRO:
-            case PROJECTOR_TYPE_GLM_EDGE:
-                {
-                    bool has_pinpoints = !clip_get_hparams(ctx_v)->image_res_candidates.empty();
-                    if (has_pinpoints) {
-                        image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
-                    } else {
-                        image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
-                    }
-                } break;
-            case PROJECTOR_TYPE_MINICPMV:
-                {
-                    int minicpmv_version = clip_get_hparams(ctx_v)->minicpmv_version;
-                    if (minicpmv_version == 2) {
-                        // minicpmv 2.5 format:
-                        // <image> (overview) </image><slice><image> (slice) </image><image> (slice) </image>\n ... </slice>
-                        slice_tmpl        = MTMD_SLICE_TMPL_MINICPMV_2_5;
-                        tok_ov_img_start  = {lookup_token("<image>")};
-                        tok_ov_img_end    = {lookup_token("</image>")};
-                        tok_slices_start  = {lookup_token("<slice>")};
-                        tok_slices_end    = {lookup_token("</slice>")};
-                        tok_sli_img_start = tok_ov_img_start;
-                        tok_sli_img_end   = tok_ov_img_end;
-                        tok_row_end       = {lookup_token("\n")};
-                        tok_row_end_trail = false; // no trailing end-of-row token
-                        ov_img_first      = true;
-                    } else if (minicpmv_version == 3 || minicpmv_version == 4 || minicpmv_version == 5 || minicpmv_version == 6 || minicpmv_version == 100045) {
-                        // minicpmv 2.6 format:
-                        // <image> (overview) </image><slice> (slice) </slice><slice> (slice) </slice>\n ...
-                        slice_tmpl        = MTMD_SLICE_TMPL_MINICPMV_2_6;
-                        tok_ov_img_start  = {lookup_token("<image>")};
-                        tok_ov_img_end    = {lookup_token("</image>")};
-                        tok_sli_img_start = {lookup_token("<slice>")};
-                        tok_sli_img_end   = {lookup_token("</slice>")};
-                        tok_row_end       = {lookup_token("\n")};
-                        tok_row_end_trail = false; // no trailing end-of-row token
-                        ov_img_first      = true;
-
-                    } else if (minicpmv_version != 0) {
-                        throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version));
-                    }
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_MINICPMV4_6:
-                {
-                    slice_tmpl        = MTMD_SLICE_TMPL_MINICPMV_2_6;
-                    tok_ov_img_start  = {lookup_token("<image>")};
-                    tok_ov_img_end    = {lookup_token("</image>")};
-                    tok_sli_img_start = {lookup_token("<slice>")};
-                    tok_sli_img_end   = {lookup_token("</slice>")};
-                    tok_row_end       = {lookup_token("\n")};
-                    tok_row_end_trail = false; // no trailing end-of-row token
-                    ov_img_first      = true;
-                    image_preproc     = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_QWEN2VL:
-            case PROJECTOR_TYPE_QWEN25VL:
-            case PROJECTOR_TYPE_QWEN3VL:
-            case PROJECTOR_TYPE_MIMOVL:
-                {
-                    // <|vision_start|> ... (image embeddings) ... <|vision_end|>
-                    img_beg = "<|vision_start|>";
-                    img_end = "<|vision_end|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_YOUTUVL:
-                {
-                    // <|vision_start|> ... (image embeddings) ... <|vision_end|>
-                    img_beg = "<|vision_start|>";
-                    img_end = "<|vision_end|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_youtuvl>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_YASA2:
-                {
-                    img_beg = "<image>";
-                    img_end = "</image>";
-                    // Currently only supprots single-tile preprocessing: any input is downscaled
-                    // to one image_size x image_size tile (64 output tokens via 8x8 adaptive avg
-                    // pool).
-                    // However, the model itself supports llava-uhd multi-tile tiling for high-res
-                    // images. This will be implemented in a future PR (dispatch on has_pinpoints
-                    // - see LDP/COGVLM branch above) and emit image_grid_pinpoints in the conversion
-                    // script.
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_GEMMA3:
-            case PROJECTOR_TYPE_GEMMA3NV:
-                {
-                    // <start_of_image> ... (image embeddings) ... <end_of_image>
-                    img_beg = "<start_of_image>";
-                    img_end = "<end_of_image>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_IDEFICS3:
-                {
-                    // https://github.com/huggingface/transformers/blob/a42ba80fa520c784c8f11a973ca9034e5f859b79/src/transformers/models/idefics3/processing_idefics3.py#L192-L215
-                    slice_tmpl         = MTMD_SLICE_TMPL_IDEFICS3;
-                    tok_ov_img_start   = {lookup_token("\n\n"), lookup_token("<fake_token_around_image>"), lookup_token("<global-img>")};
-                    tok_ov_img_end     = {lookup_token("<fake_token_around_image>")};
-                    tok_row_end        = {lookup_token("\n")};
-                    sli_img_start_tmpl = "<fake_token_around_image><row_%d_col_%d>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_idefics3>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_PIXTRAL:
-                {
-                    // https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md
-                    img_end = "[IMG_END]";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_PHI4:
-                {
-                    // Phi-4 uses media marker insertion only. Keep image boundary text empty.
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_LLAMA4:
-                {
-                    // (more details in mtmd_context constructor)
-                    img_beg = "<|image_start|>";
-                    img_end = "<|image_end|>";
-                    LOG_WRN("%s: llama 4 vision is known to have degraded quality:\n"
-                            "    https://github.com/ggml-org/llama.cpp/pull/13282\n", __func__);
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_STEP3VL:
-                {
-                    // Step3 format:
-                    //   <patch_start> (patch) <patch_end> [<patch_newline>]
-                    //   ... (all patch rows)
-                    //   <im_start> (overview) <im_end>
-                    slice_tmpl        = MTMD_SLICE_TMPL_STEP3VL;
-                    tok_ov_img_start  = {lookup_token("<im_start>")};
-                    tok_ov_img_end    = {lookup_token("<im_end>")};
-                    tok_sli_img_start = {lookup_token("<patch_start>")};
-                    tok_sli_img_end   = {lookup_token("<patch_end>")};
-                    tok_row_end       = {lookup_token("<patch_newline>")};
-                    tok_row_end_trail = false;
-                    ov_img_first      = false; // patches first, overview last
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_step3vl>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_INTERNVL:
-                {
-                    // <img> ... (image embeddings) ... </img>
-                    img_beg = "<img>";
-                    img_end = "</img>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_internvl>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_KIMIVL:
-                {
-                    // <|media_start|> ... (image embeddings) ... <|media_end|>
-                    img_beg = "<|media_start|>";
-                    img_end = "<|media_end|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_KIMIK25:
-                {
-                    // <|media_begin|> ... (image embeddings) ... <|media_end|>
-                    img_beg = "<|media_begin|>";
-                    img_end = "<|media_end|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_LIGHTONOCR:
-                {
-                    // <|im_start|> ... (image embeddings) ... <|im_end|>
-                    img_beg = "<|im_start|>";
-                    img_end = "<|im_end|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_DOTS_OCR:
-                {
-                    // <|img|> ... (image embeddings) ... <|endofimg|>
-                    img_beg = "<|img|>";
-                    img_end = "<|endofimg|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_NEMOTRON_V2_VL:
-                {
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_LFM2:
-                {
-                    // multi-tile:
-                    //   <|image_start|>
-                    //     <|img_row_1_col_1|> (tile) <|img_row_1_col_2|> (tile) ...
-                    //     <|img_thumbnail|> (thumbnail)
-                    //   <|image_end|>
-                    // single-tile:
-                    //   <|image_start|> (image) <|image_end|>
-                    img_beg            = "<|image_start|>";
-                    img_end            = "<|image_end|>";
-                    slice_tmpl         = MTMD_SLICE_TMPL_LFM2;
-                    sli_img_start_tmpl = "<|img_row_%d_col_%d|>";
-                    tok_ov_img_start   = {lookup_token("<|img_thumbnail|>")};
-                    ov_img_first       = false;
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_lfm2>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_GLM4V:
-                {
-                    // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
-                    img_beg = "<|begin_of_image|>";
-                    img_end = "<|end_of_image|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_PADDLEOCR:
-                {
-                    // <|IMAGE_START|> ... (image embeddings) ... <|IMAGE_END|>
-                    img_beg = "<|IMAGE_START|>";
-                    img_end = "<|IMAGE_END|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_GEMMA4V:
-            case PROJECTOR_TYPE_GEMMA4UV:
-                {
-                    // <|image> ... (image embeddings) ... <image|>
-                    img_beg = "<|image>";
-                    img_end = "<image|>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_DEEPSEEKOCR:
-                {
-                    img_end = "\n"; // prevent empty batch on llama-server
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_DEEPSEEKOCR2:
-                {
-                    img_end = "\n"; // prevent empty batch on llama-server
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr2>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_HUNYUANVL:
-                {
-                    // note: these use fullwidth ｜ (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary
-                    img_beg = "<｜hy_place▁holder▁no▁100｜>";
-                    img_end = "<｜hy_place▁holder▁no▁101｜>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_EXAONE4_5:
-                {
-                    // <vision> ... (image embeddings) ... </vision>
-                    img_beg = "<vision>";
-                    img_end = "</vision>";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
-            case PROJECTOR_TYPE_GRANITE4_VISION:
-                {
-                    img_beg = "<image>";
-                    img_end = "";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);
-                } break;
-            default:
-                throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));
-        }
-
-        GGML_ASSERT(image_preproc != nullptr);
-    //end of lcpp code block
-    // =====
-}
-
-//kcpp: legacy shim created during upstream PR 21031
-bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, struct clip_image_f32_batch * res_imgs) {
-    if(!image_preproc)
-    {
-        LOG_DBG("\nSetup Clip Vision Preprocessing...\n");
-        setup_init_vision_shim_kcpp(ctx);
-        if(!image_preproc)
-        {
-            LOG_ERR("\nError: image_preproc failed to init!\n");
-            return false;
-        }
-    }
-    bool ok = image_preproc->preprocess(*img, *res_imgs);
-    if (!ok) {
-        LOG_ERR("\nError: Unable to preprocess image\n");
-        return false;
-    }
-
-    return true;
-}
-
 void clip_free(clip_ctx * ctx) {
     if (ctx == nullptr) {
         return;
@@ -4928,8 +4478,14 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
             GGML_ABORT("Unknown projector type");
     }
 
-    if (ggml_backend_is_cpu(ctx->backend)) {
-        ggml_backend_cpu_set_n_threads(ctx->backend, n_threads);
+    // ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads);
+    ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu);
+    ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
+    if (reg) {
+        auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
+        if (ggml_backend_set_n_threads_fn) {
+            ggml_backend_set_n_threads_fn(ctx->backend_cpu, n_threads);
+        }
     }
 
     auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf);
@@ -5286,30 +4842,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
     }
 }
 
-int clip_is_minicpmv(const struct clip_ctx * ctx) {
-    // TODO: remove this function
-    if (ctx->proj_type() == PROJECTOR_TYPE_MINICPMV) {
-        return ctx->model.hparams.minicpmv_version;
-    }
-    if (ctx->proj_type() == PROJECTOR_TYPE_MINICPMV4_6) {
-        return 46;
-    }
-    return 0;
-}
-
-bool clip_is_glm(const struct clip_ctx * ctx) {
-    // TODO: remove this function
-    return ctx->proj_type() == PROJECTOR_TYPE_GLM_EDGE;
-}
-
 bool clip_is_llava(const struct clip_ctx * ctx) {
     return ctx->model.hparams.has_llava_projector;
 }
 
-bool clip_is_gemma4v(const struct clip_ctx * ctx) { //for kcpp use
-    return (ctx->proj_type() == PROJECTOR_TYPE_GEMMA4V || ctx->proj_type() == PROJECTOR_TYPE_GEMMA4UV);
-}
-
 bool clip_has_vision_encoder(const struct clip_ctx * ctx) {
     return ctx->model.modality == CLIP_MODALITY_VISION;
 }
@@ -5339,11 +4875,6 @@ projector_type clip_get_projector_type(const struct clip_ctx * ctx) {
     return ctx->proj_type();
 }
 
-int clip_get_projector_type_ext(clip_ctx * ctx) {
-    return ctx->proj_type();
-}
-
-
 void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel, int n_frames, float * mel) {
     clip_image_f32 * audio = new clip_image_f32;
     audio->nx = n_frames;
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
index 09d50fba9..7ce2b683b 100644
--- a/tools/mtmd/clip.h
+++ b/tools/mtmd/clip.h
@@ -100,21 +100,12 @@ struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch
  */
 void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img);
 
-/** interpret bytes as an image file with length bytes_length, and use the result to populate img */
-bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img, const int maxdims);
-
-/** preprocess img and store the result in res_imgs, pad_to_square may be overridden to false depending on model configuration */
-bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs );
-
 bool clip_image_encode      (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
 bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
 
-int clip_is_minicpmv(const struct clip_ctx * ctx);
-bool clip_is_glm(const struct clip_ctx * ctx);
-bool clip_is_gemma4v(const struct clip_ctx * ctx);
 bool clip_is_llava(const struct clip_ctx * ctx);
-void set_clip_uses_gpu(bool usegpu);
-int clip_get_projector_type_ext(clip_ctx * ctx);
+// note for contributor: this clip_is_(model) pattern is deprecated
+//                       do NOT add new functions like this
 
 bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec);
 
@@ -123,7 +114,6 @@ void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel
 
 bool clip_has_vision_encoder(const struct clip_ctx * ctx);
 bool clip_has_audio_encoder(const struct clip_ctx * ctx);
-
 bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) ;
 
 std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
diff --git a/tools/mtmd/llava.cpp b/tools/mtmd/llava.cpp
deleted file mode 100644
index 5929f38bb..000000000
--- a/tools/mtmd/llava.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-#include "clip.h"
-#include "clip-impl.h"
-#include "llava.h"
-#include "mtmd-audio.h"
-
-#include "llama.h"
-#include "ggml-cpp.h"
-
-#include <algorithm>
-#include <cerrno>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <limits>
-#include <vector>
-#include <memory>
-
-
-// convenience cpp wrapper
-struct clip_image_f32_batch_deleter {
-    void operator()(clip_image_f32_batch * val) { clip_image_f32_batch_free(val); }
-};
-typedef std::unique_ptr<clip_image_f32_batch, clip_image_f32_batch_deleter> clip_image_f32_batch_ptr;
-
-
-static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, struct clip_image_f32_batch * preprocessed_img, float * image_embd, int * n_img_pos, int *nx, int *ny) {
-
-    const int64_t t_img_enc_start_us = ggml_time_us();
-    const char * mm_patch_merge_type = clip_patch_merge_type(ctx_clip);
-    const size_t n_imgs = clip_image_f32_batch_n_images(preprocessed_img);
-    clip_image_f32 * img_res = clip_image_f32_get_img(preprocessed_img, 0);
-    *n_img_pos = clip_n_output_tokens(ctx_clip, img_res);
-    *nx = clip_n_output_tokens_x(ctx_clip,img_res);
-    *ny = clip_n_output_tokens_y(ctx_clip,img_res);
-    bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd); // image_embd shape is 576 x 4096
-    if (!encoded) {
-        LOG_ERR("Unable to encode image\n");
-        return false;
-    }
-
-    LOG_INF("%s: CLIP output tokens nx:%d, ny:%d\n", __func__, *nx,*ny);
-    LOG_INF("%s: image embedding created: %d tokens\n", __func__, *n_img_pos);
-
-    const int64_t t_img_enc_end_us = ggml_time_us();
-    float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;
-
-    LOG_INF("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);
-
-    return true;
-}
-
-bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out, int * nx_out, int * ny_out, bool clip_is_mrope) {
-    // Granite vision uses up to 10 patches + base patch
-    int num_max_patches = 11;
-    if (clip_is_minicpmv(ctx_clip)) {
-        num_max_patches = 10;
-    }
-    if (clip_is_glm(ctx_clip)) {
-        num_max_patches = 1;
-    }
-    float * image_embd;
-    clip_image_f32_batch_ptr preprocessed_img(clip_image_f32_batch_init());
-    if (!clip_image_preprocess(ctx_clip, img, preprocessed_img.get())) {
-        LOG_ERR("%s: unable to preprocess image\n", __func__);
-        return false;
-    }
-
-    if (clip_is_mrope) {
-        // qwen2vl don't split image into chunks, so `num_max_patches` is not needed.
-        //sometimes they resize the image LARGER than before (padding up), so we must account for that
-        int max_nx = img->nx;
-        int max_ny = img->ny;
-        for(int i=0;i<preprocessed_img->entries.size();++i)
-        {
-            int a = preprocessed_img->entries[i].get()->nx;
-            int b = preprocessed_img->entries[i].get()->ny;
-            max_nx = std::max(max_nx,a);
-            max_ny = std::max(max_ny,b);
-        }
-        image_embd = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, max_nx, max_ny));
-    }
-    else if(clip_is_gemma4v(ctx_clip)) //gemma4 vision
-    {
-        const size_t n_imgs = clip_image_f32_batch_n_images(preprocessed_img.get());
-        clip_image_f32 * img_res = clip_image_f32_get_img(preprocessed_img.get(), 0);
-        int n_img_pos = clip_n_output_tokens(ctx_clip, img_res);
-        int nbytes = n_img_pos * clip_n_mmproj_embd(ctx_clip) * sizeof(float);
-        image_embd = (float *)malloc(nbytes);
-    }
-    else
-    {
-        image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*num_max_patches); // TODO: base on gridsize/llava model
-    }
-    if (!image_embd) {
-        LOG_ERR("Unable to allocate memory for image embeddings\n");
-        return false;
-    }
-
-    int n_img_pos;
-    int nx = 0, ny = 0;
-    if (!encode_image_with_clip(ctx_clip, n_threads, preprocessed_img.get(), image_embd, &n_img_pos, &nx, &ny)) {
-        LOG_ERR("%s: cannot encode image, aborting\n", __func__);
-        free(image_embd);
-        return false;
-    }
-    *image_embd_out = image_embd;
-    *n_img_pos_out = n_img_pos;
-    *nx_out = nx;
-    *ny_out = ny;
-
-    return true;
-}
-
-struct llava_embd_batch {
-    std::vector<llama_pos>      pos;
-    std::vector<int32_t>        n_seq_id;
-    std::vector<llama_seq_id>   seq_id_0;
-    std::vector<llama_seq_id *> seq_ids;
-    std::vector<int8_t>         logits;
-    llama_batch batch;
-    llava_embd_batch(float * embd, int32_t n_tokens, llama_pos pos_0, llama_seq_id seq_id) {
-        pos     .resize(n_tokens);
-        n_seq_id.resize(n_tokens);
-        seq_ids .resize(n_tokens + 1);
-        logits  .resize(n_tokens);
-        seq_id_0.resize(1);
-        seq_id_0[0] = seq_id;
-        seq_ids [n_tokens] = nullptr;
-        batch = {
-            /*n_tokens       =*/ n_tokens,
-            /*tokens         =*/ nullptr,
-            /*embd           =*/ embd,
-            /*pos            =*/ pos.data(),
-            /*n_seq_id       =*/ n_seq_id.data(),
-            /*seq_id         =*/ seq_ids.data(),
-            /*logits         =*/ logits.data(),
-        };
-        for (int i = 0; i < n_tokens; i++) {
-            batch.pos     [i] = pos_0 + i;
-            batch.n_seq_id[i] = 1;
-            batch.seq_id  [i] = seq_id_0.data();
-            batch.logits  [i] = false;
-        }
-    }
-};
-
-
-//kcpp helper function
-bool audio_embd_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const mtmd_audio_mel & mel_spec, float ** image_embd_out, int * n_img_pos_out)
-{
-    clip_image_f32_ptr mel_f32(clip_image_f32_init());
-    mel_f32->nx  = mel_spec.n_len;
-    mel_f32->ny  = mel_spec.n_mel;
-    mel_f32->buf = std::move(mel_spec.data);
-    size_t n_tokens = clip_n_output_tokens(ctx_clip, mel_f32.get());
-
-    clip_image_f32_batch batch_f32;
-    batch_f32.is_audio = true;
-    batch_f32.entries.push_back(std::move(mel_f32));
-
-    int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip);
-    float * audio_embd = (float *)malloc(n_tokens * n_mmproj_embd * sizeof(float));
-    bool ok = clip_image_batch_encode(
-        ctx_clip,
-        n_threads,
-        &batch_f32,
-        audio_embd);
-    *image_embd_out = audio_embd;
-    *n_img_pos_out = n_tokens;
-    return ok;
-}
\ No newline at end of file
diff --git a/tools/quantclip.cpp b/tools/quantclip.cpp
index c816a7b45..9b61a826c 100644
--- a/tools/quantclip.cpp
+++ b/tools/quantclip.cpp
@@ -1,7 +1,6 @@
 #include "ggml.h"
 #include "common.h"
 #include "mtmd/clip.h"
-#include "mtmd/llava.h"
 #include "llama.h"
 
 #include "base64.hpp"