diff --git a/common/arg.cpp b/common/arg.cpp index 6f5fe377d5..0a479c6aaa 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2595,6 +2595,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mmproj_use_gpu = value; } ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD")); + add_opt(common_arg( + // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet + {"-mmdev", "--mmproj-device"}, "DEVICE", + "device to use for multimodal projector (none = don't offload, default: auto)\n" + "use --list-devices to see a list of available devices", + [](common_params & params, const std::string & value) { + if (value == "none") { + params.mmproj_use_gpu = false; + params.mmproj_device = nullptr; + return; + } + auto devices = parse_device_list(value); + // parse_device_list pushes nullptr at back so devices is length 2 for single device. + if (devices.size() > 2) { + throw std::invalid_argument("only one device may be specified for mmproj"); + } + params.mmproj_use_gpu = true; + params.mmproj_device = devices.front(); + } + ).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason add_opt(common_arg( {"--image", "--audio", "--video"}, "FILE", "path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n", diff --git a/common/common.h b/common/common.h index d8a16897b8..de49dac9f6 100644 --- a/common/common.h +++ b/common/common.h @@ -581,9 +581,10 @@ struct common_params { // multimodal models (see tools/mtmd) struct common_params_model mmproj; - bool mmproj_use_gpu = true; // use GPU for multimodal model - bool no_mmproj = false; // explicitly disable multimodal model - std::vector image; // path to image file(s) ; TODO: change the name to "media" + bool mmproj_use_gpu = true; // use GPU for multimodal model + ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model + bool no_mmproj = false; // explicitly disable multimodal model + std::vector image; // path to image file(s) ; TODO: change the name to "media" int image_min_tokens = -1; int image_max_tokens = -1; int mtmd_batch_max_tokens = 1024; diff --git a/tools/cli/README.md b/tools/cli/README.md index b3543ed4da..c9cbacafcd 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -162,6 +162,7 @@ | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)
use --list-devices to see a list of available devices
(env: MTMD_BACKEND_DEVICE) | | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b9dd5e8452..45e33042dd 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -186,14 +186,13 @@ struct clip_ctx { throw std::runtime_error("failed to initialize CPU backend"); } if (ctx_params.use_gpu) { - auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE"); - if (backend_name != nullptr) { - backend = ggml_backend_init_by_name(backend_name, nullptr); + if (ctx_params.device != nullptr) { + backend = ggml_backend_dev_init(ctx_params.device, nullptr); if (!backend) { - LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name); + throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", + __func__, ggml_backend_dev_name(ctx_params.device))); } - } - if (!backend) { + } else { backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr); } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index a5b7137752..e07f258156 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -48,6 +48,7 @@ enum clip_flash_attn_type { struct clip_context_params { bool use_gpu; + ggml_backend_dev_t device; enum clip_flash_attn_type flash_attn_type; int image_min_tokens; int image_max_tokens; diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp index b88a16f0f8..2719dae9b2 100644 --- a/tools/mtmd/debug/mtmd-debug.cpp +++ b/tools/mtmd/debug/mtmd-debug.cpp @@ -84,6 +84,7 @@ int main(int argc, char ** argv) { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 07b45b6440..f6c787fdb6 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -154,6 +154,7 @@ struct mtmd_cli_context { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 0f5cb8c7ae..95f17f7af8 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_ mtmd_context_params mtmd_context_params_default() { mtmd_context_params params { /* use_gpu */ true, + /* device */ nullptr, /* print_timings */ true, /* n_threads */ 4, /* image_marker */ nullptr, @@ -564,6 +565,7 @@ struct mtmd_context { clip_context_params ctx_clip_params { /* use_gpu */ ctx_params.use_gpu, + /* device */ ctx_params.device, /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type), /* image_min_tokens */ ctx_params.image_min_tokens, /* image_max_tokens */ ctx_params.image_max_tokens, diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef4f99c0b6..ef88efd316 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data); struct mtmd_context_params { bool use_gpu; + ggml_backend_dev_t device; bool print_timings; int n_threads; const char * image_marker; // deprecated, use media_marker instead diff --git a/tools/server/README.md b/tools/server/README.md index f5d747eee8..93736c3edf 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -178,6 +178,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md
(env: LLAMA_ARG_MMPROJ_URL) | | `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)
(env: LLAMA_ARG_MMPROJ_AUTO) | | `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)
(env: LLAMA_ARG_MMPROJ_OFFLOAD) | +| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)
use --list-devices to see a list of available devices
(env: MTMD_BACKEND_DEVICE) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)
(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)
(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) | @@ -196,11 +197,11 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | -| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | -| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | +| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)
(env: LLAMA_ARG_RERANKING) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 21ff783941..1293c86402 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -998,6 +998,7 @@ private: mtmd_context_params mparams = mtmd_context_params_default(); if (has_mmproj) { mparams.use_gpu = params_base.mmproj_use_gpu; + mparams.device = params_base.mmproj_device; mparams.print_timings = false; mparams.n_threads = params_base.cpuparams.n_threads; mparams.flash_attn_type = params_base.flash_attn_type; diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index fd7522f8de..368123baf5 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -86,6 +86,7 @@ int main(int argc, char ** argv) { mtmd_context_params mtmd_params = mtmd_context_params_default(); mtmd_params.use_gpu = params.mmproj_use_gpu; + mtmd_params.device = params.mmproj_device; mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params)); if (!mctx) { LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str());