fix(api): strip MCP image base64 from tool results in the jinja path (#2374) (#2376)

When a tool/MCP result carries an image, the OpenAI-compatible chat
adapter's jinja code path left the base64 payload in the rendered
prompt as plain text (a single 1024x1024 jpeg bloated the context by
~120k tokens), while the legacy path already stripped it via
strip_mcpcontent_of_media.

- format_jinja now strips the base64 from tool-role string content
  before rendering, matching the legacy path; the image itself is
  still swept out and attached separately.
- sweep_media_from_messages now also recognizes MCP-style image
  content blocks (type == "image") inside a content list, so images
  delivered that way are attached instead of dropped.
This commit is contained in:
Tai An
2026-08-01 19:40:17 -07:00
committed by GitHub
parent da8a5e4461
commit 4423b3af55
+7
View File
@@ -3823,6 +3823,9 @@ def format_jinja(messages_orig, tools, chat_template_kwargs=None):
for m in messages:
if m.get("content") is None:
m["content"] = ""
# strip mcp image b64 from tool string content so it isn't rendered as text (image is swept out separately)
elif m.get("role", "") == "tool" and isinstance(m.get("content"), str):
m["content"] = strip_mcpcontent_of_media(m["content"])
# fix image placeholders, erase them and slap a reference onto the turn text message
mediacount = 1
for m in messages:
@@ -4238,6 +4241,10 @@ def sweep_media_from_messages(messages_array):
url = item.get("image_url", {}).get("url", "")
if url.startswith("data:image"):
images.append(url.split(",", 1)[1])
elif item.get("type") == "image": #handle mcp image content blocks in a list
data = item.get("data", "")
if data:
images.append(data.split(",", 1)[1] if data.startswith("data:") else data)
elif item.get("type") == "input_audio":
data = item.get("input_audio", {}).get("data")
if data: