From de119ea47f9b51a02aafaa4499ef04584288fc47 Mon Sep 17 00:00:00 2001 From: Dong Chen Date: Wed, 30 Sep 2026 13:02:36 -0700 Subject: [PATCH 1/4] openai: keep tool message content parts in one message FromChatRequest splits array content into one api.Message per part. That is intended for user content, but a tool message is a single result: the split also dropped its tool_call_id and tool name, since those were only copied when the message had tool_calls. Renderers that resolve tool results by ID then see extra, unnamed results; for example, Kimi K3 fails with "tool message 2 has no resolvable tool name" on a tool result with two text parts, which the OpenAI spec allows. Merge the parts of a tool message into one message with the text joined, the images kept, and ToolCallID and ToolName set, matching how function_call_output items are converted for /v1/responses. --- openai/openai.go | 11 +++++++++ openai/openai_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/openai/openai.go b/openai/openai.go index 010481518..efed4e048 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -634,6 +634,7 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api } messages = append(messages, api.Message{Role: msg.Role, Content: content, Thinking: msg.Reasoning, ToolCalls: toolCalls, ToolName: toolName, ToolCallID: msg.ToolCallID}) case []any: + start := len(messages) for _, c := range content { data, ok := c.(map[string]any) if !ok { @@ -682,6 +683,16 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api return nil, errors.New("invalid message format") } } + // a tool message is a single result, so keep its parts together and + // preserve the tool call it answers + if strings.ToLower(msg.Role) == "tool" && len(messages) > start { + tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID} + for _, part := range messages[start:] { + tool.Content += part.Content + tool.Images = append(tool.Images, part.Images...) + } + messages = append(messages[:start], tool) + } // since we might have added multiple messages above, if we have tools // calls we'll add them to the last message if len(messages) > 0 && len(msg.ToolCalls) > 0 { diff --git a/openai/openai_test.go b/openai/openai_test.go index c92e19563..9f32d1df8 100644 --- a/openai/openai_test.go +++ b/openai/openai_test.go @@ -157,6 +157,60 @@ func TestFromChatRequest_WithImage(t *testing.T) { } } +func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { + imgData, _ := base64.StdEncoding.DecodeString(image) + + req := ChatCompletionRequest{ + Model: "test-model", + Messages: []Message{ + {Role: "user", Content: "What's in the screenshot?"}, + { + Role: "assistant", + ToolCalls: []ToolCall{{ + ID: "call_1", + Type: "function", + Function: struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + }{Name: "screenshot", Arguments: "{}"}, + }}, + }, + { + Role: "tool", + ToolCallID: "call_1", + Content: []any{ + map[string]any{"type": "text", "text": "Captured "}, + map[string]any{ + "type": "image_url", + "image_url": map[string]any{"url": prefix + image}, + }, + map[string]any{"type": "text", "text": "screen.png"}, + }, + }, + }, + } + + result, err := FromChatRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(result.Messages) != 3 { + t.Fatalf("expected 3 messages, got %d", len(result.Messages)) + } + + tool := result.Messages[2] + if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" { + t.Errorf("expected tool message for call_1/screenshot, got role=%q id=%q name=%q", tool.Role, tool.ToolCallID, tool.ToolName) + } + if tool.Content != "Captured screen.png" { + t.Errorf("expected content 'Captured screen.png', got %q", tool.Content) + } + if len(tool.Images) != 1 || string(tool.Images[0]) != string(imgData) { + t.Errorf("expected the tool image to be kept, got %d images", len(tool.Images)) + } +} + func TestFromCompleteRequest_Basic(t *testing.T) { temp := float32(0.8) req := CompletionRequest{ From 77cef85de46d999bcc11635b173928ba15fcd816 Mon Sep 17 00:00:00 2001 From: Dong Chen Date: Thu, 1 Oct 2026 10:23:31 -0700 Subject: [PATCH 2/4] openai: keep empty tool results sent as an empty content array A tool message with "content": [] produced no message at all, so the tool call it answers had no result in the rendered prompt. Emit an empty tool message that keeps its ToolCallID and ToolName, and skip it only when it has no parts, call id, or name. --- openai/openai.go | 2 +- openai/openai_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/openai/openai.go b/openai/openai.go index efed4e048..a3cc19a65 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -685,7 +685,7 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api } // a tool message is a single result, so keep its parts together and // preserve the tool call it answers - if strings.ToLower(msg.Role) == "tool" && len(messages) > start { + if strings.ToLower(msg.Role) == "tool" && (len(messages) > start || toolName != "" || msg.ToolCallID != "") { tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID} for _, part := range messages[start:] { tool.Content += part.Content diff --git a/openai/openai_test.go b/openai/openai_test.go index 9f32d1df8..6d72a9907 100644 --- a/openai/openai_test.go +++ b/openai/openai_test.go @@ -211,6 +211,31 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { } } +func TestFromChatRequest_ToolMessageEmptyArrayContent(t *testing.T) { + var req ChatCompletionRequest + if err := json.Unmarshal([]byte(`{"model":"test-model","messages":[ + {"role":"assistant","tool_calls":[{"id":"call_1","type":"function","function":{"name":"screenshot","arguments":"{}"}}]}, + {"role":"tool","tool_call_id":"call_1","content":[]}, + {"role":"tool","content":[]} + ]}`), &req); err != nil { + t.Fatal(err) + } + + result, err := FromChatRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // the empty result without a call id or name is skipped + if len(result.Messages) != 2 { + t.Fatalf("expected 2 messages, got %d", len(result.Messages)) + } + tool := result.Messages[1] + if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" || tool.Content != "" { + t.Errorf("expected empty tool message for call_1/screenshot, got role=%q id=%q name=%q content=%q", tool.Role, tool.ToolCallID, tool.ToolName, tool.Content) + } +} + func TestFromCompleteRequest_Basic(t *testing.T) { temp := float32(0.8) req := CompletionRequest{ From bea35c898464ed250e6067f69fdd98ce588a0055 Mon Sep 17 00:00:00 2001 From: Dong Chen Date: Thu, 1 Oct 2026 19:42:33 -0700 Subject: [PATCH 3/4] openai: keep image positions when merging tool result parts Merging a tool result's parts collected its images separately from the text, so renderers placed them all before the text: "Before: After: " rendered as "Before: After:". Write an [img] placeholder at each image's position so renderContentWithImageTags and chatPrompt put each image back where it was. --- openai/openai.go | 5 +++-- openai/openai_test.go | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/openai/openai.go b/openai/openai.go index a3cc19a65..8a8550977 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -684,11 +684,12 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api } } // a tool message is a single result, so keep its parts together and - // preserve the tool call it answers + // preserve the tool call it answers. an [img] placeholder keeps each + // image at its position among the text parts if strings.ToLower(msg.Role) == "tool" && (len(messages) > start || toolName != "" || msg.ToolCallID != "") { tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID} for _, part := range messages[start:] { - tool.Content += part.Content + tool.Content += part.Content + strings.Repeat("[img]", len(part.Images)) tool.Images = append(tool.Images, part.Images...) } messages = append(messages[:start], tool) diff --git a/openai/openai_test.go b/openai/openai_test.go index 6d72a9907..1b9ec1e7c 100644 --- a/openai/openai_test.go +++ b/openai/openai_test.go @@ -158,7 +158,9 @@ func TestFromChatRequest_WithImage(t *testing.T) { } func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { + const image2 = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==` imgData, _ := base64.StdEncoding.DecodeString(image) + img2Data, _ := base64.StdEncoding.DecodeString(image2) req := ChatCompletionRequest{ Model: "test-model", @@ -179,12 +181,16 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { Role: "tool", ToolCallID: "call_1", Content: []any{ - map[string]any{"type": "text", "text": "Captured "}, + map[string]any{"type": "text", "text": "Before: "}, map[string]any{ "type": "image_url", "image_url": map[string]any{"url": prefix + image}, }, - map[string]any{"type": "text", "text": "screen.png"}, + map[string]any{"type": "text", "text": " After: "}, + map[string]any{ + "type": "image_url", + "image_url": map[string]any{"url": prefix + image2}, + }, }, }, }, @@ -203,11 +209,12 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" { t.Errorf("expected tool message for call_1/screenshot, got role=%q id=%q name=%q", tool.Role, tool.ToolCallID, tool.ToolName) } - if tool.Content != "Captured screen.png" { - t.Errorf("expected content 'Captured screen.png', got %q", tool.Content) + // each image keeps its position among the text parts + if tool.Content != "Before: [img] After: [img]" { + t.Errorf("expected content 'Before: [img] After: [img]', got %q", tool.Content) } - if len(tool.Images) != 1 || string(tool.Images[0]) != string(imgData) { - t.Errorf("expected the tool image to be kept, got %d images", len(tool.Images)) + if len(tool.Images) != 2 || string(tool.Images[0]) != string(imgData) || string(tool.Images[1]) != string(img2Data) { + t.Errorf("expected both tool images to be kept in order, got %d images", len(tool.Images)) } } From 31547a6cd76ed5eeffbb1f6469a5585e6334f674 Mon Sep 17 00:00:00 2001 From: Dong Chen Date: Fri, 2 Oct 2026 20:41:11 -0700 Subject: [PATCH 4/4] openai: split tool results with images instead of adding [img] markers The [img] placeholders mixed internal markers into tool text: the native llama-server path sends them as literal text, so an image-only result gained a text part, and tool output containing [img] or [img-N] moved or dropped image tags. api.Message cannot carry image positions without in-band markers, so stop merging tool results that include images. Text-only tool results, the only array form the Chat Completions spec allows for tool messages, are still merged into one message (now with a strings.Builder). Results with images keep the per-part split in order, which preserves image positions, with the tool call id and name set on every part. --- openai/openai.go | 27 ++++++++++++++-------- openai/openai_test.go | 52 ++++++++++++++++++++++++++++++++----------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/openai/openai.go b/openai/openai.go index 8a8550977..556c2fd8e 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -11,6 +11,7 @@ import ( "fmt" "log/slog" "net/http" + "slices" "strings" "time" @@ -683,16 +684,24 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api return nil, errors.New("invalid message format") } } - // a tool message is a single result, so keep its parts together and - // preserve the tool call it answers. an [img] placeholder keeps each - // image at its position among the text parts - if strings.ToLower(msg.Role) == "tool" && (len(messages) > start || toolName != "" || msg.ToolCallID != "") { - tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID} - for _, part := range messages[start:] { - tool.Content += part.Content + strings.Repeat("[img]", len(part.Images)) - tool.Images = append(tool.Images, part.Images...) + // a tool message is a single result, so every message built from it + // keeps the tool call it answers. text parts are merged into one + // message; parts with images stay split so each image keeps its + // position among the text + if strings.ToLower(msg.Role) == "tool" { + parts := messages[start:] + if slices.ContainsFunc(parts, func(m api.Message) bool { return len(m.Images) > 0 }) { + for i := range parts { + parts[i].ToolName = toolName + parts[i].ToolCallID = msg.ToolCallID + } + } else if len(parts) > 0 || toolName != "" || msg.ToolCallID != "" { + var sb strings.Builder + for _, part := range parts { + sb.WriteString(part.Content) + } + messages = append(messages[:start], api.Message{Role: msg.Role, Content: sb.String(), ToolName: toolName, ToolCallID: msg.ToolCallID}) } - messages = append(messages[:start], tool) } // since we might have added multiple messages above, if we have tools // calls we'll add them to the last message diff --git a/openai/openai_test.go b/openai/openai_test.go index 1b9ec1e7c..59be1bbab 100644 --- a/openai/openai_test.go +++ b/openai/openai_test.go @@ -158,6 +158,30 @@ func TestFromChatRequest_WithImage(t *testing.T) { } func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { + var req ChatCompletionRequest + if err := json.Unmarshal([]byte(`{"model":"test-model","messages":[ + {"role":"assistant","tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{}"}}]}, + {"role":"tool","tool_call_id":"call_1","content":[{"type":"text","text":"sunny [img]"},{"type":"text","text":", 72F [img-99]"}]} + ]}`), &req); err != nil { + t.Fatal(err) + } + + result, err := FromChatRequest(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // text parts are merged into one result; tool text is kept as sent + if len(result.Messages) != 2 { + t.Fatalf("expected 2 messages, got %d", len(result.Messages)) + } + tool := result.Messages[1] + if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "get_weather" || tool.Content != "sunny [img], 72F [img-99]" { + t.Errorf("expected tool message for call_1/get_weather, got role=%q id=%q name=%q content=%q", tool.Role, tool.ToolCallID, tool.ToolName, tool.Content) + } +} + +func TestFromChatRequest_ToolMessageArrayContentWithImages(t *testing.T) { const image2 = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==` imgData, _ := base64.StdEncoding.DecodeString(image) img2Data, _ := base64.StdEncoding.DecodeString(image2) @@ -201,20 +225,22 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - if len(result.Messages) != 3 { - t.Fatalf("expected 3 messages, got %d", len(result.Messages)) + // parts with images stay split in order, each answering the same call + if len(result.Messages) != 6 { + t.Fatalf("expected 6 messages, got %d", len(result.Messages)) } - - tool := result.Messages[2] - if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" { - t.Errorf("expected tool message for call_1/screenshot, got role=%q id=%q name=%q", tool.Role, tool.ToolCallID, tool.ToolName) - } - // each image keeps its position among the text parts - if tool.Content != "Before: [img] After: [img]" { - t.Errorf("expected content 'Before: [img] After: [img]', got %q", tool.Content) - } - if len(tool.Images) != 2 || string(tool.Images[0]) != string(imgData) || string(tool.Images[1]) != string(img2Data) { - t.Errorf("expected both tool images to be kept in order, got %d images", len(tool.Images)) + wantContent := []string{"Before: ", "", " After: ", ""} + wantImages := [][]byte{nil, imgData, nil, img2Data} + for i, msg := range result.Messages[2:] { + if msg.Role != "tool" || msg.ToolCallID != "call_1" || msg.ToolName != "screenshot" { + t.Errorf("part %d: expected tool message for call_1/screenshot, got role=%q id=%q name=%q", i, msg.Role, msg.ToolCallID, msg.ToolName) + } + if msg.Content != wantContent[i] { + t.Errorf("part %d: expected content %q, got %q", i, wantContent[i], msg.Content) + } + if wantImages[i] == nil && len(msg.Images) != 0 || wantImages[i] != nil && (len(msg.Images) != 1 || string(msg.Images[0]) != string(wantImages[i])) { + t.Errorf("part %d: unexpected images, got %d", i, len(msg.Images)) + } } }