mirror of
https://github.com/ollama/ollama.git
synced 2026-10-20 18:41:17 +00:00
5a0ff3116d
A format on a thinking model ran two generations: an unconstrained one, cancelled once the parser reported content, then a re-rendered prompt with the parsed thinking under the grammar. The restart cost a second prefill, dropped the chunk that crossed the boundary, needed a harmony prompt hack, stitched metrics across the two requests, and on MLX could leak a stray first token into the JSON. The generate endpoint never deferred at all, so its JSON was forced inside the thinking. Both handlers now make one completion request that names the strings ending the response's thinking, from the builtin parser or the generic thinking parser, and the runner constrains only the content after them in a single generation. The prompt is evaluated once and metrics pass straight through. A raw generate prompt names no strings, since nothing says where its response starts, and its format applies from the first token as before. A format now applies to whatever follows the thinking, so a tool call can no longer take the place of formatted content, which was already the case with thinking off; harmony is the exception, since its tool calls precede the final message. The per-token metrics flag both runners carried for the cancelled first pass has no caller left and goes with the two-pass code and its tests. Fixes #18441 Fixes #17544 Fixes #14196 Fixes #10929
49 lines
2.1 KiB
Go
49 lines
2.1 KiB
Go
package mlxrunner
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
|
|
"github.com/ollama/ollama/llm"
|
|
)
|
|
|
|
func TestRequestGrammar(t *testing.T) {
|
|
schema := `{"type":"object","properties":{"answer":{"type":"string"}}}`
|
|
tag := `{"type":"structural_tag","format":{"type":"json_schema","json_schema":` + schema + `}}`
|
|
for _, tt := range []struct {
|
|
name string
|
|
req llm.CompletionRequest
|
|
want string
|
|
}{
|
|
{name: "unset"},
|
|
{name: "null", req: llm.CompletionRequest{Format: json.RawMessage(`null`)}},
|
|
{name: "empty", req: llm.CompletionRequest{Format: json.RawMessage(`""`)}},
|
|
{
|
|
name: "json",
|
|
req: llm.CompletionRequest{Format: json.RawMessage(`"json"`)},
|
|
want: `{"type":"structural_tag","format":{"type":"json_schema","json_schema":{"type":"object"}}}`,
|
|
},
|
|
{name: "schema", req: llm.CompletionRequest{Format: json.RawMessage(schema)}, want: tag},
|
|
{
|
|
name: "schema after thinking",
|
|
req: llm.CompletionRequest{Format: json.RawMessage(schema), ThinkingClose: []string{"</think>"}},
|
|
want: `{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"any_text","excludes":["</think>"]},` +
|
|
`{"type":"optional","content":{"type":"sequence","elements":[{"type":"const_string","value":"</think>"},` +
|
|
`{"type":"json_schema","json_schema":` + schema + `}]}}]}}`,
|
|
},
|
|
{
|
|
name: "schema after thinking with two closings",
|
|
req: llm.CompletionRequest{Format: json.RawMessage(schema), ThinkingClose: []string{"<|final|>", "<|final|>json"}},
|
|
want: `{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"any_text","excludes":["<|final|>","<|final|>json"]},` +
|
|
`{"type":"optional","content":{"type":"sequence","elements":[{"type":"or","elements":[{"type":"const_string","value":"<|final|>"},{"type":"const_string","value":"<|final|>json"}]},` +
|
|
`{"type":"json_schema","json_schema":` + schema + `}]}}]}}`,
|
|
},
|
|
{name: "thinking without a format", req: llm.CompletionRequest{ThinkingClose: []string{"</think>"}}},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := string(requestGrammar(tt.req)); got != tt.want {
|
|
t.Fatalf("requestGrammar = %s, want %s", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|