diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index d9efc54d8a..6032696a8d 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -525,8 +525,9 @@ const SAMPLING_PARAM_KEYS = ['temperature', 'topP', 'topK', 'presencePenalty', ' /** * Reports the provider request the primary agent resolved, before the fake * model replaced it: the model, which sampling params survived (top level or - * `modelKwargs`), and the context budget left for input. Reads the graph's - * agent context because the hook's `agents` are the unresolved agent configs. + * `modelKwargs`), the `reasoning_effort` it carries, and the context budget + * left for input. Reads the graph's agent context because the hook's `agents` + * are the unresolved agent configs. */ function llmConfigReportResponses(graph) { const agent = graph?.agentContexts?.values().next().value; @@ -538,6 +539,7 @@ function llmConfigReportResponses(graph) { const report = { model: clientOptions.model ?? clientOptions.modelName ?? null, sampling, + reasoningEffort: modelKwargs.reasoning_effort ?? null, maxContextTokens: agent?.maxContextTokens ?? null, }; return { responses: [`E2E llm config: ${JSON.stringify(report)}`] }; diff --git a/e2e/specs/mock/scenarios/gemini-argon.spec.ts b/e2e/specs/mock/scenarios/gemini-argon.spec.ts index a55a7f0d40..932bdfc40d 100644 --- a/e2e/specs/mock/scenarios/gemini-argon.spec.ts +++ b/e2e/specs/mock/scenarios/gemini-argon.spec.ts @@ -8,6 +8,8 @@ import { sendMessageAndWaitForCompletion, } from '../helpers'; import { withMongo } from '../db'; +import { closeMobileDrawer } from '../agents.helpers'; +import { openPanel } from './panels'; const ARGON = 'gemini-4-argon'; const PRE_ARGON = 'gemini-2.5-pro'; @@ -26,6 +28,7 @@ async function selectGeminiModel(page: Page, model: string) { type LlmConfigReport = { model: string | null; sampling: string[]; + reasoningEffort: string | null; maxContextTokens: number | null; }; @@ -39,6 +42,17 @@ async function reportLlmConfig(page: Page): Promise { return JSON.parse(text.slice(text.indexOf('{'))) as LlmConfigReport; } +/** Moves the Parameters panel's Thinking Level slider to its last step, High, as a keyboard user does. */ +async function chooseHighThinkingLevel(page: Page) { + await openPanel(page, 'parameters', 'Parameters'); + const reasoning = page.getByRole('region', { name: 'Reasoning' }); + const slider = reasoning.getByRole('slider', { name: 'Thinking Level' }); + await slider.focus(); + await slider.press('End'); + await expect(reasoning.getByRole('textbox', { name: 'Thinking Level' })).toHaveValue('High'); + await closeMobileDrawer(page); +} + function conversationIdOf(page: Page): string { const match = new URL(page.url()).pathname.match(/^\/c\/([^/]+)\/?$/); expect(match?.[1], 'conversation should have a persisted id').toBeTruthy(); @@ -63,6 +77,28 @@ test('Gemini 4 Argon on a Gemini gateway sends no ignored sampling params @scena expect(argon.maxContextTokens ?? 0).toBeGreaterThan(0); }); +test('A thinking level chosen for Gemini 4 Argon reaches the gateway as reasoning_effort @scenario:argon-gateway-receives-the-chosen-thinking-level', async ({ + page, +}) => { + await selectGeminiModel(page, ARGON); + await chooseHighThinkingLevel(page); + const argon = await reportLlmConfig(page); + + expect(argon.model).toBe('gemini-4-argon'); + expect(argon.reasoningEffort).toBe('high'); +}); + +test('A Gemini 4 Argon turn left on Auto thinking sends no reasoning_effort @scenario:argon-gateway-sends-no-effort-on-auto-thinking', async ({ + page, +}) => { + await selectGeminiModel(page, ARGON); + const argon = await reportLlmConfig(page); + + /** Argon fills in its own default level for the request; the gateway should still pick its default. */ + expect(argon.model).toBe('gemini-4-argon'); + expect(argon.reasoningEffort).toBeNull(); +}); + test('A Gemini 4 Argon turn is billed at Argon rates @scenario:an-argon-turn-is-billed-at-argon-rates', async ({ page, }) => { diff --git a/packages/api/src/endpoints/google/llm.ts b/packages/api/src/endpoints/google/llm.ts index b5b8d47075..f3fc121cdf 100644 --- a/packages/api/src/endpoints/google/llm.ts +++ b/packages/api/src/endpoints/google/llm.ts @@ -499,6 +499,13 @@ export function getGoogleConfig( provider: Providers.VERTEXAI | Providers.GOOGLE; /** @type {GoogleClientOptions | VertexAIClientOptions} */ llmConfig: VertexAIClientOptions | GoogleClientOptions; + /** + * The thinking level in the form Gemini's OpenAI-compatible API takes (`reasoning_effort`), for a + * gateway that cannot read `thinkingConfig`. Set only for a level that was selected or configured + * as a default, never for the default a Flash model fills in, so the model default still applies. + * @see https://ai.google.dev/gemini-api/docs/openai#thinking + */ + reasoningEffort?: string; } { let creds: t.GoogleCredentials = {}; if (acceptRawApiKey && typeof credentials === 'string') { @@ -539,6 +546,11 @@ export function getGoogleConfig( ...modelOptions } = sanitizeModelOptions(options.modelOptions); + /** The level the user selected, else the one the endpoint configures as its default. */ + const selectedThinkingLevel = + normalizeGoogleThinkingLevel(thinkingLevel) ?? + normalizeGoogleThinkingLevel(options.defaultParams?.thinkingLevel); + let enableWebSearch = web_search; let enableUrlContext = url_context; @@ -606,9 +618,8 @@ export function getGoogleConfig( const thinkingConfig: GoogleThinkingConfig = { includeThoughts: true, }; - const normalizedThinkingLevel = normalizeGoogleThinkingLevel(thinkingLevel); - if (normalizedThinkingLevel) { - thinkingConfig.thinkingLevel = normalizedThinkingLevel; + if (selectedThinkingLevel) { + thinkingConfig.thinkingLevel = selectedThinkingLevel; } if (provider === Providers.GOOGLE) { (llmConfig as { thinkingConfig?: GoogleThinkingConfig }).thinkingConfig = thinkingConfig; @@ -800,6 +811,16 @@ export function getGoogleConfig( } } + const resolvedThinkingLevel = (llmConfig as { thinkingConfig?: GoogleThinkingConfig }) + .thinkingConfig?.thinkingLevel; + const reasoningEffort = + selectedThinkingLevel != null && + resolvedThinkingLevel != null && + resolvedThinkingLevel !== 'THINKING_LEVEL_UNSPECIFIED' && + options.dropParams?.includes('thinkingLevel') !== true + ? resolvedThinkingLevel.toLowerCase() + : undefined; + // Return the final shape return { /** @type {GoogleAIToolType[]} */ @@ -808,5 +829,6 @@ export function getGoogleConfig( provider, /** @type {GoogleClientOptions | VertexAIClientOptions} */ llmConfig, + ...(reasoningEffort != null && { reasoningEffort }), }; } diff --git a/packages/api/src/endpoints/openai/config.google.spec.ts b/packages/api/src/endpoints/openai/config.google.spec.ts index 4fea2d1b3d..058f272842 100644 --- a/packages/api/src/endpoints/openai/config.google.spec.ts +++ b/packages/api/src/endpoints/openai/config.google.spec.ts @@ -1,3 +1,4 @@ +import { ReasoningParameterFormat } from 'librechat-data-provider'; import { getOpenAIConfig } from './config'; describe('getOpenAIConfig - Google Compatibility', () => { @@ -575,5 +576,248 @@ describe('getOpenAIConfig - Google Compatibility', () => { expect(result.llmConfig.temperature).toBe(0.8); }); }); + + describe('Thinking level as reasoning_effort', () => { + const apiKey = JSON.stringify({ GOOGLE_API_KEY: 'test-google-key' }); + type ConfigOptions = NonNullable[1]>; + + /** Builds the Google-compatible custom endpoint config for one model selection. */ + function getGeminiConfig( + modelOptions: object, + rest: Omit = {}, + customParams: ConfigOptions['customParams'] = {}, + ) { + const options: ConfigOptions = { + modelOptions, + customParams: { defaultParamsEndpoint: 'google', ...customParams }, + reverseProxyUrl: 'https://generativelanguage.googleapis.com/v1beta/openai', + ...rest, + }; + return getOpenAIConfig(apiKey, options, 'Gemini (Custom)'); + } + + const sentEffort = (result: ReturnType) => + result.llmConfig.modelKwargs?.reasoning_effort; + + it.each([ + ['minimal', 'minimal'], + ['low', 'low'], + ['medium', 'medium'], + ['high', 'high'], + ['HIGH', 'high'], + ])('sends an explicit %s thinking level as reasoning_effort %s', (thinkingLevel, effort) => { + const result = getGeminiConfig({ model: 'gemini-3.1-pro-preview', thinkingLevel }); + + expect(result.llmConfig.modelKwargs).toMatchObject({ reasoning_effort: effort }); + expect(result.llmConfig).not.toHaveProperty('thinkingConfig'); + expect(result.llmConfig.modelKwargs).not.toHaveProperty('thinkingConfig'); + expect(result.llmConfig.modelKwargs).not.toHaveProperty('thinkingLevel'); + }); + + it.each([undefined, '', 'THINKING_LEVEL_UNSPECIFIED'])( + 'sends no reasoning_effort when the thinking level is %j so the model default applies', + (thinkingLevel) => { + const result = getGeminiConfig({ model: 'gemini-3.1-pro-preview', thinkingLevel }); + + expect(sentEffort(result)).toBeUndefined(); + }, + ); + + it.each([ + 'gemini-3.5-flash', + 'gemini-3.5-flash-lite', + 'gemini-3.6-flash', + 'gemini-3.7-flash', + 'gemini-3.8-flash', + 'gemini-4-argon', + 'gemini-4-argon-cyber-permissive', + ])('does not forward the default level a Flash model fills in (%s)', (model) => { + const result = getGeminiConfig({ model }); + + expect(sentEffort(result)).toBeUndefined(); + }); + + it.each([ + ['gemini-4-argon', 'minimal', 'low'], + ['gemini-4-argon', 'high', 'high'], + ['gemini-4-argon-cyber-permissive', 'low', 'low'], + ['gemini-3.8-flash', 'minimal', 'low'], + ['gemini-3.7-flash', 'minimal', 'low'], + ['gemini-3.6-flash', 'minimal', 'minimal'], + ['gemini-3.5-flash-lite', 'minimal', 'minimal'], + ['gemini-3.5-flash', 'low', 'low'], + ])('sends %s thinking level %s as reasoning_effort %s', (model, thinkingLevel, effort) => { + const result = getGeminiConfig({ model, thinkingLevel }); + + expect(sentEffort(result)).toBe(effort); + }); + + it('sends no reasoning_effort when thinking is turned off', () => { + const result = getGeminiConfig({ + model: 'gemini-3.1-pro-preview', + thinking: false, + thinkingLevel: 'high', + }); + + expect(sentEffort(result)).toBeUndefined(); + }); + + it('leaves thinkingBudget unsent and adds no reasoning_effort before Gemini 3', () => { + const result = getGeminiConfig({ + model: 'gemini-2.5-pro', + thinkingBudget: 2048, + thinkingLevel: 'high', + }); + const kwargs = result.llmConfig.modelKwargs ?? {}; + + expect(sentEffort(result)).toBeUndefined(); + expect(result.llmConfig).not.toHaveProperty('thinkingConfig'); + expect(kwargs).not.toHaveProperty('thinkingConfig'); + expect(kwargs).not.toHaveProperty('thinkingBudget'); + }); + + it('lets an addParams reasoning_effort replace the selected thinking level', () => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'low' }, + { addParams: { reasoning_effort: 'high' } }, + ); + + expect(sentEffort(result)).toBe('high'); + }); + + it('keeps an addParams reasoning object instead of adding reasoning_effort beside it', () => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'low' }, + { addParams: { reasoning: { effort: 'medium' } } }, + ); + + expect(result.llmConfig.reasoning).toEqual({ effort: 'medium' }); + expect(sentEffort(result)).toBeUndefined(); + }); + + it.each(['thinking_config', 'thinkingConfig'])( + 'does not send reasoning_effort beside an addParams Google %s', + (key) => { + const extraBody = { google: { [key]: { thinking_level: 'high' } } }; + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'low' }, + { addParams: { extra_body: extraBody } }, + ); + + expect(result.llmConfig.modelKwargs?.extra_body).toEqual(extraBody); + expect(sentEffort(result)).toBeUndefined(); + }, + ); + + it.each([ + ['gemini-3.1-pro-preview', 'reasoning_effort'], + ['gemini-3.1-pro-preview', 'thinkingLevel'], + ['gemini-3.1-pro-preview', 'thinkingConfig'], + ['gemini-3.7-flash', 'reasoning_effort'], + ['gemini-3.7-flash', 'thinkingLevel'], + ['gemini-3.7-flash', 'thinkingConfig'], + ])('sends no reasoning_effort for %s when %s is in dropParams', (model, param) => { + const result = getGeminiConfig({ model, thinkingLevel: 'high' }, { dropParams: [param] }); + + expect(sentEffort(result)).toBeUndefined(); + }); + + it('keeps an addParams reasoning_effort when only thinkingLevel is dropped', () => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'high' }, + { addParams: { reasoning_effort: 'low' }, dropParams: ['thinkingLevel'] }, + ); + + expect(sentEffort(result)).toBe('low'); + }); + + it('sends no reasoning_effort when the endpoint disables reasoning parameters', () => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'high' }, + {}, + { reasoningFormat: ReasoningParameterFormat.disabled }, + ); + + expect(sentEffort(result)).toBeUndefined(); + }); + + describe('configured default', () => { + const withDefault = (level: string | number) => ({ + paramDefinitions: [{ key: 'thinkingLevel', default: level }], + }); + + it('sends the endpoint default thinking level when none is selected', () => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview' }, + {}, + withDefault('high'), + ); + + expect(sentEffort(result)).toBe('high'); + }); + + it.each([undefined, ''])( + 'sends the endpoint default when the selected thinking level is %j', + (thinkingLevel) => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel }, + {}, + withDefault('medium'), + ); + + expect(sentEffort(result)).toBe('medium'); + }, + ); + + it('lets a selected thinking level replace the endpoint default', () => { + const result = getGeminiConfig( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'low' }, + {}, + withDefault('high'), + ); + + expect(sentEffort(result)).toBe('low'); + }); + + it.each([ + ['gemini-3.7-flash', 'minimal', 'low'], + ['gemini-3.5-flash', 'high', 'high'], + ])('applies the model rules to the endpoint default (%s %s)', (model, level, effort) => { + const result = getGeminiConfig({ model }, {}, withDefault(level)); + + expect(sentEffort(result)).toBe(effort); + }); + + it.each([['unset'], [''], [7], ['THINKING_LEVEL_UNSPECIFIED']])( + 'sends nothing for the unusable endpoint default %j', + (level) => { + for (const model of ['gemini-3.1-pro-preview', 'gemini-3.5-flash']) { + expect( + sentEffort(getGeminiConfig({ model }, {}, withDefault(level))), + ).toBeUndefined(); + } + }, + ); + + it.each([ + ['before Gemini 3', { model: 'gemini-2.5-pro' }, {}], + ['with thinking off', { model: 'gemini-3.1-pro-preview', thinking: false }, {}], + [ + 'with thinkingLevel dropped', + { model: 'gemini-3.1-pro-preview' }, + { dropParams: ['thinkingLevel'] }, + ], + [ + 'with reasoning_effort dropped', + { model: 'gemini-3.1-pro-preview' }, + { dropParams: ['reasoning_effort'] }, + ], + ])('sends no endpoint default %s', (_name, modelOptions, rest) => { + const result = getGeminiConfig(modelOptions, rest, withDefault('high')); + + expect(sentEffort(result)).toBeUndefined(); + }); + }); + }); }); }); diff --git a/packages/api/src/endpoints/openai/config.ts b/packages/api/src/endpoints/openai/config.ts index cd014c5aba..19455f2979 100644 --- a/packages/api/src/endpoints/openai/config.ts +++ b/packages/api/src/endpoints/openai/config.ts @@ -178,6 +178,8 @@ export function getOpenAIConfig( * them from the forwarded `addParams` too so the model does not receive * params it rejects. `defaultParams` is applied inside `getGoogleConfig` * (and only read here for tool detection), so it needs no sanitizing. + * The transform drops `thinkingConfig`, so a selected thinking level reaches + * the gateway as `reasoning_effort` instead. */ const transformed = transformToOpenAIConfig({ addParams: stripGeminiFlashBlockedParams( @@ -190,6 +192,7 @@ export function getOpenAIConfig( tools: googleResult.tools, llmConfig: googleResult.llmConfig, fromEndpoint: EModelEndpoint.google, + reasoningEffort: googleResult.reasoningEffort, }); llmConfig = transformed.llmConfig; tools = transformed.tools; diff --git a/packages/api/src/endpoints/openai/requests.spec.ts b/packages/api/src/endpoints/openai/requests.spec.ts index 57cfe7bc59..747b339b71 100644 --- a/packages/api/src/endpoints/openai/requests.spec.ts +++ b/packages/api/src/endpoints/openai/requests.spec.ts @@ -225,3 +225,93 @@ describe('Azure full-hostname instances', () => { ); }); }); + +describe('Gemini-compatible custom endpoint requests', () => { + const GATEWAY_URL = 'https://gateway.example.com/v1beta/openai'; + + /** Sends one chat completion through the real client and returns the body the gateway receives. */ + async function getRequestBody( + modelOptions: object, + rest: { addParams?: Record } = {}, + ): Promise> { + const bodies: Record[] = []; + const fetch: NonNullable['fetch']> = async (_url, init) => { + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ + id: 'chatcmpl_test', + object: 'chat.completion', + created: 1, + model: 'gemini-3.1-pro-preview', + choices: [ + { index: 0, message: { role: 'assistant', content: 'Four.' }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 5, completion_tokens: 1, total_tokens: 6 }, + }); + }; + const options: NonNullable[1]> = { + streaming: false, + reverseProxyUrl: GATEWAY_URL, + customParams: { defaultParamsEndpoint: 'google' }, + modelOptions, + ...rest, + }; + const { llmConfig, configOptions } = getOpenAIConfig( + 'test-google-key', + options, + 'Gemini (Custom)', + ); + const model = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { + ...llmConfig, + verbosity: undefined, + configuration: { ...configOptions, fetch }, + }, + }); + + await model.invoke('What is 2 + 2?'); + + expect(bodies).toHaveLength(1); + return bodies[0]; + } + + it.each(['minimal', 'low', 'medium', 'high'])( + 'sends a selected %s thinking level as reasoning_effort', + async (thinkingLevel) => { + const body = await getRequestBody({ model: 'gemini-3.1-pro-preview', thinkingLevel }); + + expect(body).toMatchObject({ + model: 'gemini-3.1-pro-preview', + reasoning_effort: thinkingLevel, + }); + for (const key of ['thinkingConfig', 'thinking_config', 'thinkingLevel', 'extra_body']) { + expect(body).not.toHaveProperty(key); + } + }, + ); + + it('sends no reasoning_effort when no thinking level is selected', async () => { + const body = await getRequestBody({ model: 'gemini-3.1-pro-preview' }); + + expect(body).toMatchObject({ model: 'gemini-3.1-pro-preview' }); + expect(body).not.toHaveProperty('reasoning_effort'); + }); + + it('sends the level a Flash model accepts when the selection is not supported', async () => { + const body = await getRequestBody({ model: 'gemini-4-argon', thinkingLevel: 'minimal' }); + + expect(body).toMatchObject({ reasoning_effort: 'low' }); + }); + + it.each([ + ['reasoning_effort', { reasoning_effort: 'high' }], + ['reasoning object', { reasoning: { effort: 'high' } }], + ])('lets an addParams %s replace the selected thinking level', async (_name, addParams) => { + const body = await getRequestBody( + { model: 'gemini-3.1-pro-preview', thinkingLevel: 'low' }, + { addParams }, + ); + + expect(body).toMatchObject({ reasoning_effort: 'high' }); + }); +}); diff --git a/packages/api/src/endpoints/openai/transform.ts b/packages/api/src/endpoints/openai/transform.ts index 9d7b420c3d..9d047f3ce9 100644 --- a/packages/api/src/endpoints/openai/transform.ts +++ b/packages/api/src/endpoints/openai/transform.ts @@ -20,12 +20,42 @@ const googleToolsToFilter = new Set(['googleSearch', 'urlContext']); export type ConfigTools = Array> | Array; +/** + * Google documents `reasoning_effort` and `thinking_config` as overlapping controls + * that cannot be sent together, so an `addParams` `thinking_config` keeps the field. + */ +function hasGoogleThinkingConfig(extraBody: unknown): boolean { + if (extraBody == null || typeof extraBody !== 'object') { + return false; + } + const google = (extraBody as Record).google; + if (google == null || typeof google !== 'object') { + return false; + } + return 'thinking_config' in google || 'thinkingConfig' in google; +} + +/** Whether `addParams` or the source config already carries its own reasoning control. */ +function hasReasoningControl( + openAIConfig: Partial, + modelKwargs: Record, +): boolean { + return ( + openAIConfig.reasoning != null || + modelKwargs.reasoning != null || + modelKwargs.reasoning_effort != null || + hasGoogleThinkingConfig(modelKwargs.extra_body) + ); +} + /** * Transforms a Non-OpenAI LLM config to an OpenAI-conformant config. * Non-OpenAI parameters are moved to modelKwargs. * Also extracts configuration options that belong in configOptions. * Handles addParams and dropParams for parameter customization. * Filters out provider-specific tools that have no OpenAI equivalent. + * Sends `reasoningEffort` as `reasoning_effort` unless `addParams` already sets a reasoning + * control; `dropParams` removes it like any other param. */ export function transformToOpenAIConfig({ tools, @@ -35,6 +65,7 @@ export function transformToOpenAIConfig({ inferenceParameters, llmConfig, fromEndpoint, + reasoningEffort, }: { tools?: ConfigTools; addParams?: Record; @@ -43,6 +74,7 @@ export function transformToOpenAIConfig({ inferenceParameters?: Record; llmConfig: ClientOptions; fromEndpoint: string; + reasoningEffort?: string; }): { tools: ConfigTools; llmConfig: t.OAIClientOptions; @@ -129,6 +161,11 @@ export function transformToOpenAIConfig({ } } + if (reasoningEffort != null && !hasReasoningControl(openAIConfig, modelKwargs)) { + modelKwargs.reasoning_effort = reasoningEffort; + hasModelKwargs = true; + } + if (hasModelKwargs) { openAIConfig.modelKwargs = modelKwargs; }