LibreChat/api/app/clients/tools/structured/specs/imageTools-agent.spec.js
dvorak33 a3868fdbe8
🖼️ fix: Bound Flux Tool Images and Preserve Original Attachments (#13749)
Bound model-facing Flux images to the 5 MiB base64 budget for standard and finetuned generation. Preserve original image data for attachment storage under the existing resolution policy.

Fixes #6954.
2026-11-09 22:41:29 +01:00

520 lines
19 KiB
JavaScript

/**
* Regression tests for image tool agent mode: verifies that invoke() returns
* a ToolMessage with base64 in artifact.content rather than serialized into content.
*
* Root cause: FluxAPI/StableDiffusion extend LangChain's Tool but did not
* set responseFormat = 'content_and_artifact'. LangChain's invoke() would then
* JSON.stringify the entire [content, artifact] tuple into ToolMessage.content,
* dumping base64 into token counting and causing context exhaustion.
*/
const axios = require('axios');
const fetch = require('node-fetch');
const { logger } = require('@librechat/data-schemas');
const { ContentTypes } = require('librechat-data-provider');
const { ToolMessage } = require('@librechat/agents/langchain/messages');
const StableDiffusionAPI = require('../StableDiffusion');
const FluxAPI = require('../FluxAPI');
jest.mock('axios');
jest.mock('node-fetch');
jest.mock('@librechat/data-schemas', () => ({
logger: { info: jest.fn(), warn: jest.fn(), debug: jest.fn(), error: jest.fn() },
}));
jest.mock('path', () => ({
resolve: jest.fn(),
join: jest.fn().mockReturnValue('/mock/path'),
relative: jest.fn().mockReturnValue('relative/path'),
extname: jest.fn().mockReturnValue('.png'),
}));
jest.mock('fs', () => ({
existsSync: jest.fn().mockReturnValue(true),
mkdirSync: jest.fn(),
promises: { writeFile: jest.fn(), readFile: jest.fn(), unlink: jest.fn() },
}));
const FAKE_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=';
const makeToolCall = (name, args) => ({
id: 'call_test_123',
name,
args,
type: 'tool_call',
});
describe('image tools - agent mode ToolMessage format', () => {
const ENV_KEYS = ['FLUX_API_KEY', 'SD_WEBUI_URL', 'PROXY'];
let savedEnv = {};
beforeEach(() => {
jest.clearAllMocks();
for (const key of ENV_KEYS) {
savedEnv[key] = process.env[key];
}
process.env.FLUX_API_KEY = 'test-flux-key';
process.env.SD_WEBUI_URL = 'http://localhost:7860';
delete process.env.PROXY;
});
afterEach(() => {
for (const key of ENV_KEYS) {
if (savedEnv[key] === undefined) {
delete process.env[key];
} else {
process.env[key] = savedEnv[key];
}
}
savedEnv = {};
});
describe('FluxAPI', () => {
beforeEach(() => {
jest.useFakeTimers();
axios.post.mockResolvedValue({ data: { id: 'task-123' } });
axios.get.mockResolvedValue({
data: { status: 'Ready', result: { sample: 'https://example.com/image.png' } },
});
fetch.mockResolvedValue({
arrayBuffer: () => Promise.resolve(Buffer.from(FAKE_BASE64, 'base64')),
});
});
afterEach(() => {
jest.useRealTimers();
});
it('sets responseFormat to content_and_artifact when isAgent is true', () => {
const flux = new FluxAPI({ isAgent: true });
expect(flux.responseFormat).toBe('content_and_artifact');
});
it('does not set responseFormat when isAgent is false', () => {
const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn() });
expect(flux.responseFormat).not.toBe('content_and_artifact');
});
it('keeps tenant context without retaining the request object', () => {
const req = {
user: { id: 'user-1', tenantId: 'tenant-a' },
body: { conversationId: 'convo-1', isTemporary: 'true' },
config: { interfaceConfig: { retentionMode: 'all' } },
socket: {},
};
const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn(), req });
expect(flux.tenantId).toBe('tenant-a');
expect(flux.req).toBeUndefined();
expect(flux.retentionRequest).toEqual({
user: { id: 'user-1', tenantId: 'tenant-a' },
body: { conversationId: 'convo-1', isTemporary: 'true' },
config: { interfaceConfig: { retentionMode: 'all' } },
});
});
it('passes minimal retention context when saving generated images', async () => {
const processFileURL = jest.fn().mockResolvedValue({ filepath: '/images/generated.png' });
const req = {
user: { id: 'user-1', tenantId: 'tenant-a' },
body: { conversationId: 'convo-1', isTemporary: 'true' },
config: { interfaceConfig: { retentionMode: 'all' } },
socket: {},
};
const flux = new FluxAPI({
isAgent: false,
processFileURL,
req,
userId: 'user-1',
fileStrategy: 'local',
});
const invokePromise = flux.invoke(
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
);
await jest.runAllTimersAsync();
await invokePromise;
expect(processFileURL).toHaveBeenCalledWith(
expect.objectContaining({
req: {
user: { id: 'user-1', tenantId: 'tenant-a' },
body: { conversationId: 'convo-1', isTemporary: 'true' },
config: { interfaceConfig: { retentionMode: 'all' } },
},
}),
);
});
it('passes minimal retention context when saving finetuned generated images', async () => {
const processFileURL = jest.fn().mockResolvedValue({ filepath: '/images/generated.png' });
const req = {
user: { id: 'user-1', tenantId: 'tenant-a' },
body: { conversationId: 'convo-1', isTemporary: 'true' },
config: { interfaceConfig: { retentionMode: 'all' } },
socket: {},
};
const flux = new FluxAPI({
isAgent: false,
processFileURL,
req,
userId: 'user-1',
fileStrategy: 'local',
});
const invokePromise = flux.invoke(
makeToolCall('flux', {
action: 'generate_finetuned',
prompt: 'a box',
finetune_id: 'ft-abc123',
endpoint: '/v1/flux-pro-finetuned',
}),
);
await jest.runAllTimersAsync();
await invokePromise;
expect(processFileURL).toHaveBeenCalledWith(
expect.objectContaining({
req: {
user: { id: 'user-1', tenantId: 'tenant-a' },
body: { conversationId: 'convo-1', isTemporary: 'true' },
config: { interfaceConfig: { retentionMode: 'all' } },
},
}),
);
});
it('invoke() returns ToolMessage with base64 in artifact, not serialized in content', async () => {
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
);
await jest.runAllTimersAsync();
const result = await invokePromise;
expect(result).toBeInstanceOf(ToolMessage);
const contentStr =
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
expect(contentStr).not.toContain(FAKE_BASE64);
expect(result.artifact).toBeDefined();
const artifactContent = result.artifact?.content;
expect(Array.isArray(artifactContent)).toBe(true);
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
expect(artifactContent[0].image_url.url).toContain('base64');
});
it('invoke() returns ToolMessage with base64 in artifact for generate_finetuned action', async () => {
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(
makeToolCall('flux', {
action: 'generate_finetuned',
prompt: 'a box',
finetune_id: 'ft-abc123',
endpoint: '/v1/flux-pro-finetuned',
}),
);
await jest.runAllTimersAsync();
const result = await invokePromise;
expect(result).toBeInstanceOf(ToolMessage);
const contentStr =
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
expect(contentStr).not.toContain(FAKE_BASE64);
expect(result.artifact).toBeDefined();
const artifactContent = result.artifact?.content;
expect(Array.isArray(artifactContent)).toBe(true);
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
expect(artifactContent[0].image_url.url).toContain('base64');
});
it.each([
['generate', { prompt: 'a box', endpoint: '/v1/flux-dev' }],
[
'generate_finetuned',
{ prompt: 'a box', finetune_id: 'ft-abc123', endpoint: '/v1/flux-pro-finetuned' },
],
])('%s polls the cluster polling_url returned by the global endpoint', async (action, args) => {
process.env.FLUX_API_BASE_URL = 'https://api.bfl.ai';
const pollingUrl = 'https://api.eu2.bfl.ai/v1/get_result?id=task-123';
axios.post.mockResolvedValue({ data: { id: 'task-123', polling_url: pollingUrl } });
try {
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(makeToolCall('flux', { action, ...args }));
await jest.runAllTimersAsync();
const result = await invokePromise;
expect(result.artifact?.content?.[0]?.type).toBe(ContentTypes.IMAGE_URL);
expect(axios.get).toHaveBeenCalledWith(
pollingUrl,
expect.objectContaining({
params: undefined,
headers: expect.objectContaining({ 'x-key': 'test-flux-key' }),
}),
);
} finally {
delete process.env.FLUX_API_BASE_URL;
}
});
it('submits to the global endpoint when FLUX_API_BASE_URL is unset', async () => {
delete process.env.FLUX_API_BASE_URL;
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
);
await jest.runAllTimersAsync();
await invokePromise;
expect(axios.post).toHaveBeenCalledWith(
'https://api.bfl.ai/v1/flux-dev',
expect.any(Object),
expect.any(Object),
);
});
it('falls back to get_result on the base URL when polling_url is on another domain', async () => {
process.env.FLUX_API_BASE_URL = 'https://api.bfl.ai';
axios.post.mockResolvedValue({
data: { id: 'task-123', polling_url: 'https://evil.example.com/v1/get_result?id=task-123' },
});
try {
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
);
await jest.runAllTimersAsync();
await invokePromise;
expect(axios.get).toHaveBeenCalledWith(
'https://api.bfl.ai/v1/get_result',
expect.objectContaining({ params: { id: 'task-123' } }),
);
} finally {
delete process.env.FLUX_API_BASE_URL;
}
});
it('invoke() returns ToolMessage with error string in content when task submission fails', async () => {
axios.post.mockRejectedValue(new Error('Network error'));
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
);
await jest.runAllTimersAsync();
const result = await invokePromise;
expect(result).toBeInstanceOf(ToolMessage);
const contentStr =
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
expect(contentStr).toBe('Error: The tool request failed.');
expect(contentStr).not.toContain('Network error');
expect(result.artifact).toBeDefined();
});
describe('approved error text', () => {
const SECRET = 'PRIVATE-PROVIDER-DETAIL';
const FALLBACK = 'Error: The tool request failed.';
const providerError = (status) =>
Object.assign(new Error(`Request failed with status code ${status}`), {
isAxiosError: true,
response: { status, data: { detail: SECRET } },
});
const contentOf = async (flux, args) => {
const invokePromise = flux.invoke(makeToolCall('flux', args));
await jest.runAllTimersAsync();
const result = await invokePromise;
return typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
};
/** True when any logged argument, including an error message, carries the text. */
const logged = (text) =>
[logger.warn, logger.error, logger.info, logger.debug]
.flatMap((method) => method.mock.calls.flat())
.some((arg) =>
`${arg instanceof Error ? arg.message : JSON.stringify(arg)}`.includes(text),
);
const generate = { prompt: 'a box', endpoint: '/v1/flux-dev' };
const finetuned = { action: 'generate_finetuned', prompt: 'a box', finetune_id: 'ft-1' };
it('words a rejected generation request from its status, never the provider body', async () => {
axios.post.mockRejectedValue(providerError(401));
const content = await contentOf(new FluxAPI({ isAgent: true }), generate);
expect(content).toBe('Error: Flux rejected the configured credentials (status 401).');
expect(logger.warn).toHaveBeenCalledWith('[Flux] Tool request failed', { status: 401 });
expect(logged(SECRET)).toBe(false);
});
it('words a rejected finetuned generation request from its status', async () => {
axios.post.mockRejectedValue(providerError(429));
const content = await contentOf(new FluxAPI({ isAgent: true }), finetuned);
expect(content).toBe(
'Error: Flux is rate limiting requests; try again later (status 429).',
);
expect(logged(SECRET)).toBe(false);
});
it.each([
['generation', generate],
['finetuned generation', finetuned],
])('returns the fallback when the %s image cannot be downloaded', async (_label, args) => {
fetch.mockRejectedValue(
new Error(`request to https://cdn.example.com/?sig=${SECRET} failed`),
);
const content = await contentOf(new FluxAPI({ isAgent: true }), args);
expect(content).toBe(FALLBACK);
expect(logged(SECRET)).toBe(false);
});
it.each([
['generation', generate],
['finetuned generation', finetuned],
])('returns the fallback when the %s image cannot be saved', async (_label, args) => {
const processFileURL = jest
.fn()
.mockRejectedValue(new Error(`EACCES: permission denied, open '/srv/${SECRET}.png'`));
const flux = new FluxAPI({ isAgent: false, processFileURL, userId: 'user-1' });
const content = await contentOf(flux, args);
expect(content).toBe(FALLBACK);
expect(logged(SECRET)).toBe(false);
});
it('words a failing finetune listing from its status for an agent', async () => {
axios.get.mockRejectedValue(providerError(503));
const content = await contentOf(new FluxAPI({ isAgent: true }), {
action: 'list_finetunes',
});
expect(content).toBe('Error: Flux is unavailable; try again later (status 503).');
expect(logged(SECRET)).toBe(false);
});
it('keeps the approved text in the error a non-agent finetune listing returns', async () => {
axios.get.mockRejectedValue(providerError(503));
const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn() });
const result = await flux.getMyFinetunes('test-flux-key');
expect(result).toBeInstanceOf(Error);
expect(result.message).toBe('Error: Flux is unavailable; try again later (status 503).');
expect(logged(SECRET)).toBe(false);
});
});
it('routes a finetuned endpoint through generateFinetunedImage even when action is left as "generate"', async () => {
const flux = new FluxAPI({ isAgent: true });
const invokePromise = flux.invoke(
makeToolCall('flux', {
prompt: 'a box',
endpoint: '/v1/flux-pro-finetuned',
finetune_id: 'ft-abc123',
finetune_strength: 0.8,
guidance: 3,
}),
);
await jest.runAllTimersAsync();
const result = await invokePromise;
expect(axios.post).toHaveBeenCalledWith(
expect.stringContaining('/v1/flux-pro-finetuned'),
expect.objectContaining({
finetune_id: 'ft-abc123',
finetune_strength: 0.8,
guidance: 3,
}),
expect.anything(),
);
expect(result).toBeInstanceOf(ToolMessage);
expect(result.artifact).toBeDefined();
const artifactContent = result.artifact?.content;
expect(Array.isArray(artifactContent)).toBe(true);
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
expect(artifactContent[0].image_url.url).toContain('base64');
});
it('rejects a finetuned endpoint without finetune_id even when action is left as "generate"', async () => {
const flux = new FluxAPI({ isAgent: true });
await expect(
flux.invoke(
makeToolCall('flux', {
prompt: 'a box',
endpoint: '/v1/flux-pro-finetuned',
}),
),
).rejects.toThrow(/finetune_id/);
});
});
describe('StableDiffusion', () => {
beforeEach(() => {
axios.post.mockResolvedValue({
data: {
images: [FAKE_BASE64],
info: JSON.stringify({ height: 1024, width: 1024, seed: 42, infotexts: [] }),
},
});
});
it('sets responseFormat to content_and_artifact when isAgent is true', () => {
const sd = new StableDiffusionAPI({ isAgent: true, override: true });
expect(sd.responseFormat).toBe('content_and_artifact');
});
it('does not set responseFormat when isAgent is false', () => {
const sd = new StableDiffusionAPI({
isAgent: false,
override: true,
uploadImageBuffer: jest.fn(),
});
expect(sd.responseFormat).not.toBe('content_and_artifact');
});
it('invoke() returns ToolMessage with base64 in artifact, not serialized in content', async () => {
const sd = new StableDiffusionAPI({ isAgent: true, override: true, userId: 'user-1' });
const result = await sd.invoke(
makeToolCall('stable-diffusion', { prompt: 'a box', negative_prompt: '' }),
);
expect(result).toBeInstanceOf(ToolMessage);
const contentStr =
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
expect(contentStr).not.toContain(FAKE_BASE64);
expect(result.artifact).toBeDefined();
const artifactContent = result.artifact?.content;
expect(Array.isArray(artifactContent)).toBe(true);
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
expect(artifactContent[0].image_url.url).toContain('base64');
});
it('invoke() returns ToolMessage with error string in content when API fails', async () => {
axios.post.mockRejectedValue(new Error('Connection refused'));
const sd = new StableDiffusionAPI({ isAgent: true, override: true, userId: 'user-1' });
const result = await sd.invoke(
makeToolCall('stable-diffusion', { prompt: 'a box', negative_prompt: '' }),
);
expect(result).toBeInstanceOf(ToolMessage);
const contentStr =
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
expect(contentStr).toContain('Error making API request');
});
});
});