mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 14:08:21 +00:00
Bound model-facing Flux images to the 5 MiB base64 budget for standard and finetuned generation. Preserve original image data for attachment storage under the existing resolution policy. Fixes #6954.
520 lines
19 KiB
JavaScript
520 lines
19 KiB
JavaScript
/**
|
|
* Regression tests for image tool agent mode: verifies that invoke() returns
|
|
* a ToolMessage with base64 in artifact.content rather than serialized into content.
|
|
*
|
|
* Root cause: FluxAPI/StableDiffusion extend LangChain's Tool but did not
|
|
* set responseFormat = 'content_and_artifact'. LangChain's invoke() would then
|
|
* JSON.stringify the entire [content, artifact] tuple into ToolMessage.content,
|
|
* dumping base64 into token counting and causing context exhaustion.
|
|
*/
|
|
|
|
const axios = require('axios');
|
|
const fetch = require('node-fetch');
|
|
const { logger } = require('@librechat/data-schemas');
|
|
const { ContentTypes } = require('librechat-data-provider');
|
|
const { ToolMessage } = require('@librechat/agents/langchain/messages');
|
|
const StableDiffusionAPI = require('../StableDiffusion');
|
|
const FluxAPI = require('../FluxAPI');
|
|
|
|
jest.mock('axios');
|
|
jest.mock('node-fetch');
|
|
jest.mock('@librechat/data-schemas', () => ({
|
|
logger: { info: jest.fn(), warn: jest.fn(), debug: jest.fn(), error: jest.fn() },
|
|
}));
|
|
jest.mock('path', () => ({
|
|
resolve: jest.fn(),
|
|
join: jest.fn().mockReturnValue('/mock/path'),
|
|
relative: jest.fn().mockReturnValue('relative/path'),
|
|
extname: jest.fn().mockReturnValue('.png'),
|
|
}));
|
|
jest.mock('fs', () => ({
|
|
existsSync: jest.fn().mockReturnValue(true),
|
|
mkdirSync: jest.fn(),
|
|
promises: { writeFile: jest.fn(), readFile: jest.fn(), unlink: jest.fn() },
|
|
}));
|
|
|
|
const FAKE_BASE64 =
|
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=';
|
|
|
|
const makeToolCall = (name, args) => ({
|
|
id: 'call_test_123',
|
|
name,
|
|
args,
|
|
type: 'tool_call',
|
|
});
|
|
|
|
describe('image tools - agent mode ToolMessage format', () => {
|
|
const ENV_KEYS = ['FLUX_API_KEY', 'SD_WEBUI_URL', 'PROXY'];
|
|
let savedEnv = {};
|
|
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
for (const key of ENV_KEYS) {
|
|
savedEnv[key] = process.env[key];
|
|
}
|
|
process.env.FLUX_API_KEY = 'test-flux-key';
|
|
process.env.SD_WEBUI_URL = 'http://localhost:7860';
|
|
delete process.env.PROXY;
|
|
});
|
|
|
|
afterEach(() => {
|
|
for (const key of ENV_KEYS) {
|
|
if (savedEnv[key] === undefined) {
|
|
delete process.env[key];
|
|
} else {
|
|
process.env[key] = savedEnv[key];
|
|
}
|
|
}
|
|
savedEnv = {};
|
|
});
|
|
|
|
describe('FluxAPI', () => {
|
|
beforeEach(() => {
|
|
jest.useFakeTimers();
|
|
axios.post.mockResolvedValue({ data: { id: 'task-123' } });
|
|
axios.get.mockResolvedValue({
|
|
data: { status: 'Ready', result: { sample: 'https://example.com/image.png' } },
|
|
});
|
|
fetch.mockResolvedValue({
|
|
arrayBuffer: () => Promise.resolve(Buffer.from(FAKE_BASE64, 'base64')),
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
jest.useRealTimers();
|
|
});
|
|
|
|
it('sets responseFormat to content_and_artifact when isAgent is true', () => {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
expect(flux.responseFormat).toBe('content_and_artifact');
|
|
});
|
|
|
|
it('does not set responseFormat when isAgent is false', () => {
|
|
const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn() });
|
|
expect(flux.responseFormat).not.toBe('content_and_artifact');
|
|
});
|
|
|
|
it('keeps tenant context without retaining the request object', () => {
|
|
const req = {
|
|
user: { id: 'user-1', tenantId: 'tenant-a' },
|
|
body: { conversationId: 'convo-1', isTemporary: 'true' },
|
|
config: { interfaceConfig: { retentionMode: 'all' } },
|
|
socket: {},
|
|
};
|
|
const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn(), req });
|
|
|
|
expect(flux.tenantId).toBe('tenant-a');
|
|
expect(flux.req).toBeUndefined();
|
|
expect(flux.retentionRequest).toEqual({
|
|
user: { id: 'user-1', tenantId: 'tenant-a' },
|
|
body: { conversationId: 'convo-1', isTemporary: 'true' },
|
|
config: { interfaceConfig: { retentionMode: 'all' } },
|
|
});
|
|
});
|
|
|
|
it('passes minimal retention context when saving generated images', async () => {
|
|
const processFileURL = jest.fn().mockResolvedValue({ filepath: '/images/generated.png' });
|
|
const req = {
|
|
user: { id: 'user-1', tenantId: 'tenant-a' },
|
|
body: { conversationId: 'convo-1', isTemporary: 'true' },
|
|
config: { interfaceConfig: { retentionMode: 'all' } },
|
|
socket: {},
|
|
};
|
|
const flux = new FluxAPI({
|
|
isAgent: false,
|
|
processFileURL,
|
|
req,
|
|
userId: 'user-1',
|
|
fileStrategy: 'local',
|
|
});
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
await invokePromise;
|
|
|
|
expect(processFileURL).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
req: {
|
|
user: { id: 'user-1', tenantId: 'tenant-a' },
|
|
body: { conversationId: 'convo-1', isTemporary: 'true' },
|
|
config: { interfaceConfig: { retentionMode: 'all' } },
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('passes minimal retention context when saving finetuned generated images', async () => {
|
|
const processFileURL = jest.fn().mockResolvedValue({ filepath: '/images/generated.png' });
|
|
const req = {
|
|
user: { id: 'user-1', tenantId: 'tenant-a' },
|
|
body: { conversationId: 'convo-1', isTemporary: 'true' },
|
|
config: { interfaceConfig: { retentionMode: 'all' } },
|
|
socket: {},
|
|
};
|
|
const flux = new FluxAPI({
|
|
isAgent: false,
|
|
processFileURL,
|
|
req,
|
|
userId: 'user-1',
|
|
fileStrategy: 'local',
|
|
});
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', {
|
|
action: 'generate_finetuned',
|
|
prompt: 'a box',
|
|
finetune_id: 'ft-abc123',
|
|
endpoint: '/v1/flux-pro-finetuned',
|
|
}),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
await invokePromise;
|
|
|
|
expect(processFileURL).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
req: {
|
|
user: { id: 'user-1', tenantId: 'tenant-a' },
|
|
body: { conversationId: 'convo-1', isTemporary: 'true' },
|
|
config: { interfaceConfig: { retentionMode: 'all' } },
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('invoke() returns ToolMessage with base64 in artifact, not serialized in content', async () => {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
const result = await invokePromise;
|
|
|
|
expect(result).toBeInstanceOf(ToolMessage);
|
|
const contentStr =
|
|
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
|
|
expect(contentStr).not.toContain(FAKE_BASE64);
|
|
|
|
expect(result.artifact).toBeDefined();
|
|
const artifactContent = result.artifact?.content;
|
|
expect(Array.isArray(artifactContent)).toBe(true);
|
|
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
|
|
expect(artifactContent[0].image_url.url).toContain('base64');
|
|
});
|
|
|
|
it('invoke() returns ToolMessage with base64 in artifact for generate_finetuned action', async () => {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', {
|
|
action: 'generate_finetuned',
|
|
prompt: 'a box',
|
|
finetune_id: 'ft-abc123',
|
|
endpoint: '/v1/flux-pro-finetuned',
|
|
}),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
const result = await invokePromise;
|
|
|
|
expect(result).toBeInstanceOf(ToolMessage);
|
|
const contentStr =
|
|
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
|
|
expect(contentStr).not.toContain(FAKE_BASE64);
|
|
|
|
expect(result.artifact).toBeDefined();
|
|
const artifactContent = result.artifact?.content;
|
|
expect(Array.isArray(artifactContent)).toBe(true);
|
|
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
|
|
expect(artifactContent[0].image_url.url).toContain('base64');
|
|
});
|
|
|
|
it.each([
|
|
['generate', { prompt: 'a box', endpoint: '/v1/flux-dev' }],
|
|
[
|
|
'generate_finetuned',
|
|
{ prompt: 'a box', finetune_id: 'ft-abc123', endpoint: '/v1/flux-pro-finetuned' },
|
|
],
|
|
])('%s polls the cluster polling_url returned by the global endpoint', async (action, args) => {
|
|
process.env.FLUX_API_BASE_URL = 'https://api.bfl.ai';
|
|
const pollingUrl = 'https://api.eu2.bfl.ai/v1/get_result?id=task-123';
|
|
axios.post.mockResolvedValue({ data: { id: 'task-123', polling_url: pollingUrl } });
|
|
|
|
try {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(makeToolCall('flux', { action, ...args }));
|
|
await jest.runAllTimersAsync();
|
|
const result = await invokePromise;
|
|
|
|
expect(result.artifact?.content?.[0]?.type).toBe(ContentTypes.IMAGE_URL);
|
|
expect(axios.get).toHaveBeenCalledWith(
|
|
pollingUrl,
|
|
expect.objectContaining({
|
|
params: undefined,
|
|
headers: expect.objectContaining({ 'x-key': 'test-flux-key' }),
|
|
}),
|
|
);
|
|
} finally {
|
|
delete process.env.FLUX_API_BASE_URL;
|
|
}
|
|
});
|
|
|
|
it('submits to the global endpoint when FLUX_API_BASE_URL is unset', async () => {
|
|
delete process.env.FLUX_API_BASE_URL;
|
|
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
await invokePromise;
|
|
|
|
expect(axios.post).toHaveBeenCalledWith(
|
|
'https://api.bfl.ai/v1/flux-dev',
|
|
expect.any(Object),
|
|
expect.any(Object),
|
|
);
|
|
});
|
|
|
|
it('falls back to get_result on the base URL when polling_url is on another domain', async () => {
|
|
process.env.FLUX_API_BASE_URL = 'https://api.bfl.ai';
|
|
axios.post.mockResolvedValue({
|
|
data: { id: 'task-123', polling_url: 'https://evil.example.com/v1/get_result?id=task-123' },
|
|
});
|
|
|
|
try {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
await invokePromise;
|
|
|
|
expect(axios.get).toHaveBeenCalledWith(
|
|
'https://api.bfl.ai/v1/get_result',
|
|
expect.objectContaining({ params: { id: 'task-123' } }),
|
|
);
|
|
} finally {
|
|
delete process.env.FLUX_API_BASE_URL;
|
|
}
|
|
});
|
|
|
|
it('invoke() returns ToolMessage with error string in content when task submission fails', async () => {
|
|
axios.post.mockRejectedValue(new Error('Network error'));
|
|
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
const result = await invokePromise;
|
|
|
|
expect(result).toBeInstanceOf(ToolMessage);
|
|
const contentStr =
|
|
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
|
|
expect(contentStr).toBe('Error: The tool request failed.');
|
|
expect(contentStr).not.toContain('Network error');
|
|
expect(result.artifact).toBeDefined();
|
|
});
|
|
|
|
describe('approved error text', () => {
|
|
const SECRET = 'PRIVATE-PROVIDER-DETAIL';
|
|
const FALLBACK = 'Error: The tool request failed.';
|
|
|
|
const providerError = (status) =>
|
|
Object.assign(new Error(`Request failed with status code ${status}`), {
|
|
isAxiosError: true,
|
|
response: { status, data: { detail: SECRET } },
|
|
});
|
|
|
|
const contentOf = async (flux, args) => {
|
|
const invokePromise = flux.invoke(makeToolCall('flux', args));
|
|
await jest.runAllTimersAsync();
|
|
const result = await invokePromise;
|
|
return typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
|
|
};
|
|
|
|
/** True when any logged argument, including an error message, carries the text. */
|
|
const logged = (text) =>
|
|
[logger.warn, logger.error, logger.info, logger.debug]
|
|
.flatMap((method) => method.mock.calls.flat())
|
|
.some((arg) =>
|
|
`${arg instanceof Error ? arg.message : JSON.stringify(arg)}`.includes(text),
|
|
);
|
|
|
|
const generate = { prompt: 'a box', endpoint: '/v1/flux-dev' };
|
|
const finetuned = { action: 'generate_finetuned', prompt: 'a box', finetune_id: 'ft-1' };
|
|
|
|
it('words a rejected generation request from its status, never the provider body', async () => {
|
|
axios.post.mockRejectedValue(providerError(401));
|
|
|
|
const content = await contentOf(new FluxAPI({ isAgent: true }), generate);
|
|
|
|
expect(content).toBe('Error: Flux rejected the configured credentials (status 401).');
|
|
expect(logger.warn).toHaveBeenCalledWith('[Flux] Tool request failed', { status: 401 });
|
|
expect(logged(SECRET)).toBe(false);
|
|
});
|
|
|
|
it('words a rejected finetuned generation request from its status', async () => {
|
|
axios.post.mockRejectedValue(providerError(429));
|
|
|
|
const content = await contentOf(new FluxAPI({ isAgent: true }), finetuned);
|
|
|
|
expect(content).toBe(
|
|
'Error: Flux is rate limiting requests; try again later (status 429).',
|
|
);
|
|
expect(logged(SECRET)).toBe(false);
|
|
});
|
|
|
|
it.each([
|
|
['generation', generate],
|
|
['finetuned generation', finetuned],
|
|
])('returns the fallback when the %s image cannot be downloaded', async (_label, args) => {
|
|
fetch.mockRejectedValue(
|
|
new Error(`request to https://cdn.example.com/?sig=${SECRET} failed`),
|
|
);
|
|
|
|
const content = await contentOf(new FluxAPI({ isAgent: true }), args);
|
|
|
|
expect(content).toBe(FALLBACK);
|
|
expect(logged(SECRET)).toBe(false);
|
|
});
|
|
|
|
it.each([
|
|
['generation', generate],
|
|
['finetuned generation', finetuned],
|
|
])('returns the fallback when the %s image cannot be saved', async (_label, args) => {
|
|
const processFileURL = jest
|
|
.fn()
|
|
.mockRejectedValue(new Error(`EACCES: permission denied, open '/srv/${SECRET}.png'`));
|
|
const flux = new FluxAPI({ isAgent: false, processFileURL, userId: 'user-1' });
|
|
|
|
const content = await contentOf(flux, args);
|
|
|
|
expect(content).toBe(FALLBACK);
|
|
expect(logged(SECRET)).toBe(false);
|
|
});
|
|
|
|
it('words a failing finetune listing from its status for an agent', async () => {
|
|
axios.get.mockRejectedValue(providerError(503));
|
|
|
|
const content = await contentOf(new FluxAPI({ isAgent: true }), {
|
|
action: 'list_finetunes',
|
|
});
|
|
|
|
expect(content).toBe('Error: Flux is unavailable; try again later (status 503).');
|
|
expect(logged(SECRET)).toBe(false);
|
|
});
|
|
|
|
it('keeps the approved text in the error a non-agent finetune listing returns', async () => {
|
|
axios.get.mockRejectedValue(providerError(503));
|
|
const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn() });
|
|
|
|
const result = await flux.getMyFinetunes('test-flux-key');
|
|
|
|
expect(result).toBeInstanceOf(Error);
|
|
expect(result.message).toBe('Error: Flux is unavailable; try again later (status 503).');
|
|
expect(logged(SECRET)).toBe(false);
|
|
});
|
|
});
|
|
|
|
it('routes a finetuned endpoint through generateFinetunedImage even when action is left as "generate"', async () => {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
const invokePromise = flux.invoke(
|
|
makeToolCall('flux', {
|
|
prompt: 'a box',
|
|
endpoint: '/v1/flux-pro-finetuned',
|
|
finetune_id: 'ft-abc123',
|
|
finetune_strength: 0.8,
|
|
guidance: 3,
|
|
}),
|
|
);
|
|
await jest.runAllTimersAsync();
|
|
const result = await invokePromise;
|
|
|
|
expect(axios.post).toHaveBeenCalledWith(
|
|
expect.stringContaining('/v1/flux-pro-finetuned'),
|
|
expect.objectContaining({
|
|
finetune_id: 'ft-abc123',
|
|
finetune_strength: 0.8,
|
|
guidance: 3,
|
|
}),
|
|
expect.anything(),
|
|
);
|
|
|
|
expect(result).toBeInstanceOf(ToolMessage);
|
|
expect(result.artifact).toBeDefined();
|
|
const artifactContent = result.artifact?.content;
|
|
expect(Array.isArray(artifactContent)).toBe(true);
|
|
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
|
|
expect(artifactContent[0].image_url.url).toContain('base64');
|
|
});
|
|
|
|
it('rejects a finetuned endpoint without finetune_id even when action is left as "generate"', async () => {
|
|
const flux = new FluxAPI({ isAgent: true });
|
|
|
|
await expect(
|
|
flux.invoke(
|
|
makeToolCall('flux', {
|
|
prompt: 'a box',
|
|
endpoint: '/v1/flux-pro-finetuned',
|
|
}),
|
|
),
|
|
).rejects.toThrow(/finetune_id/);
|
|
});
|
|
});
|
|
|
|
describe('StableDiffusion', () => {
|
|
beforeEach(() => {
|
|
axios.post.mockResolvedValue({
|
|
data: {
|
|
images: [FAKE_BASE64],
|
|
info: JSON.stringify({ height: 1024, width: 1024, seed: 42, infotexts: [] }),
|
|
},
|
|
});
|
|
});
|
|
|
|
it('sets responseFormat to content_and_artifact when isAgent is true', () => {
|
|
const sd = new StableDiffusionAPI({ isAgent: true, override: true });
|
|
expect(sd.responseFormat).toBe('content_and_artifact');
|
|
});
|
|
|
|
it('does not set responseFormat when isAgent is false', () => {
|
|
const sd = new StableDiffusionAPI({
|
|
isAgent: false,
|
|
override: true,
|
|
uploadImageBuffer: jest.fn(),
|
|
});
|
|
expect(sd.responseFormat).not.toBe('content_and_artifact');
|
|
});
|
|
|
|
it('invoke() returns ToolMessage with base64 in artifact, not serialized in content', async () => {
|
|
const sd = new StableDiffusionAPI({ isAgent: true, override: true, userId: 'user-1' });
|
|
const result = await sd.invoke(
|
|
makeToolCall('stable-diffusion', { prompt: 'a box', negative_prompt: '' }),
|
|
);
|
|
|
|
expect(result).toBeInstanceOf(ToolMessage);
|
|
const contentStr =
|
|
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
|
|
expect(contentStr).not.toContain(FAKE_BASE64);
|
|
|
|
expect(result.artifact).toBeDefined();
|
|
const artifactContent = result.artifact?.content;
|
|
expect(Array.isArray(artifactContent)).toBe(true);
|
|
expect(artifactContent[0].type).toBe(ContentTypes.IMAGE_URL);
|
|
expect(artifactContent[0].image_url.url).toContain('base64');
|
|
});
|
|
|
|
it('invoke() returns ToolMessage with error string in content when API fails', async () => {
|
|
axios.post.mockRejectedValue(new Error('Connection refused'));
|
|
|
|
const sd = new StableDiffusionAPI({ isAgent: true, override: true, userId: 'user-1' });
|
|
const result = await sd.invoke(
|
|
makeToolCall('stable-diffusion', { prompt: 'a box', negative_prompt: '' }),
|
|
);
|
|
|
|
expect(result).toBeInstanceOf(ToolMessage);
|
|
const contentStr =
|
|
typeof result.content === 'string' ? result.content : JSON.stringify(result.content);
|
|
expect(contentStr).toContain('Error making API request');
|
|
});
|
|
});
|
|
});
|