mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 23:13:35 +00:00
fix(agents): keep file context in the message that carries the file
With execute_code and resendFiles, a code-environment file is loaded into the turn's attachments on every turn. addPreviousAttachments treated every attachment as submitted by the current request, so the file's text was cut from the message it was attached to and added to the newest message. The bytes in the middle of the history changed each turn and the provider cache stopped at that point. Only files named in req.body.files move to the current message. A refreshed tool file stays in its own message, and the latest message skips files an earlier message already carries, so the text appears once and the attachment limits count it once. Reported-by: FFourtyTwo
This commit is contained in:
parent
28d8780dad
commit
ac8cdcc1ea
3 changed files with 132 additions and 15 deletions
|
|
@ -1936,8 +1936,14 @@ class BaseClient {
|
|||
const attachmentsProcessed =
|
||||
this.options.attachments && !(this.options.attachments instanceof Promise);
|
||||
if (attachmentsProcessed) {
|
||||
/* Only files this request submits move to the current message. A tool file the
|
||||
* turn refreshes (code environment) keeps its context in the message that carries
|
||||
* it; moving it would rewrite the history in front of the new turn. */
|
||||
const submittedFileIds = new Set(
|
||||
(this.options.req?.body?.files ?? []).map((file) => file?.file_id),
|
||||
);
|
||||
for (const attachment of this.options.attachments) {
|
||||
if (attachment?.file_id) {
|
||||
if (attachment?.file_id && submittedFileIds.has(attachment.file_id)) {
|
||||
contextSeen.add(attachment.file_id);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -537,8 +537,11 @@ class AgentClient extends BaseClient {
|
|||
const currentAttachments = this.options.attachments ? await this.options.attachments : [];
|
||||
const compatibleHistoricalAttachments =
|
||||
this.getModelBoundAttachmentsForEndpoint(historicalAttachments);
|
||||
const compatibleCurrentAttachments =
|
||||
this.getModelBoundAttachmentsForEndpoint(currentAttachments);
|
||||
/* A tool file refreshed this turn is replayed with its message; count it once. */
|
||||
const replayedFileIds = collectFileIds(compatibleHistoricalAttachments);
|
||||
const compatibleCurrentAttachments = this.getModelBoundAttachmentsForEndpoint(
|
||||
currentAttachments,
|
||||
).filter((file) => !replayedFileIds.has(file?.file_id));
|
||||
this.turnSteerAttachmentHistory = undefined;
|
||||
this.turnHistoricalAttachmentIds = collectHistoricalAttachmentIds(
|
||||
historicalAttachments,
|
||||
|
|
@ -2482,6 +2485,12 @@ class AgentClient extends BaseClient {
|
|||
const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : [];
|
||||
const modelBoundRequestAttachments =
|
||||
this.getModelBoundAttachmentsForEndpoint(requestAttachments);
|
||||
/* A tool file the turn refreshes keeps its text and media in the message it was
|
||||
* attached to; the latest message only takes files no earlier message carries. */
|
||||
const replayedFileIds = collectFileIds(Object.values(this.message_file_map ?? {}).flat());
|
||||
const isNotReplayed = (file) => !replayedFileIds.has(file?.file_id);
|
||||
const turnAttachments = requestAttachments.filter(isNotReplayed);
|
||||
const turnModelBoundAttachments = modelBoundRequestAttachments.filter(isNotReplayed);
|
||||
const retainedHistoricalFileContexts =
|
||||
this.options.resendFiles === false
|
||||
? orderedMessages
|
||||
|
|
@ -2497,7 +2506,7 @@ class AgentClient extends BaseClient {
|
|||
const sharedAttachmentFiles = [
|
||||
...Object.values(this.message_file_map ?? {}).flat(),
|
||||
...(this.modelBoundHistoricalSteerFiles ?? []),
|
||||
...modelBoundRequestAttachments,
|
||||
...turnModelBoundAttachments,
|
||||
];
|
||||
const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles);
|
||||
this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({
|
||||
|
|
@ -2563,8 +2572,8 @@ class AgentClient extends BaseClient {
|
|||
|
||||
if (requestAttachmentsSource) {
|
||||
const attachments = this.getProcessableAttachmentsForEndpoint(
|
||||
requestAttachments,
|
||||
modelBoundRequestAttachments,
|
||||
turnAttachments,
|
||||
turnModelBoundAttachments,
|
||||
);
|
||||
const latestMessage = orderedMessages[orderedMessages.length - 1];
|
||||
this.modelBoundCurrentFiles = [...modelBoundRequestAttachments];
|
||||
|
|
@ -2582,20 +2591,20 @@ class AgentClient extends BaseClient {
|
|||
/** Agent-scoped extraction only depends on the shared attachment IDs. */
|
||||
agentScopedContextPromise = startAgentScopedContext();
|
||||
|
||||
if (this.message_file_map) {
|
||||
this.message_file_map[latestMessage.messageId] = attachments;
|
||||
} else {
|
||||
this.message_file_map = {
|
||||
[latestMessage.messageId]: attachments,
|
||||
};
|
||||
}
|
||||
this.message_file_map ??= {};
|
||||
this.message_file_map[latestMessage.messageId] = [
|
||||
...(this.message_file_map[latestMessage.messageId] ?? []),
|
||||
...attachments,
|
||||
];
|
||||
|
||||
const [, files] = await Promise.all([
|
||||
this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments),
|
||||
this.addFileContextToMessage(latestMessage, turnModelBoundAttachments),
|
||||
this.processAttachments(latestMessage, attachments),
|
||||
]);
|
||||
|
||||
this.options.attachments = files;
|
||||
this.options.attachments = files.concat(
|
||||
requestAttachments.filter((file) => !isNotReplayed(file)),
|
||||
);
|
||||
} else {
|
||||
agentScopedContextPromise = startAgentScopedContext();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6881,6 +6881,108 @@ describe('AgentClient - titleConvo', () => {
|
|||
);
|
||||
});
|
||||
|
||||
describe('file context placement across turns', () => {
|
||||
const FILE_TEXT = 'services:\n web: {}';
|
||||
const makeHistory = () => [
|
||||
{
|
||||
messageId: 'm1',
|
||||
parentMessageId: null,
|
||||
sender: 'User',
|
||||
text: 'How many services?',
|
||||
isCreatedByUser: true,
|
||||
files: [{ file_id: 'doc' }],
|
||||
},
|
||||
{
|
||||
messageId: 'm2',
|
||||
parentMessageId: 'm1',
|
||||
sender: 'Assistant',
|
||||
text: 'One.',
|
||||
isCreatedByUser: false,
|
||||
},
|
||||
{
|
||||
messageId: 'm3',
|
||||
parentMessageId: 'm2',
|
||||
sender: 'User',
|
||||
text: 'Thanks.',
|
||||
isCreatedByUser: true,
|
||||
},
|
||||
];
|
||||
const buildPrompt = async ({ attached, requestFiles = [], resendFiles = true }) => {
|
||||
const file = makeTextFile('doc', 'compose.yaml', FILE_TEXT);
|
||||
require('~/models').getFiles.mockResolvedValue([file]);
|
||||
client.options.resendFiles = resendFiles;
|
||||
client.message_file_map = {};
|
||||
mockReq.body.files = requestFiles;
|
||||
client.options.attachments = attached ? [file] : [];
|
||||
const messages = await client.addPreviousAttachments(makeHistory());
|
||||
const { prompt } = await client.buildMessages(messages, 'm3', {});
|
||||
return prompt.map((message) => message.content);
|
||||
};
|
||||
const countOccurrences = (prompt) => JSON.stringify(prompt).split('web: {}').length - 1;
|
||||
|
||||
it('keeps the file in its own message when it also arrives as a tool attachment', async () => {
|
||||
const limits = jest.spyOn(client, 'assertTurnAttachmentLimits');
|
||||
const beforeToolRound = await buildPrompt({ attached: false });
|
||||
limits.mockClear();
|
||||
const afterToolRound = await buildPrompt({ attached: true });
|
||||
|
||||
expect(afterToolRound).toEqual(beforeToolRound);
|
||||
expect(limits).toHaveBeenCalled();
|
||||
for (const [shared] of limits.mock.calls) {
|
||||
expect(shared.filter(({ file_id }) => file_id === 'doc')).toHaveLength(1);
|
||||
}
|
||||
expect(countOccurrences(afterToolRound)).toBe(1);
|
||||
expect(afterToolRound[0]).toContain('web: {}');
|
||||
expect(afterToolRound[2]).not.toContain('web: {}');
|
||||
expect(client.options.attachments.map(({ file_id }) => file_id)).toEqual(['doc']);
|
||||
});
|
||||
|
||||
it('places a file submitted again with this request on the latest message only', async () => {
|
||||
const prompt = await buildPrompt({ attached: true, requestFiles: [{ file_id: 'doc' }] });
|
||||
|
||||
expect(countOccurrences(prompt)).toBe(1);
|
||||
expect(prompt[0]).not.toContain('web: {}');
|
||||
expect(prompt[2]).toContain('web: {}');
|
||||
expect(prompt).toMatchInlineSnapshot(`
|
||||
[
|
||||
"How many services?",
|
||||
"One.",
|
||||
"Attached document(s):
|
||||
\`\`\`md# "compose.yaml"
|
||||
services:
|
||||
web: {}
|
||||
|
||||
\`\`\`
|
||||
Thanks.",
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
it('places the request file on the latest message when files are not resent', async () => {
|
||||
const prompt = await buildPrompt({
|
||||
attached: true,
|
||||
requestFiles: [{ file_id: 'doc' }],
|
||||
resendFiles: false,
|
||||
});
|
||||
|
||||
expect(countOccurrences(prompt)).toBe(1);
|
||||
expect(prompt[2]).toContain('web: {}');
|
||||
expect(prompt).toMatchInlineSnapshot(`
|
||||
[
|
||||
"How many services?",
|
||||
"One.",
|
||||
"Attached document(s):
|
||||
\`\`\`md# "compose.yaml"
|
||||
services:
|
||||
web: {}
|
||||
|
||||
\`\`\`
|
||||
Thanks.",
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
it('counts resubmitted historical files during steering without mutating rejected state', () => {
|
||||
const historical = Array.from({ length: 11 }, (_, index) =>
|
||||
makeTextFile(`history-${index}`, `history-${index}.txt`, 'context'),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue