From ac8cdcc1ea0cc32af80954a5698bdd2e37920447 Mon Sep 17 00:00:00 2001 From: shars <32410570+Shabonasar@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:32:23 +0300 Subject: [PATCH 1/2] fix(agents): keep file context in the message that carries the file With execute_code and resendFiles, a code-environment file is loaded into the turn's attachments on every turn. addPreviousAttachments treated every attachment as submitted by the current request, so the file's text was cut from the message it was attached to and added to the newest message. The bytes in the middle of the history changed each turn and the provider cache stopped at that point. Only files named in req.body.files move to the current message. A refreshed tool file stays in its own message, and the latest message skips files an earlier message already carries, so the text appears once and the attachment limits count it once. Reported-by: FFourtyTwo --- api/app/clients/BaseClient.js | 8 +- api/server/controllers/agents/client.js | 37 ++++--- api/server/controllers/agents/client.test.js | 102 +++++++++++++++++++ 3 files changed, 132 insertions(+), 15 deletions(-) diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 4e74116405..8f069946cf 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1936,8 +1936,14 @@ class BaseClient { const attachmentsProcessed = this.options.attachments && !(this.options.attachments instanceof Promise); if (attachmentsProcessed) { + /* Only files this request submits move to the current message. A tool file the + * turn refreshes (code environment) keeps its context in the message that carries + * it; moving it would rewrite the history in front of the new turn. */ + const submittedFileIds = new Set( + (this.options.req?.body?.files ?? []).map((file) => file?.file_id), + ); for (const attachment of this.options.attachments) { - if (attachment?.file_id) { + if (attachment?.file_id && submittedFileIds.has(attachment.file_id)) { contextSeen.add(attachment.file_id); } } diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 1a68393c4f..1f8f529996 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -537,8 +537,11 @@ class AgentClient extends BaseClient { const currentAttachments = this.options.attachments ? await this.options.attachments : []; const compatibleHistoricalAttachments = this.getModelBoundAttachmentsForEndpoint(historicalAttachments); - const compatibleCurrentAttachments = - this.getModelBoundAttachmentsForEndpoint(currentAttachments); + /* A tool file refreshed this turn is replayed with its message; count it once. */ + const replayedFileIds = collectFileIds(compatibleHistoricalAttachments); + const compatibleCurrentAttachments = this.getModelBoundAttachmentsForEndpoint( + currentAttachments, + ).filter((file) => !replayedFileIds.has(file?.file_id)); this.turnSteerAttachmentHistory = undefined; this.turnHistoricalAttachmentIds = collectHistoricalAttachmentIds( historicalAttachments, @@ -2482,6 +2485,12 @@ class AgentClient extends BaseClient { const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : []; const modelBoundRequestAttachments = this.getModelBoundAttachmentsForEndpoint(requestAttachments); + /* A tool file the turn refreshes keeps its text and media in the message it was + * attached to; the latest message only takes files no earlier message carries. */ + const replayedFileIds = collectFileIds(Object.values(this.message_file_map ?? {}).flat()); + const isNotReplayed = (file) => !replayedFileIds.has(file?.file_id); + const turnAttachments = requestAttachments.filter(isNotReplayed); + const turnModelBoundAttachments = modelBoundRequestAttachments.filter(isNotReplayed); const retainedHistoricalFileContexts = this.options.resendFiles === false ? orderedMessages @@ -2497,7 +2506,7 @@ class AgentClient extends BaseClient { const sharedAttachmentFiles = [ ...Object.values(this.message_file_map ?? {}).flat(), ...(this.modelBoundHistoricalSteerFiles ?? []), - ...modelBoundRequestAttachments, + ...turnModelBoundAttachments, ]; const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles); this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ @@ -2563,8 +2572,8 @@ class AgentClient extends BaseClient { if (requestAttachmentsSource) { const attachments = this.getProcessableAttachmentsForEndpoint( - requestAttachments, - modelBoundRequestAttachments, + turnAttachments, + turnModelBoundAttachments, ); const latestMessage = orderedMessages[orderedMessages.length - 1]; this.modelBoundCurrentFiles = [...modelBoundRequestAttachments]; @@ -2582,20 +2591,20 @@ class AgentClient extends BaseClient { /** Agent-scoped extraction only depends on the shared attachment IDs. */ agentScopedContextPromise = startAgentScopedContext(); - if (this.message_file_map) { - this.message_file_map[latestMessage.messageId] = attachments; - } else { - this.message_file_map = { - [latestMessage.messageId]: attachments, - }; - } + this.message_file_map ??= {}; + this.message_file_map[latestMessage.messageId] = [ + ...(this.message_file_map[latestMessage.messageId] ?? []), + ...attachments, + ]; const [, files] = await Promise.all([ - this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments), + this.addFileContextToMessage(latestMessage, turnModelBoundAttachments), this.processAttachments(latestMessage, attachments), ]); - this.options.attachments = files; + this.options.attachments = files.concat( + requestAttachments.filter((file) => !isNotReplayed(file)), + ); } else { agentScopedContextPromise = startAgentScopedContext(); } diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 9b5cabe411..e5595e0405 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -6881,6 +6881,108 @@ describe('AgentClient - titleConvo', () => { ); }); + describe('file context placement across turns', () => { + const FILE_TEXT = 'services:\n web: {}'; + const makeHistory = () => [ + { + messageId: 'm1', + parentMessageId: null, + sender: 'User', + text: 'How many services?', + isCreatedByUser: true, + files: [{ file_id: 'doc' }], + }, + { + messageId: 'm2', + parentMessageId: 'm1', + sender: 'Assistant', + text: 'One.', + isCreatedByUser: false, + }, + { + messageId: 'm3', + parentMessageId: 'm2', + sender: 'User', + text: 'Thanks.', + isCreatedByUser: true, + }, + ]; + const buildPrompt = async ({ attached, requestFiles = [], resendFiles = true }) => { + const file = makeTextFile('doc', 'compose.yaml', FILE_TEXT); + require('~/models').getFiles.mockResolvedValue([file]); + client.options.resendFiles = resendFiles; + client.message_file_map = {}; + mockReq.body.files = requestFiles; + client.options.attachments = attached ? [file] : []; + const messages = await client.addPreviousAttachments(makeHistory()); + const { prompt } = await client.buildMessages(messages, 'm3', {}); + return prompt.map((message) => message.content); + }; + const countOccurrences = (prompt) => JSON.stringify(prompt).split('web: {}').length - 1; + + it('keeps the file in its own message when it also arrives as a tool attachment', async () => { + const limits = jest.spyOn(client, 'assertTurnAttachmentLimits'); + const beforeToolRound = await buildPrompt({ attached: false }); + limits.mockClear(); + const afterToolRound = await buildPrompt({ attached: true }); + + expect(afterToolRound).toEqual(beforeToolRound); + expect(limits).toHaveBeenCalled(); + for (const [shared] of limits.mock.calls) { + expect(shared.filter(({ file_id }) => file_id === 'doc')).toHaveLength(1); + } + expect(countOccurrences(afterToolRound)).toBe(1); + expect(afterToolRound[0]).toContain('web: {}'); + expect(afterToolRound[2]).not.toContain('web: {}'); + expect(client.options.attachments.map(({ file_id }) => file_id)).toEqual(['doc']); + }); + + it('places a file submitted again with this request on the latest message only', async () => { + const prompt = await buildPrompt({ attached: true, requestFiles: [{ file_id: 'doc' }] }); + + expect(countOccurrences(prompt)).toBe(1); + expect(prompt[0]).not.toContain('web: {}'); + expect(prompt[2]).toContain('web: {}'); + expect(prompt).toMatchInlineSnapshot(` + [ + "How many services?", + "One.", + "Attached document(s): + \`\`\`md# "compose.yaml" + services: + web: {} + + \`\`\` + Thanks.", + ] + `); + }); + + it('places the request file on the latest message when files are not resent', async () => { + const prompt = await buildPrompt({ + attached: true, + requestFiles: [{ file_id: 'doc' }], + resendFiles: false, + }); + + expect(countOccurrences(prompt)).toBe(1); + expect(prompt[2]).toContain('web: {}'); + expect(prompt).toMatchInlineSnapshot(` + [ + "How many services?", + "One.", + "Attached document(s): + \`\`\`md# "compose.yaml" + services: + web: {} + + \`\`\` + Thanks.", + ] + `); + }); + }); + it('counts resubmitted historical files during steering without mutating rejected state', () => { const historical = Array.from({ length: 11 }, (_, index) => makeTextFile(`history-${index}`, `history-${index}.txt`, 'context'), From aeaf4875fdf6eb7cc054179e5cf1e93dfa0571c6 Mon Sep 17 00:00:00 2001 From: shars <32410570+Shabonasar@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:13:58 +0300 Subject: [PATCH 2/2] fix(agents): keep the latest message's own files and test tool-refreshed images The check for files already carried by an earlier message now ignores the latest message's own entry, so the latest message map entry is replaced as before and regenerating a message with files behaves as it did. Add a test that an image refreshed by a tool stays in the message that carries it. --- api/server/controllers/agents/client.js | 19 +++++++++----- api/server/controllers/agents/client.test.js | 27 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 1f8f529996..245c1fa848 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -2487,7 +2487,12 @@ class AgentClient extends BaseClient { this.getModelBoundAttachmentsForEndpoint(requestAttachments); /* A tool file the turn refreshes keeps its text and media in the message it was * attached to; the latest message only takes files no earlier message carries. */ - const replayedFileIds = collectFileIds(Object.values(this.message_file_map ?? {}).flat()); + const latestMessageId = orderedMessages[orderedMessages.length - 1]?.messageId; + const replayedFileIds = collectFileIds( + Object.entries(this.message_file_map ?? {}) + .filter(([messageId]) => messageId !== latestMessageId) + .flatMap(([, files]) => files), + ); const isNotReplayed = (file) => !replayedFileIds.has(file?.file_id); const turnAttachments = requestAttachments.filter(isNotReplayed); const turnModelBoundAttachments = modelBoundRequestAttachments.filter(isNotReplayed); @@ -2591,11 +2596,13 @@ class AgentClient extends BaseClient { /** Agent-scoped extraction only depends on the shared attachment IDs. */ agentScopedContextPromise = startAgentScopedContext(); - this.message_file_map ??= {}; - this.message_file_map[latestMessage.messageId] = [ - ...(this.message_file_map[latestMessage.messageId] ?? []), - ...attachments, - ]; + if (this.message_file_map) { + this.message_file_map[latestMessage.messageId] = attachments; + } else { + this.message_file_map = { + [latestMessage.messageId]: attachments, + }; + } const [, files] = await Promise.all([ this.addFileContextToMessage(latestMessage, turnModelBoundAttachments), diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index e5595e0405..78b6e2e0c1 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -6981,6 +6981,33 @@ describe('AgentClient - titleConvo', () => { ] `); }); + + it('keeps an image that a tool refreshes in its own message', async () => { + const image = makeUploadedFile('shot', 'shot.png', 'image/png'); + require('~/models').getFiles.mockResolvedValue([image]); + client.addImageURLs = jest.fn(async (message, files) => { + message.image_urls = files.map(({ file_id }) => ({ + type: 'image_url', + image_url: { url: `data:image/png;base64,${file_id}` }, + })); + return files; + }); + client.options.resendFiles = true; + client.message_file_map = {}; + mockReq.body.files = []; + client.options.attachments = [image]; + const history = makeHistory(); + history[0].files = [{ file_id: 'shot' }]; + + const messages = await client.addPreviousAttachments(history); + const { prompt } = await client.buildMessages(messages, 'm3', {}); + + const carriers = prompt.flatMap((message, index) => + JSON.stringify(message.content).includes('base64,shot') ? [index] : [], + ); + expect(carriers).toEqual([0]); + expect(client.addImageURLs).toHaveBeenCalledTimes(1); + }); }); it('counts resubmitted historical files during steering without mutating rejected state', () => {