diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index bfe2c6dde3..fe039603c2 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1921,8 +1921,14 @@ class BaseClient { const attachmentsProcessed = this.options.attachments && !(this.options.attachments instanceof Promise); if (attachmentsProcessed) { + /* Only files this request submits move to the current message. A tool file the + * turn refreshes (code environment) keeps its context in the message that carries + * it; moving it would rewrite the history in front of the new turn. */ + const submittedFileIds = new Set( + (this.options.req?.body?.files ?? []).map((file) => file?.file_id), + ); for (const attachment of this.options.attachments) { - if (attachment?.file_id) { + if (attachment?.file_id && submittedFileIds.has(attachment.file_id)) { contextSeen.add(attachment.file_id); } } diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index ee3b217dc4..0eb9538816 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -546,8 +546,11 @@ class AgentClient extends BaseClient { const currentAttachments = this.options.attachments ? await this.options.attachments : []; const compatibleHistoricalAttachments = this.getModelBoundAttachmentsForEndpoint(historicalAttachments); - const compatibleCurrentAttachments = - this.getModelBoundAttachmentsForEndpoint(currentAttachments); + /* A tool file refreshed this turn is replayed with its message; count it once. */ + const replayedFileIds = collectFileIds(compatibleHistoricalAttachments); + const compatibleCurrentAttachments = this.getModelBoundAttachmentsForEndpoint( + currentAttachments, + ).filter((file) => !replayedFileIds.has(file?.file_id)); this.turnSteerAttachmentHistory = undefined; this.turnHistoricalAttachmentIds = collectHistoricalAttachmentIds( historicalAttachments, @@ -2488,6 +2491,17 @@ class AgentClient extends BaseClient { const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : []; const modelBoundRequestAttachments = this.getModelBoundAttachmentsForEndpoint(requestAttachments); + /* A tool file the turn refreshes keeps its text and media in the message it was + * attached to; the latest message only takes files no earlier message carries. */ + const latestMessageId = orderedMessages[orderedMessages.length - 1]?.messageId; + const replayedFileIds = collectFileIds( + Object.entries(this.message_file_map ?? {}) + .filter(([messageId]) => messageId !== latestMessageId) + .flatMap(([, files]) => files), + ); + const isNotReplayed = (file) => !replayedFileIds.has(file?.file_id); + const turnAttachments = requestAttachments.filter(isNotReplayed); + const turnModelBoundAttachments = modelBoundRequestAttachments.filter(isNotReplayed); const retainedHistoricalFileContexts = this.options.resendFiles === false ? orderedMessages @@ -2503,7 +2517,7 @@ class AgentClient extends BaseClient { const sharedAttachmentFiles = [ ...Object.values(this.message_file_map ?? {}).flat(), ...(this.modelBoundHistoricalSteerFiles ?? []), - ...modelBoundRequestAttachments, + ...turnModelBoundAttachments, ]; const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles); this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ @@ -2569,8 +2583,8 @@ class AgentClient extends BaseClient { if (requestAttachmentsSource) { const attachments = this.getProcessableAttachmentsForEndpoint( - requestAttachments, - modelBoundRequestAttachments, + turnAttachments, + turnModelBoundAttachments, ); const latestMessage = orderedMessages[orderedMessages.length - 1]; this.modelBoundCurrentFiles = [...modelBoundRequestAttachments]; @@ -2597,11 +2611,13 @@ class AgentClient extends BaseClient { } const [, files] = await Promise.all([ - this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments), + this.addFileContextToMessage(latestMessage, turnModelBoundAttachments), this.processAttachments(latestMessage, attachments), ]); - this.options.attachments = files; + this.options.attachments = files.concat( + requestAttachments.filter((file) => !isNotReplayed(file)), + ); } else { agentScopedContextPromise = startAgentScopedContext(); } diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 819fcd8a55..425fd24f06 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -6920,6 +6920,135 @@ describe('AgentClient - titleConvo', () => { ); }); + describe('file context placement across turns', () => { + const FILE_TEXT = 'services:\n web: {}'; + const makeHistory = () => [ + { + messageId: 'm1', + parentMessageId: null, + sender: 'User', + text: 'How many services?', + isCreatedByUser: true, + files: [{ file_id: 'doc' }], + }, + { + messageId: 'm2', + parentMessageId: 'm1', + sender: 'Assistant', + text: 'One.', + isCreatedByUser: false, + }, + { + messageId: 'm3', + parentMessageId: 'm2', + sender: 'User', + text: 'Thanks.', + isCreatedByUser: true, + }, + ]; + const buildPrompt = async ({ attached, requestFiles = [], resendFiles = true }) => { + const file = makeTextFile('doc', 'compose.yaml', FILE_TEXT); + require('~/models').getFiles.mockResolvedValue([file]); + client.options.resendFiles = resendFiles; + client.message_file_map = {}; + mockReq.body.files = requestFiles; + client.options.attachments = attached ? [file] : []; + const messages = await client.addPreviousAttachments(makeHistory()); + const { prompt } = await client.buildMessages(messages, 'm3', {}); + return prompt.map((message) => message.content); + }; + const countOccurrences = (prompt) => JSON.stringify(prompt).split('web: {}').length - 1; + + it('keeps the file in its own message when it also arrives as a tool attachment', async () => { + const limits = jest.spyOn(client, 'assertTurnAttachmentLimits'); + const beforeToolRound = await buildPrompt({ attached: false }); + limits.mockClear(); + const afterToolRound = await buildPrompt({ attached: true }); + + expect(afterToolRound).toEqual(beforeToolRound); + expect(limits).toHaveBeenCalled(); + for (const [shared] of limits.mock.calls) { + expect(shared.filter(({ file_id }) => file_id === 'doc')).toHaveLength(1); + } + expect(countOccurrences(afterToolRound)).toBe(1); + expect(afterToolRound[0]).toContain('web: {}'); + expect(afterToolRound[2]).not.toContain('web: {}'); + expect(client.options.attachments.map(({ file_id }) => file_id)).toEqual(['doc']); + }); + + it('places a file submitted again with this request on the latest message only', async () => { + const prompt = await buildPrompt({ attached: true, requestFiles: [{ file_id: 'doc' }] }); + + expect(countOccurrences(prompt)).toBe(1); + expect(prompt[0]).not.toContain('web: {}'); + expect(prompt[2]).toContain('web: {}'); + expect(prompt).toMatchInlineSnapshot(` + [ + "How many services?", + "One.", + "Attached document(s): + \`\`\`md# "compose.yaml" + services: + web: {} + + \`\`\` + Thanks.", + ] + `); + }); + + it('places the request file on the latest message when files are not resent', async () => { + const prompt = await buildPrompt({ + attached: true, + requestFiles: [{ file_id: 'doc' }], + resendFiles: false, + }); + + expect(countOccurrences(prompt)).toBe(1); + expect(prompt[2]).toContain('web: {}'); + expect(prompt).toMatchInlineSnapshot(` + [ + "How many services?", + "One.", + "Attached document(s): + \`\`\`md# "compose.yaml" + services: + web: {} + + \`\`\` + Thanks.", + ] + `); + }); + + it('keeps an image that a tool refreshes in its own message', async () => { + const image = makeUploadedFile('shot', 'shot.png', 'image/png'); + require('~/models').getFiles.mockResolvedValue([image]); + client.addImageURLs = jest.fn(async (message, files) => { + message.image_urls = files.map(({ file_id }) => ({ + type: 'image_url', + image_url: { url: `data:image/png;base64,${file_id}` }, + })); + return files; + }); + client.options.resendFiles = true; + client.message_file_map = {}; + mockReq.body.files = []; + client.options.attachments = [image]; + const history = makeHistory(); + history[0].files = [{ file_id: 'shot' }]; + + const messages = await client.addPreviousAttachments(history); + const { prompt } = await client.buildMessages(messages, 'm3', {}); + + const carriers = prompt.flatMap((message, index) => + JSON.stringify(message.content).includes('base64,shot') ? [index] : [], + ); + expect(carriers).toEqual([0]); + expect(client.addImageURLs).toHaveBeenCalledTimes(1); + }); + }); + it('counts resubmitted historical files during steering without mutating rejected state', () => { const historical = Array.from({ length: 11 }, (_, index) => makeTextFile(`history-${index}`, `history-${index}.txt`, 'context'),