diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 4e74116405..8f069946cf 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1936,8 +1936,14 @@ class BaseClient { const attachmentsProcessed = this.options.attachments && !(this.options.attachments instanceof Promise); if (attachmentsProcessed) { + /* Only files this request submits move to the current message. A tool file the + * turn refreshes (code environment) keeps its context in the message that carries + * it; moving it would rewrite the history in front of the new turn. */ + const submittedFileIds = new Set( + (this.options.req?.body?.files ?? []).map((file) => file?.file_id), + ); for (const attachment of this.options.attachments) { - if (attachment?.file_id) { + if (attachment?.file_id && submittedFileIds.has(attachment.file_id)) { contextSeen.add(attachment.file_id); } } diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 1a68393c4f..1f8f529996 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -537,8 +537,11 @@ class AgentClient extends BaseClient { const currentAttachments = this.options.attachments ? await this.options.attachments : []; const compatibleHistoricalAttachments = this.getModelBoundAttachmentsForEndpoint(historicalAttachments); - const compatibleCurrentAttachments = - this.getModelBoundAttachmentsForEndpoint(currentAttachments); + /* A tool file refreshed this turn is replayed with its message; count it once. */ + const replayedFileIds = collectFileIds(compatibleHistoricalAttachments); + const compatibleCurrentAttachments = this.getModelBoundAttachmentsForEndpoint( + currentAttachments, + ).filter((file) => !replayedFileIds.has(file?.file_id)); this.turnSteerAttachmentHistory = undefined; this.turnHistoricalAttachmentIds = collectHistoricalAttachmentIds( historicalAttachments, @@ -2482,6 +2485,12 @@ class AgentClient extends BaseClient { const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : []; const modelBoundRequestAttachments = this.getModelBoundAttachmentsForEndpoint(requestAttachments); + /* A tool file the turn refreshes keeps its text and media in the message it was + * attached to; the latest message only takes files no earlier message carries. */ + const replayedFileIds = collectFileIds(Object.values(this.message_file_map ?? {}).flat()); + const isNotReplayed = (file) => !replayedFileIds.has(file?.file_id); + const turnAttachments = requestAttachments.filter(isNotReplayed); + const turnModelBoundAttachments = modelBoundRequestAttachments.filter(isNotReplayed); const retainedHistoricalFileContexts = this.options.resendFiles === false ? orderedMessages @@ -2497,7 +2506,7 @@ class AgentClient extends BaseClient { const sharedAttachmentFiles = [ ...Object.values(this.message_file_map ?? {}).flat(), ...(this.modelBoundHistoricalSteerFiles ?? []), - ...modelBoundRequestAttachments, + ...turnModelBoundAttachments, ]; const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles); this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ @@ -2563,8 +2572,8 @@ class AgentClient extends BaseClient { if (requestAttachmentsSource) { const attachments = this.getProcessableAttachmentsForEndpoint( - requestAttachments, - modelBoundRequestAttachments, + turnAttachments, + turnModelBoundAttachments, ); const latestMessage = orderedMessages[orderedMessages.length - 1]; this.modelBoundCurrentFiles = [...modelBoundRequestAttachments]; @@ -2582,20 +2591,20 @@ class AgentClient extends BaseClient { /** Agent-scoped extraction only depends on the shared attachment IDs. */ agentScopedContextPromise = startAgentScopedContext(); - if (this.message_file_map) { - this.message_file_map[latestMessage.messageId] = attachments; - } else { - this.message_file_map = { - [latestMessage.messageId]: attachments, - }; - } + this.message_file_map ??= {}; + this.message_file_map[latestMessage.messageId] = [ + ...(this.message_file_map[latestMessage.messageId] ?? []), + ...attachments, + ]; const [, files] = await Promise.all([ - this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments), + this.addFileContextToMessage(latestMessage, turnModelBoundAttachments), this.processAttachments(latestMessage, attachments), ]); - this.options.attachments = files; + this.options.attachments = files.concat( + requestAttachments.filter((file) => !isNotReplayed(file)), + ); } else { agentScopedContextPromise = startAgentScopedContext(); } diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 9b5cabe411..e5595e0405 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -6881,6 +6881,108 @@ describe('AgentClient - titleConvo', () => { ); }); + describe('file context placement across turns', () => { + const FILE_TEXT = 'services:\n web: {}'; + const makeHistory = () => [ + { + messageId: 'm1', + parentMessageId: null, + sender: 'User', + text: 'How many services?', + isCreatedByUser: true, + files: [{ file_id: 'doc' }], + }, + { + messageId: 'm2', + parentMessageId: 'm1', + sender: 'Assistant', + text: 'One.', + isCreatedByUser: false, + }, + { + messageId: 'm3', + parentMessageId: 'm2', + sender: 'User', + text: 'Thanks.', + isCreatedByUser: true, + }, + ]; + const buildPrompt = async ({ attached, requestFiles = [], resendFiles = true }) => { + const file = makeTextFile('doc', 'compose.yaml', FILE_TEXT); + require('~/models').getFiles.mockResolvedValue([file]); + client.options.resendFiles = resendFiles; + client.message_file_map = {}; + mockReq.body.files = requestFiles; + client.options.attachments = attached ? [file] : []; + const messages = await client.addPreviousAttachments(makeHistory()); + const { prompt } = await client.buildMessages(messages, 'm3', {}); + return prompt.map((message) => message.content); + }; + const countOccurrences = (prompt) => JSON.stringify(prompt).split('web: {}').length - 1; + + it('keeps the file in its own message when it also arrives as a tool attachment', async () => { + const limits = jest.spyOn(client, 'assertTurnAttachmentLimits'); + const beforeToolRound = await buildPrompt({ attached: false }); + limits.mockClear(); + const afterToolRound = await buildPrompt({ attached: true }); + + expect(afterToolRound).toEqual(beforeToolRound); + expect(limits).toHaveBeenCalled(); + for (const [shared] of limits.mock.calls) { + expect(shared.filter(({ file_id }) => file_id === 'doc')).toHaveLength(1); + } + expect(countOccurrences(afterToolRound)).toBe(1); + expect(afterToolRound[0]).toContain('web: {}'); + expect(afterToolRound[2]).not.toContain('web: {}'); + expect(client.options.attachments.map(({ file_id }) => file_id)).toEqual(['doc']); + }); + + it('places a file submitted again with this request on the latest message only', async () => { + const prompt = await buildPrompt({ attached: true, requestFiles: [{ file_id: 'doc' }] }); + + expect(countOccurrences(prompt)).toBe(1); + expect(prompt[0]).not.toContain('web: {}'); + expect(prompt[2]).toContain('web: {}'); + expect(prompt).toMatchInlineSnapshot(` + [ + "How many services?", + "One.", + "Attached document(s): + \`\`\`md# "compose.yaml" + services: + web: {} + + \`\`\` + Thanks.", + ] + `); + }); + + it('places the request file on the latest message when files are not resent', async () => { + const prompt = await buildPrompt({ + attached: true, + requestFiles: [{ file_id: 'doc' }], + resendFiles: false, + }); + + expect(countOccurrences(prompt)).toBe(1); + expect(prompt[2]).toContain('web: {}'); + expect(prompt).toMatchInlineSnapshot(` + [ + "How many services?", + "One.", + "Attached document(s): + \`\`\`md# "compose.yaml" + services: + web: {} + + \`\`\` + Thanks.", + ] + `); + }); + }); + it('counts resubmitted historical files during steering without mutating rejected state', () => { const historical = Array.from({ length: 11 }, (_, index) => makeTextFile(`history-${index}`, `history-${index}.txt`, 'context'),