mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 14:08:21 +00:00
Merge 2296a180ae into b2244f7b53
This commit is contained in:
commit
3a117fe256
3 changed files with 159 additions and 8 deletions
|
|
@ -1921,8 +1921,14 @@ class BaseClient {
|
||||||
const attachmentsProcessed =
|
const attachmentsProcessed =
|
||||||
this.options.attachments && !(this.options.attachments instanceof Promise);
|
this.options.attachments && !(this.options.attachments instanceof Promise);
|
||||||
if (attachmentsProcessed) {
|
if (attachmentsProcessed) {
|
||||||
|
/* Only files this request submits move to the current message. A tool file the
|
||||||
|
* turn refreshes (code environment) keeps its context in the message that carries
|
||||||
|
* it; moving it would rewrite the history in front of the new turn. */
|
||||||
|
const submittedFileIds = new Set(
|
||||||
|
(this.options.req?.body?.files ?? []).map((file) => file?.file_id),
|
||||||
|
);
|
||||||
for (const attachment of this.options.attachments) {
|
for (const attachment of this.options.attachments) {
|
||||||
if (attachment?.file_id) {
|
if (attachment?.file_id && submittedFileIds.has(attachment.file_id)) {
|
||||||
contextSeen.add(attachment.file_id);
|
contextSeen.add(attachment.file_id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -546,8 +546,11 @@ class AgentClient extends BaseClient {
|
||||||
const currentAttachments = this.options.attachments ? await this.options.attachments : [];
|
const currentAttachments = this.options.attachments ? await this.options.attachments : [];
|
||||||
const compatibleHistoricalAttachments =
|
const compatibleHistoricalAttachments =
|
||||||
this.getModelBoundAttachmentsForEndpoint(historicalAttachments);
|
this.getModelBoundAttachmentsForEndpoint(historicalAttachments);
|
||||||
const compatibleCurrentAttachments =
|
/* A tool file refreshed this turn is replayed with its message; count it once. */
|
||||||
this.getModelBoundAttachmentsForEndpoint(currentAttachments);
|
const replayedFileIds = collectFileIds(compatibleHistoricalAttachments);
|
||||||
|
const compatibleCurrentAttachments = this.getModelBoundAttachmentsForEndpoint(
|
||||||
|
currentAttachments,
|
||||||
|
).filter((file) => !replayedFileIds.has(file?.file_id));
|
||||||
this.turnSteerAttachmentHistory = undefined;
|
this.turnSteerAttachmentHistory = undefined;
|
||||||
this.turnHistoricalAttachmentIds = collectHistoricalAttachmentIds(
|
this.turnHistoricalAttachmentIds = collectHistoricalAttachmentIds(
|
||||||
historicalAttachments,
|
historicalAttachments,
|
||||||
|
|
@ -2488,6 +2491,17 @@ class AgentClient extends BaseClient {
|
||||||
const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : [];
|
const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : [];
|
||||||
const modelBoundRequestAttachments =
|
const modelBoundRequestAttachments =
|
||||||
this.getModelBoundAttachmentsForEndpoint(requestAttachments);
|
this.getModelBoundAttachmentsForEndpoint(requestAttachments);
|
||||||
|
/* A tool file the turn refreshes keeps its text and media in the message it was
|
||||||
|
* attached to; the latest message only takes files no earlier message carries. */
|
||||||
|
const latestMessageId = orderedMessages[orderedMessages.length - 1]?.messageId;
|
||||||
|
const replayedFileIds = collectFileIds(
|
||||||
|
Object.entries(this.message_file_map ?? {})
|
||||||
|
.filter(([messageId]) => messageId !== latestMessageId)
|
||||||
|
.flatMap(([, files]) => files),
|
||||||
|
);
|
||||||
|
const isNotReplayed = (file) => !replayedFileIds.has(file?.file_id);
|
||||||
|
const turnAttachments = requestAttachments.filter(isNotReplayed);
|
||||||
|
const turnModelBoundAttachments = modelBoundRequestAttachments.filter(isNotReplayed);
|
||||||
const retainedHistoricalFileContexts =
|
const retainedHistoricalFileContexts =
|
||||||
this.options.resendFiles === false
|
this.options.resendFiles === false
|
||||||
? orderedMessages
|
? orderedMessages
|
||||||
|
|
@ -2503,7 +2517,7 @@ class AgentClient extends BaseClient {
|
||||||
const sharedAttachmentFiles = [
|
const sharedAttachmentFiles = [
|
||||||
...Object.values(this.message_file_map ?? {}).flat(),
|
...Object.values(this.message_file_map ?? {}).flat(),
|
||||||
...(this.modelBoundHistoricalSteerFiles ?? []),
|
...(this.modelBoundHistoricalSteerFiles ?? []),
|
||||||
...modelBoundRequestAttachments,
|
...turnModelBoundAttachments,
|
||||||
];
|
];
|
||||||
const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles);
|
const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles);
|
||||||
this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({
|
this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({
|
||||||
|
|
@ -2569,8 +2583,8 @@ class AgentClient extends BaseClient {
|
||||||
|
|
||||||
if (requestAttachmentsSource) {
|
if (requestAttachmentsSource) {
|
||||||
const attachments = this.getProcessableAttachmentsForEndpoint(
|
const attachments = this.getProcessableAttachmentsForEndpoint(
|
||||||
requestAttachments,
|
turnAttachments,
|
||||||
modelBoundRequestAttachments,
|
turnModelBoundAttachments,
|
||||||
);
|
);
|
||||||
const latestMessage = orderedMessages[orderedMessages.length - 1];
|
const latestMessage = orderedMessages[orderedMessages.length - 1];
|
||||||
this.modelBoundCurrentFiles = [...modelBoundRequestAttachments];
|
this.modelBoundCurrentFiles = [...modelBoundRequestAttachments];
|
||||||
|
|
@ -2597,11 +2611,13 @@ class AgentClient extends BaseClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
const [, files] = await Promise.all([
|
const [, files] = await Promise.all([
|
||||||
this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments),
|
this.addFileContextToMessage(latestMessage, turnModelBoundAttachments),
|
||||||
this.processAttachments(latestMessage, attachments),
|
this.processAttachments(latestMessage, attachments),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
this.options.attachments = files;
|
this.options.attachments = files.concat(
|
||||||
|
requestAttachments.filter((file) => !isNotReplayed(file)),
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
agentScopedContextPromise = startAgentScopedContext();
|
agentScopedContextPromise = startAgentScopedContext();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6920,6 +6920,135 @@ describe('AgentClient - titleConvo', () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('file context placement across turns', () => {
|
||||||
|
const FILE_TEXT = 'services:\n web: {}';
|
||||||
|
const makeHistory = () => [
|
||||||
|
{
|
||||||
|
messageId: 'm1',
|
||||||
|
parentMessageId: null,
|
||||||
|
sender: 'User',
|
||||||
|
text: 'How many services?',
|
||||||
|
isCreatedByUser: true,
|
||||||
|
files: [{ file_id: 'doc' }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
messageId: 'm2',
|
||||||
|
parentMessageId: 'm1',
|
||||||
|
sender: 'Assistant',
|
||||||
|
text: 'One.',
|
||||||
|
isCreatedByUser: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
messageId: 'm3',
|
||||||
|
parentMessageId: 'm2',
|
||||||
|
sender: 'User',
|
||||||
|
text: 'Thanks.',
|
||||||
|
isCreatedByUser: true,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const buildPrompt = async ({ attached, requestFiles = [], resendFiles = true }) => {
|
||||||
|
const file = makeTextFile('doc', 'compose.yaml', FILE_TEXT);
|
||||||
|
require('~/models').getFiles.mockResolvedValue([file]);
|
||||||
|
client.options.resendFiles = resendFiles;
|
||||||
|
client.message_file_map = {};
|
||||||
|
mockReq.body.files = requestFiles;
|
||||||
|
client.options.attachments = attached ? [file] : [];
|
||||||
|
const messages = await client.addPreviousAttachments(makeHistory());
|
||||||
|
const { prompt } = await client.buildMessages(messages, 'm3', {});
|
||||||
|
return prompt.map((message) => message.content);
|
||||||
|
};
|
||||||
|
const countOccurrences = (prompt) => JSON.stringify(prompt).split('web: {}').length - 1;
|
||||||
|
|
||||||
|
it('keeps the file in its own message when it also arrives as a tool attachment', async () => {
|
||||||
|
const limits = jest.spyOn(client, 'assertTurnAttachmentLimits');
|
||||||
|
const beforeToolRound = await buildPrompt({ attached: false });
|
||||||
|
limits.mockClear();
|
||||||
|
const afterToolRound = await buildPrompt({ attached: true });
|
||||||
|
|
||||||
|
expect(afterToolRound).toEqual(beforeToolRound);
|
||||||
|
expect(limits).toHaveBeenCalled();
|
||||||
|
for (const [shared] of limits.mock.calls) {
|
||||||
|
expect(shared.filter(({ file_id }) => file_id === 'doc')).toHaveLength(1);
|
||||||
|
}
|
||||||
|
expect(countOccurrences(afterToolRound)).toBe(1);
|
||||||
|
expect(afterToolRound[0]).toContain('web: {}');
|
||||||
|
expect(afterToolRound[2]).not.toContain('web: {}');
|
||||||
|
expect(client.options.attachments.map(({ file_id }) => file_id)).toEqual(['doc']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('places a file submitted again with this request on the latest message only', async () => {
|
||||||
|
const prompt = await buildPrompt({ attached: true, requestFiles: [{ file_id: 'doc' }] });
|
||||||
|
|
||||||
|
expect(countOccurrences(prompt)).toBe(1);
|
||||||
|
expect(prompt[0]).not.toContain('web: {}');
|
||||||
|
expect(prompt[2]).toContain('web: {}');
|
||||||
|
expect(prompt).toMatchInlineSnapshot(`
|
||||||
|
[
|
||||||
|
"How many services?",
|
||||||
|
"One.",
|
||||||
|
"Attached document(s):
|
||||||
|
\`\`\`md# "compose.yaml"
|
||||||
|
services:
|
||||||
|
web: {}
|
||||||
|
|
||||||
|
\`\`\`
|
||||||
|
Thanks.",
|
||||||
|
]
|
||||||
|
`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('places the request file on the latest message when files are not resent', async () => {
|
||||||
|
const prompt = await buildPrompt({
|
||||||
|
attached: true,
|
||||||
|
requestFiles: [{ file_id: 'doc' }],
|
||||||
|
resendFiles: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(countOccurrences(prompt)).toBe(1);
|
||||||
|
expect(prompt[2]).toContain('web: {}');
|
||||||
|
expect(prompt).toMatchInlineSnapshot(`
|
||||||
|
[
|
||||||
|
"How many services?",
|
||||||
|
"One.",
|
||||||
|
"Attached document(s):
|
||||||
|
\`\`\`md# "compose.yaml"
|
||||||
|
services:
|
||||||
|
web: {}
|
||||||
|
|
||||||
|
\`\`\`
|
||||||
|
Thanks.",
|
||||||
|
]
|
||||||
|
`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps an image that a tool refreshes in its own message', async () => {
|
||||||
|
const image = makeUploadedFile('shot', 'shot.png', 'image/png');
|
||||||
|
require('~/models').getFiles.mockResolvedValue([image]);
|
||||||
|
client.addImageURLs = jest.fn(async (message, files) => {
|
||||||
|
message.image_urls = files.map(({ file_id }) => ({
|
||||||
|
type: 'image_url',
|
||||||
|
image_url: { url: `data:image/png;base64,${file_id}` },
|
||||||
|
}));
|
||||||
|
return files;
|
||||||
|
});
|
||||||
|
client.options.resendFiles = true;
|
||||||
|
client.message_file_map = {};
|
||||||
|
mockReq.body.files = [];
|
||||||
|
client.options.attachments = [image];
|
||||||
|
const history = makeHistory();
|
||||||
|
history[0].files = [{ file_id: 'shot' }];
|
||||||
|
|
||||||
|
const messages = await client.addPreviousAttachments(history);
|
||||||
|
const { prompt } = await client.buildMessages(messages, 'm3', {});
|
||||||
|
|
||||||
|
const carriers = prompt.flatMap((message, index) =>
|
||||||
|
JSON.stringify(message.content).includes('base64,shot') ? [index] : [],
|
||||||
|
);
|
||||||
|
expect(carriers).toEqual([0]);
|
||||||
|
expect(client.addImageURLs).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('counts resubmitted historical files during steering without mutating rejected state', () => {
|
it('counts resubmitted historical files during steering without mutating rejected state', () => {
|
||||||
const historical = Array.from({ length: 11 }, (_, index) =>
|
const historical = Array.from({ length: 11 }, (_, index) =>
|
||||||
makeTextFile(`history-${index}`, `history-${index}.txt`, 'context'),
|
makeTextFile(`history-${index}`, `history-${index}.txt`, 'context'),
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue