🏍️ feat: Apply Stream Metadata Through Shared Chat Handlers (#18398)

This commit is contained in:
Danny Avila 2026-11-18 07:51:48 -05:00 • committed by GitHub
parent 9cfa807950
commit 83bd676dec
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 395 additions and 4 deletions

View file

@ -919,6 +919,215 @@ describe('side frames', () => {
});
});
describe('state beside the message list', () => {
/** Main chat's handlers for that state, recording what they are called with and in what order. */
const record = () => {
const calls: Array<[string, ...unknown[]]> = [];
const log =
(name: string, result?: unknown) =>
(...args: unknown[]): unknown => {
calls.push([name, ...args]);
return result;
};
const init = {
usage: {
bindResponse: log('bindResponse'),
contextHandler: log('context'),
usageHandler: log('usage'),
tapStream: log('tap'),
tapContent: log('tapContent'),
finalizeUsage: log('finalize'),
backfillUsage: log('backfill'),
seedLiveFromContent: log('seedLive'),
},
onAttachment: log('attachment'),
onTitle: log('title'),
steers: {
resolve: log('resolve', ['steer-1']),
markLiveApplied: log('markLive'),
update: log('update'),
seed: log('seed'),
settleApplied: log('settle'),
},
} as unknown as Parameters<typeof createEventApplier>[0];
return { calls, init, names: () => calls.map(([name]) => name) };
};
const context: ChatEvent = {
type: 'context_usage',
data: { breakdown: { maxContextTokens: 1000 } },
} as ChatEvent;
const usage: ChatEvent = { type: 'token_usage', data: { output_tokens: 5 } } as ChatEvent;
const attachment: ChatEvent = {
type: 'attachment',
data: { messageId: 'resp-1', file_id: 'file-1', filename: 'chart.png' },
} as ChatEvent;
const title: ChatEvent = {
type: 'title',
data: { event: 'title', data: { conversationId: 'thread-1', title: 'A title' } },
};
const relabel: ChatEvent = {
type: 'steer_updated',
data: { conversationId: 'thread-1', steers: [] },
};
it('gives each live frame to the handler main chat gives it, bound to the turn', () => {
const { calls, init, names } = record();
const { applyEvent } = createEventApplier(init);
const { context: ctx } = createList();
for (const event of [
created,
opened('s0', 0),
token('s0', 'Hi'),
context,
usage,
attachment,
title,
relabel,
final('Hi'),
]) {
applyEvent(event, ctx);
}
expect(names()).toEqual([
'bindResponse',
'tap',
'context',
'usage',
'attachment',
'title',
'update',
'finalize',
]);
const turn = expect.objectContaining({
userMessage: expect.objectContaining({ messageId: 'user-1' }),
});
expect(calls[2]).toEqual(['context', (context as { data: unknown }).data, turn]);
expect(calls[3]).toEqual(['usage', (usage as { data: unknown }).data, turn]);
expect(calls[4]).toEqual([
'attachment',
{ data: (attachment as { data: unknown }).data, submission: turn },
]);
expect(calls[5]).toEqual(['title', (title as { data: unknown }).data]);
expect(calls[7]).toEqual(['finalize', (final('Hi') as { data: unknown }).data, turn]);
});
it('rebuilds what a late joiner missed from the snapshot, then applies what it carries', () => {
const { calls, init, names } = record();
const { applyEvent } = createEventApplier(init);
const { context: ctx } = createList();
const collected = [{ output_tokens: 3 }];
const reading = { breakdown: { maxContextTokens: 1000 } };
const pending = [{ steerId: 'steer-2', text: 'Later' }];
applyEvent(
snapshot('Hel', {
collectedUsage: collected,
contextUsage: reading,
pendingSteers: pending,
titleEvent: { event: 'title', data: { title: 'A title' } },
replayEvents: [
{ event: 'on_token_usage', data: { output_tokens: 1 } },
{ event: 'attachment', data: { messageId: 'resp-1', file_id: 'file-1' } },
],
} as Partial<Agents.ResumeState>),
ctx,
);
expect(names()).toEqual([
'bindResponse',
'backfill',
'context',
'seedLive',
'seed',
'settle',
'title',
'usage',
'attachment',
]);
expect(calls[1][1]).toBe(collected);
expect(calls[2][1]).toBe(reading);
expect(calls[3][1]).toEqual([{ type: ContentTypes.TEXT, text: 'Hel' }]);
expect(calls[4].slice(0, 3)).toEqual(['seed', 'thread-1', pending]);
expect(calls[5]).toEqual(['settle', 'thread-1', [{ type: ContentTypes.TEXT, text: 'Hel' }]]);
});
it('clears chips with an empty snapshot list, which a steer taken while away would strand', () => {
const { calls, init } = record();
const { applyEvent } = createEventApplier(init);
const { context: ctx } = createList();
applyEvent(snapshot('Hel'), ctx);
expect(calls.find(([name]) => name === 'seed')).toEqual(['seed', 'thread-1', []]);
});
it('settles a steer chip as soon as the server says it was taken, and marks its part as drawn', () => {
const { calls, init, names } = record();
const { applyEvent } = createEventApplier(init);
const { context: ctx, read } = createList();
const part = {
type: ContentTypes.STEER,
[ContentTypes.STEER]: 'Go deeper',
steerId: 'steer-1',
};
const taken = {
type: 'steer_applied',
data: {
steerId: 'steer-1',
clientSteerId: 'client-1',
index: 1,
responseMessageId: 'resp-1',
part,
},
} as ChatEvent;
applyEvent(created, ctx);
/** Its response is not drawn yet, so the part waits; the chip does not. */
applyEvent(taken, ctx);
expect(names()).toEqual(['bindResponse', 'resolve']);
expect(calls[1]).toEqual(['resolve', 'thread-1', 'steer-1', 'client-1', undefined]);
applyEvent(opened('s0', 0), ctx);
applyEvent(token('s0', 'Hi'), ctx);
expect(names().filter((name) => name !== 'tap')).toEqual([
'bindResponse',
'resolve',
'markLive',
]);
expect(calls.find(([name]) => name === 'markLive')).toEqual(['markLive', 'steer-1']);
expect(read()[1].content?.[1]).toEqual(part);
});
it('holds a frame that needs its turn until a frame names the turn', () => {
const { calls, init, names } = record();
const { applyEvent } = createEventApplier(init);
const { context: ctx } = createList();
applyEvent(attachment, ctx);
applyEvent(usage, ctx);
expect(names()).toEqual([]);
applyEvent(created, ctx);
expect(names()).toEqual(['bindResponse', 'attachment', 'usage']);
expect(calls[1][1]).toMatchObject({ data: { file_id: 'file-1' } });
});
it('keeps no such state where no handler is given, and still applies the messages', () => {
const { applyEvent } = createEventApplier();
const { context: ctx, read } = createList();
for (const event of [created, opened('s0', 0), token('s0', 'Hi'), usage, attachment, title]) {
applyEvent(event, ctx);
}
expect(textOf(read())).toEqual(['Hi']);
});
});
describe('a Chat that applies events the way main chat does', () => {
const createState = (initial: TMessage[]): ChatState => {
let messages = initial;

View file

@ -4,11 +4,13 @@ import type {
TMessage,
ChatEvent,
ChatFrame,
TAttachment,
ChatSyncFrame,
ChatFinalFrame,
EventSubmission,
ChatCreatedFrame,
} from 'librechat-data-provider';
import type { ResumableUsage, ResumableSteers, ResumableEvents } from './resumable/host';
import type { StepEvent, StepEffect, StepContext } from './reducer';
import type { Placement, PlacementContext } from './parts/place';
import type { ChatEventContext } from './chat';
@ -46,6 +48,20 @@ export type EventApplierInit = {
schedule?: (flush: () => void) => () => void;
/** Live status a step reports beside the list: sandbox start, nested subagents, announcements. */
onEffect?: (effect: EventApplierEffect) => void;
/**
* What a stream changes beside the message list, in the hands main chat's own stream session
* puts it in: its usage handlers, its attachment collector, its title handler and its steer
* chips. Each is called for the frames, live or carried by a snapshot, that main chat calls it
* for, so a `Chat` given main chat's handlers keeps the state main chat keeps. One left out is
* simply not kept; the frame still reaches `onEvent`.
*/
usage?: ResumableUsage;
onAttachment?: ResumableEvents['attachmentHandler'];
onTitle?: ResumableEvents['titleHandler'];
steers?: Pick<
ResumableSteers,
'resolve' | 'markLiveApplied' | 'update' | 'seed' | 'settleApplied'
>;
};
export type EventApplier = {
@ -78,6 +94,25 @@ const sideTypes = new Set<ChatEvent['type']>([
const isSideFrame = (event: ChatEvent): event is SideFrame => sideTypes.has(event.type);
/** A frame that changes state kept beside the message list, and writes no message. */
type BesideFrame = Extract<
ChatEvent,
{ type: 'attachment' | 'title' | 'context_usage' | 'token_usage' | 'steer_updated' }
>;
const besideTypes = new Set<ChatEvent['type']>([
'attachment',
'title',
'context_usage',
'token_usage',
'steer_updated',
]);
const isBesideFrame = (event: ChatEvent): event is BesideFrame => besideTypes.has(event.type);
const isOutputDelta = (event: StepEvent): boolean =>
event.event === StepEvents.ON_MESSAGE_DELTA || event.event === StepEvents.ON_REASONING_DELTA;
/** How the turn's list differs from the index space the server claims; see `PlacementContext`. */
const placementContext = (submission: EventSubmission): PlacementContext => {
const userMessageId = submission.userMessage?.messageId;
@ -203,7 +238,8 @@ function createdSubmission(data: ChatCreatedFrame, messages: TMessage[]): EventS
*
* Stateful, one per `Chat`: it keeps the streaming copy of each response between events. Frames
* that describe something beside the message list (usage, titles, attachments, the composer's
* steer chips) are left to `onEvent`.
* steer chips) go to the handlers the caller supplies for them, at the points main chat's
* stream session calls its own.
*/
export function createEventApplier(init: EventApplierInit = {}): EventApplier {
const stream = createStreamState();
@ -213,6 +249,10 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
let early: StepEvent[] = [];
/** Side frames whose target has not been drawn yet, oldest first. */
let waiting: SideFrame[] = [];
/** Frames for the state beside the list that arrived before a frame named their turn. */
let aside: BesideFrame[] = [];
/** The chip ids a steer's applied frame settled for the first time, until its part is drawn. */
const settled = new WeakMap<SideFrame, string[]>();
let held: HeldFlush | undefined;
let cancel: (() => void) | undefined;
@ -241,6 +281,7 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
submission = undefined;
early = [];
waiting = [];
aside = [];
};
const settle = (effects: StepEffect[]): void => {
@ -269,6 +310,13 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
return false;
}
if (placement.changed) {
if (frame.type === 'steer_applied') {
/** Marked as its part is drawn, so the part plays its draw-in once. */
const drawn = frame.data.part?.steerId ?? frame.data.steerId;
if (settled.get(frame)?.includes(drawn) === true) {
init.steers?.markLiveApplied(drawn);
}
}
const next = [...messages];
next[placement.index] = placement.message;
context.write(next);
@ -302,17 +350,62 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
* placing ahead of them would lose them.
*/
const side = (frame: SideFrame): void => {
if (frame.type === 'steer_applied' && init.steers != null) {
/** The server's word that the steer was taken settles its chip at once; drawing the part
* can wait for its target, and a chip left pending meanwhile could be sent again. */
const { conversationId, steerId, clientSteerId, part } = frame.data;
settled.set(
frame,
init.steers.resolve(
conversationId ?? submission?.conversation?.conversationId ?? '',
steerId,
clientSteerId,
part?.quotes,
) ?? [],
);
}
land();
if (!place(frame)) {
waiting.push(frame);
}
};
/** Applies a frame for the state beside the list, or holds it until its turn is named. */
const beside = (frame: BesideFrame): void => {
if (frame.type === 'title') {
init.onTitle?.(frame.data);
} else if (frame.type === 'steer_updated') {
init.steers?.update(frame.data);
} else if (submission == null) {
aside.push(frame);
} else if (frame.type === 'attachment') {
init.onAttachment?.({ data: frame.data as TAttachment, submission });
} else if (frame.type === 'context_usage') {
init.usage?.contextHandler(frame.data, submission);
} else {
init.usage?.usageHandler(frame.data, submission);
}
};
/** A turn was just named: its usage is bound to it, and what waited on it is applied. */
const named = (next: EventSubmission): void => {
init.usage?.bindResponse(next);
const queued = aside;
aside = [];
for (const frame of queued) {
beside(frame);
}
};
const step = (event: StepEvent): void => {
if (submission == null || context == null) {
early.push(event);
return;
}
/** Streamed output feeds the live usage estimate, as main chat's dispatch feeds it. */
if (isOutputDelta(event)) {
init.usage?.tapStream(event.data as { delta?: { content?: unknown } }, submission);
}
settle(reduceStep(stream, event, stepContext(submission, context)));
};
@ -333,6 +426,10 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
side(event);
continue;
}
if (isBesideFrame(event)) {
beside(event);
continue;
}
if (event.type !== 'step') {
continue;
}
@ -350,6 +447,7 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
}
const steps = early;
const sides = waiting;
const asides = aside;
reset();
const next = createdSubmission(data, ctx.messages);
/** The list with the turn's message in it, which a regeneration's steps build on. */
@ -358,6 +456,8 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
}
submission = next;
waiting = sides;
aside = asides;
named(next);
for (const event of steps) {
step(event);
}
@ -389,6 +489,14 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
) as EventSubmission);
const next = buildResumeEventSubmission(previous, previous.userMessage, resumeState);
submission = next;
named(next);
/** The run's usage so far, which a chat that joins late did not watch arrive: its totals,
* its latest context reading, and an estimate of the output the snapshot already holds. */
init.usage?.backfillUsage(resumeState.collectedUsage ?? [], next);
if (resumeState.contextUsage != null) {
init.usage?.contextHandler(resumeState.contextUsage, next);
}
init.usage?.seedLiveFromContent(resumeState.aggregatedContent, next);
for (const runStep of resumeState.runSteps ?? []) {
step({ event: StepEvents.ON_RUN_STEP, data: runStep });
@ -426,6 +534,15 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
side({ type: 'pending_action', data: resumeState.pendingAction as Agents.PendingAction });
}
/** The chips are the server's still-queued steers, less the ones the content already holds:
* an empty list clears chips a steer applied while this chat was away would strand. */
const steerConversationId = next.conversation?.conversationId ?? conversationId;
init.steers?.seed(steerConversationId, resumeState.pendingSteers ?? []);
init.steers?.settleApplied(steerConversationId, resumeState.aggregatedContent);
if (resumeState.titleEvent != null) {
init.onTitle?.(resumeState.titleEvent);
}
replay(resumeState.replayEvents, prompted);
replay(data.pendingEvents);
};
@ -442,9 +559,14 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier {
/** The final frame is authoritative; a held copy of the stream must not land on top of it. */
drop();
mergeFinalFrame(settledFinal(event.data, submission, ctx.messages), ctx);
if (submission != null) {
init.usage?.finalizeUsage(event.data, submission);
}
reset();
} else if (isSideFrame(event)) {
side(event);
} else if (isBesideFrame(event)) {
beside(event);
} else if (event.type === 'abort' || event.type === 'error') {
/** A turn that stops here keeps every token it streamed. */
land();

View file

@ -527,6 +527,31 @@ describe('useFollowChat', () => {
expect(streams).toHaveLength(1);
});
it("hands what the stream changes beside the list to the caller's handlers, as they are now", async () => {
const { transport, streams } = createTransport(running);
const first = jest.fn();
const second = jest.fn();
const { rerender, loadMessages } = setup({ transport, beside: { onAttachment: first } });
await waitFor(() => expect(streams).toHaveLength(1));
act(() => streams[0].push(snapshot('Partial')));
await settle();
const file = { messageId: 'task-1:assistant', file_id: 'file-1', filename: 'chart.png' };
act(() => streams[0].push({ type: 'attachment', data: file } as ChatEvent));
await settle();
expect(first).toHaveBeenCalledWith(expect.objectContaining({ data: file }));
rerender({ id: 'thread-1', transport, loadMessages, beside: { onAttachment: second } });
act(() =>
streams[0].push({ type: 'attachment', data: { ...file, file_id: 'file-2' } } as ChatEvent),
);
await settle();
expect(first).toHaveBeenCalledTimes(1);
expect(second).toHaveBeenCalledWith(
expect.objectContaining({ data: expect.objectContaining({ file_id: 'file-2' }) }),
);
});
it('leaves the conversation it was following when the id changes', async () => {
const { transport, streams } = createTransport(running);
const { result, rerender, loadMessages } = setup({ transport });

View file

@ -6,6 +6,7 @@ import type {
ChatTransport,
ChatConnection,
ChatFinishResult,
EventApplierInit,
EventApplierEffect,
} from '../core';
import { Chat, MAX_RETRIES, getReconnectDelay, createEventApplier } from '../core';
@ -33,11 +34,43 @@ export type FollowChatOptions = {
enabled?: boolean;
/** Live status a step reports beside the message list. */
onEffect?: (effect: EventApplierEffect) => void;
/** Every stream event after it was applied: usage, titles, attachments. */
/**
* The handlers for what the stream changes beside the message list: usage, attachments, the
* title and steer chips. A host that passes main chat's keeps the state main chat keeps, for
* live frames and for what a late joiner missed. Read on every call.
*/
beside?: FollowBeside;
/** Every stream event after it was applied. */
onEvent?: (event: ChatEvent) => void;
onFinish?: (result: ChatFinishResult) => void;
};
export type FollowBeside = Pick<EventApplierInit, 'usage' | 'onAttachment' | 'onTitle' | 'steers'>;
/** The same handlers, looked up when a frame arrives, so the caller's latest are the ones called. */
const forwardBeside = (read: () => FollowBeside | undefined): Required<FollowBeside> => ({
usage: {
bindResponse: (submission) => read()?.usage?.bindResponse(submission),
contextHandler: (data, submission) => read()?.usage?.contextHandler(data, submission),
usageHandler: (data, submission) => read()?.usage?.usageHandler(data, submission),
tapStream: (data, submission) => read()?.usage?.tapStream(data, submission),
tapContent: (text, submission) => read()?.usage?.tapContent(text, submission),
finalizeUsage: (data, submission) => read()?.usage?.finalizeUsage(data, submission),
backfillUsage: (entries, submission) => read()?.usage?.backfillUsage(entries, submission),
seedLiveFromContent: (content, submission) =>
read()?.usage?.seedLiveFromContent(content, submission),
},
onAttachment: (params) => read()?.onAttachment?.(params),
onTitle: (frame) => read()?.onTitle?.(frame),
steers: {
resolve: (...args) => read()?.steers?.resolve(...args) ?? [],
markLiveApplied: (steerId) => read()?.steers?.markLiveApplied(steerId),
update: (event) => read()?.steers?.update(event),
seed: (...args) => read()?.steers?.seed(...args),
settleApplied: (...args) => read()?.steers?.settleApplied(...args),
},
});
export type FollowedChat = {
/** The chat, once the conversation's `id` has one. */
chat: Chat | undefined;
@ -103,12 +136,13 @@ export function useFollowChat({
live = false,
enabled = true,
onEffect,
beside,
onEvent,
onFinish,
}: FollowChatOptions): FollowedChat {
const queryClient = useQueryClient();
const host = useRef({ transport, loadMessages, live, onEffect, onEvent, onFinish });
host.current = { transport, loadMessages, live, onEffect, onEvent, onFinish };
const host = useRef({ transport, loadMessages, live, onEffect, beside, onEvent, onFinish });
host.current = { transport, loadMessages, live, onEffect, beside, onEvent, onFinish };
const [attempt, setAttempt] = useState(0);
const [entry, setEntry] = useState<Entry>();
const [load, setLoad] = useState<Load>({ id, attempt, state: 'loading' });
@ -133,6 +167,7 @@ export function useFollowChat({
const applier = createEventApplier({
schedule: perFrame,
onEffect: (effect) => host.current.onEffect?.(effect),
...forwardBeside(() => host.current.beside),
});
const chat = new Chat({
id,