diff --git a/packages/chat/src/core/__tests__/apply.spec.ts b/packages/chat/src/core/__tests__/apply.spec.ts index 20518943ce..c0725bd8ba 100644 --- a/packages/chat/src/core/__tests__/apply.spec.ts +++ b/packages/chat/src/core/__tests__/apply.spec.ts @@ -919,6 +919,215 @@ describe('side frames', () => { }); }); +describe('state beside the message list', () => { + /** Main chat's handlers for that state, recording what they are called with and in what order. */ + const record = () => { + const calls: Array<[string, ...unknown[]]> = []; + const log = + (name: string, result?: unknown) => + (...args: unknown[]): unknown => { + calls.push([name, ...args]); + return result; + }; + const init = { + usage: { + bindResponse: log('bindResponse'), + contextHandler: log('context'), + usageHandler: log('usage'), + tapStream: log('tap'), + tapContent: log('tapContent'), + finalizeUsage: log('finalize'), + backfillUsage: log('backfill'), + seedLiveFromContent: log('seedLive'), + }, + onAttachment: log('attachment'), + onTitle: log('title'), + steers: { + resolve: log('resolve', ['steer-1']), + markLiveApplied: log('markLive'), + update: log('update'), + seed: log('seed'), + settleApplied: log('settle'), + }, + } as unknown as Parameters[0]; + return { calls, init, names: () => calls.map(([name]) => name) }; + }; + + const context: ChatEvent = { + type: 'context_usage', + data: { breakdown: { maxContextTokens: 1000 } }, + } as ChatEvent; + const usage: ChatEvent = { type: 'token_usage', data: { output_tokens: 5 } } as ChatEvent; + const attachment: ChatEvent = { + type: 'attachment', + data: { messageId: 'resp-1', file_id: 'file-1', filename: 'chart.png' }, + } as ChatEvent; + const title: ChatEvent = { + type: 'title', + data: { event: 'title', data: { conversationId: 'thread-1', title: 'A title' } }, + }; + const relabel: ChatEvent = { + type: 'steer_updated', + data: { conversationId: 'thread-1', steers: [] }, + }; + + it('gives each live frame to the handler main chat gives it, bound to the turn', () => { + const { calls, init, names } = record(); + const { applyEvent } = createEventApplier(init); + const { context: ctx } = createList(); + + for (const event of [ + created, + opened('s0', 0), + token('s0', 'Hi'), + context, + usage, + attachment, + title, + relabel, + final('Hi'), + ]) { + applyEvent(event, ctx); + } + + expect(names()).toEqual([ + 'bindResponse', + 'tap', + 'context', + 'usage', + 'attachment', + 'title', + 'update', + 'finalize', + ]); + const turn = expect.objectContaining({ + userMessage: expect.objectContaining({ messageId: 'user-1' }), + }); + expect(calls[2]).toEqual(['context', (context as { data: unknown }).data, turn]); + expect(calls[3]).toEqual(['usage', (usage as { data: unknown }).data, turn]); + expect(calls[4]).toEqual([ + 'attachment', + { data: (attachment as { data: unknown }).data, submission: turn }, + ]); + expect(calls[5]).toEqual(['title', (title as { data: unknown }).data]); + expect(calls[7]).toEqual(['finalize', (final('Hi') as { data: unknown }).data, turn]); + }); + + it('rebuilds what a late joiner missed from the snapshot, then applies what it carries', () => { + const { calls, init, names } = record(); + const { applyEvent } = createEventApplier(init); + const { context: ctx } = createList(); + const collected = [{ output_tokens: 3 }]; + const reading = { breakdown: { maxContextTokens: 1000 } }; + const pending = [{ steerId: 'steer-2', text: 'Later' }]; + + applyEvent( + snapshot('Hel', { + collectedUsage: collected, + contextUsage: reading, + pendingSteers: pending, + titleEvent: { event: 'title', data: { title: 'A title' } }, + replayEvents: [ + { event: 'on_token_usage', data: { output_tokens: 1 } }, + { event: 'attachment', data: { messageId: 'resp-1', file_id: 'file-1' } }, + ], + } as Partial), + ctx, + ); + + expect(names()).toEqual([ + 'bindResponse', + 'backfill', + 'context', + 'seedLive', + 'seed', + 'settle', + 'title', + 'usage', + 'attachment', + ]); + expect(calls[1][1]).toBe(collected); + expect(calls[2][1]).toBe(reading); + expect(calls[3][1]).toEqual([{ type: ContentTypes.TEXT, text: 'Hel' }]); + expect(calls[4].slice(0, 3)).toEqual(['seed', 'thread-1', pending]); + expect(calls[5]).toEqual(['settle', 'thread-1', [{ type: ContentTypes.TEXT, text: 'Hel' }]]); + }); + + it('clears chips with an empty snapshot list, which a steer taken while away would strand', () => { + const { calls, init } = record(); + const { applyEvent } = createEventApplier(init); + const { context: ctx } = createList(); + + applyEvent(snapshot('Hel'), ctx); + + expect(calls.find(([name]) => name === 'seed')).toEqual(['seed', 'thread-1', []]); + }); + + it('settles a steer chip as soon as the server says it was taken, and marks its part as drawn', () => { + const { calls, init, names } = record(); + const { applyEvent } = createEventApplier(init); + const { context: ctx, read } = createList(); + const part = { + type: ContentTypes.STEER, + [ContentTypes.STEER]: 'Go deeper', + steerId: 'steer-1', + }; + const taken = { + type: 'steer_applied', + data: { + steerId: 'steer-1', + clientSteerId: 'client-1', + index: 1, + responseMessageId: 'resp-1', + part, + }, + } as ChatEvent; + + applyEvent(created, ctx); + /** Its response is not drawn yet, so the part waits; the chip does not. */ + applyEvent(taken, ctx); + expect(names()).toEqual(['bindResponse', 'resolve']); + expect(calls[1]).toEqual(['resolve', 'thread-1', 'steer-1', 'client-1', undefined]); + + applyEvent(opened('s0', 0), ctx); + applyEvent(token('s0', 'Hi'), ctx); + + expect(names().filter((name) => name !== 'tap')).toEqual([ + 'bindResponse', + 'resolve', + 'markLive', + ]); + expect(calls.find(([name]) => name === 'markLive')).toEqual(['markLive', 'steer-1']); + expect(read()[1].content?.[1]).toEqual(part); + }); + + it('holds a frame that needs its turn until a frame names the turn', () => { + const { calls, init, names } = record(); + const { applyEvent } = createEventApplier(init); + const { context: ctx } = createList(); + + applyEvent(attachment, ctx); + applyEvent(usage, ctx); + expect(names()).toEqual([]); + + applyEvent(created, ctx); + + expect(names()).toEqual(['bindResponse', 'attachment', 'usage']); + expect(calls[1][1]).toMatchObject({ data: { file_id: 'file-1' } }); + }); + + it('keeps no such state where no handler is given, and still applies the messages', () => { + const { applyEvent } = createEventApplier(); + const { context: ctx, read } = createList(); + + for (const event of [created, opened('s0', 0), token('s0', 'Hi'), usage, attachment, title]) { + applyEvent(event, ctx); + } + + expect(textOf(read())).toEqual(['Hi']); + }); +}); + describe('a Chat that applies events the way main chat does', () => { const createState = (initial: TMessage[]): ChatState => { let messages = initial; diff --git a/packages/chat/src/core/apply.ts b/packages/chat/src/core/apply.ts index c20ef5615d..e2c48f74e5 100644 --- a/packages/chat/src/core/apply.ts +++ b/packages/chat/src/core/apply.ts @@ -4,11 +4,13 @@ import type { TMessage, ChatEvent, ChatFrame, + TAttachment, ChatSyncFrame, ChatFinalFrame, EventSubmission, ChatCreatedFrame, } from 'librechat-data-provider'; +import type { ResumableUsage, ResumableSteers, ResumableEvents } from './resumable/host'; import type { StepEvent, StepEffect, StepContext } from './reducer'; import type { Placement, PlacementContext } from './parts/place'; import type { ChatEventContext } from './chat'; @@ -46,6 +48,20 @@ export type EventApplierInit = { schedule?: (flush: () => void) => () => void; /** Live status a step reports beside the list: sandbox start, nested subagents, announcements. */ onEffect?: (effect: EventApplierEffect) => void; + /** + * What a stream changes beside the message list, in the hands main chat's own stream session + * puts it in: its usage handlers, its attachment collector, its title handler and its steer + * chips. Each is called for the frames, live or carried by a snapshot, that main chat calls it + * for, so a `Chat` given main chat's handlers keeps the state main chat keeps. One left out is + * simply not kept; the frame still reaches `onEvent`. + */ + usage?: ResumableUsage; + onAttachment?: ResumableEvents['attachmentHandler']; + onTitle?: ResumableEvents['titleHandler']; + steers?: Pick< + ResumableSteers, + 'resolve' | 'markLiveApplied' | 'update' | 'seed' | 'settleApplied' + >; }; export type EventApplier = { @@ -78,6 +94,25 @@ const sideTypes = new Set([ const isSideFrame = (event: ChatEvent): event is SideFrame => sideTypes.has(event.type); +/** A frame that changes state kept beside the message list, and writes no message. */ +type BesideFrame = Extract< + ChatEvent, + { type: 'attachment' | 'title' | 'context_usage' | 'token_usage' | 'steer_updated' } +>; + +const besideTypes = new Set([ + 'attachment', + 'title', + 'context_usage', + 'token_usage', + 'steer_updated', +]); + +const isBesideFrame = (event: ChatEvent): event is BesideFrame => besideTypes.has(event.type); + +const isOutputDelta = (event: StepEvent): boolean => + event.event === StepEvents.ON_MESSAGE_DELTA || event.event === StepEvents.ON_REASONING_DELTA; + /** How the turn's list differs from the index space the server claims; see `PlacementContext`. */ const placementContext = (submission: EventSubmission): PlacementContext => { const userMessageId = submission.userMessage?.messageId; @@ -203,7 +238,8 @@ function createdSubmission(data: ChatCreatedFrame, messages: TMessage[]): EventS * * Stateful, one per `Chat`: it keeps the streaming copy of each response between events. Frames * that describe something beside the message list (usage, titles, attachments, the composer's - * steer chips) are left to `onEvent`. + * steer chips) go to the handlers the caller supplies for them, at the points main chat's + * stream session calls its own. */ export function createEventApplier(init: EventApplierInit = {}): EventApplier { const stream = createStreamState(); @@ -213,6 +249,10 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { let early: StepEvent[] = []; /** Side frames whose target has not been drawn yet, oldest first. */ let waiting: SideFrame[] = []; + /** Frames for the state beside the list that arrived before a frame named their turn. */ + let aside: BesideFrame[] = []; + /** The chip ids a steer's applied frame settled for the first time, until its part is drawn. */ + const settled = new WeakMap(); let held: HeldFlush | undefined; let cancel: (() => void) | undefined; @@ -241,6 +281,7 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { submission = undefined; early = []; waiting = []; + aside = []; }; const settle = (effects: StepEffect[]): void => { @@ -269,6 +310,13 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { return false; } if (placement.changed) { + if (frame.type === 'steer_applied') { + /** Marked as its part is drawn, so the part plays its draw-in once. */ + const drawn = frame.data.part?.steerId ?? frame.data.steerId; + if (settled.get(frame)?.includes(drawn) === true) { + init.steers?.markLiveApplied(drawn); + } + } const next = [...messages]; next[placement.index] = placement.message; context.write(next); @@ -302,17 +350,62 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { * placing ahead of them would lose them. */ const side = (frame: SideFrame): void => { + if (frame.type === 'steer_applied' && init.steers != null) { + /** The server's word that the steer was taken settles its chip at once; drawing the part + * can wait for its target, and a chip left pending meanwhile could be sent again. */ + const { conversationId, steerId, clientSteerId, part } = frame.data; + settled.set( + frame, + init.steers.resolve( + conversationId ?? submission?.conversation?.conversationId ?? '', + steerId, + clientSteerId, + part?.quotes, + ) ?? [], + ); + } land(); if (!place(frame)) { waiting.push(frame); } }; + /** Applies a frame for the state beside the list, or holds it until its turn is named. */ + const beside = (frame: BesideFrame): void => { + if (frame.type === 'title') { + init.onTitle?.(frame.data); + } else if (frame.type === 'steer_updated') { + init.steers?.update(frame.data); + } else if (submission == null) { + aside.push(frame); + } else if (frame.type === 'attachment') { + init.onAttachment?.({ data: frame.data as TAttachment, submission }); + } else if (frame.type === 'context_usage') { + init.usage?.contextHandler(frame.data, submission); + } else { + init.usage?.usageHandler(frame.data, submission); + } + }; + + /** A turn was just named: its usage is bound to it, and what waited on it is applied. */ + const named = (next: EventSubmission): void => { + init.usage?.bindResponse(next); + const queued = aside; + aside = []; + for (const frame of queued) { + beside(frame); + } + }; + const step = (event: StepEvent): void => { if (submission == null || context == null) { early.push(event); return; } + /** Streamed output feeds the live usage estimate, as main chat's dispatch feeds it. */ + if (isOutputDelta(event)) { + init.usage?.tapStream(event.data as { delta?: { content?: unknown } }, submission); + } settle(reduceStep(stream, event, stepContext(submission, context))); }; @@ -333,6 +426,10 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { side(event); continue; } + if (isBesideFrame(event)) { + beside(event); + continue; + } if (event.type !== 'step') { continue; } @@ -350,6 +447,7 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { } const steps = early; const sides = waiting; + const asides = aside; reset(); const next = createdSubmission(data, ctx.messages); /** The list with the turn's message in it, which a regeneration's steps build on. */ @@ -358,6 +456,8 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { } submission = next; waiting = sides; + aside = asides; + named(next); for (const event of steps) { step(event); } @@ -389,6 +489,14 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { ) as EventSubmission); const next = buildResumeEventSubmission(previous, previous.userMessage, resumeState); submission = next; + named(next); + /** The run's usage so far, which a chat that joins late did not watch arrive: its totals, + * its latest context reading, and an estimate of the output the snapshot already holds. */ + init.usage?.backfillUsage(resumeState.collectedUsage ?? [], next); + if (resumeState.contextUsage != null) { + init.usage?.contextHandler(resumeState.contextUsage, next); + } + init.usage?.seedLiveFromContent(resumeState.aggregatedContent, next); for (const runStep of resumeState.runSteps ?? []) { step({ event: StepEvents.ON_RUN_STEP, data: runStep }); @@ -426,6 +534,15 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { side({ type: 'pending_action', data: resumeState.pendingAction as Agents.PendingAction }); } + /** The chips are the server's still-queued steers, less the ones the content already holds: + * an empty list clears chips a steer applied while this chat was away would strand. */ + const steerConversationId = next.conversation?.conversationId ?? conversationId; + init.steers?.seed(steerConversationId, resumeState.pendingSteers ?? []); + init.steers?.settleApplied(steerConversationId, resumeState.aggregatedContent); + if (resumeState.titleEvent != null) { + init.onTitle?.(resumeState.titleEvent); + } + replay(resumeState.replayEvents, prompted); replay(data.pendingEvents); }; @@ -442,9 +559,14 @@ export function createEventApplier(init: EventApplierInit = {}): EventApplier { /** The final frame is authoritative; a held copy of the stream must not land on top of it. */ drop(); mergeFinalFrame(settledFinal(event.data, submission, ctx.messages), ctx); + if (submission != null) { + init.usage?.finalizeUsage(event.data, submission); + } reset(); } else if (isSideFrame(event)) { side(event); + } else if (isBesideFrame(event)) { + beside(event); } else if (event.type === 'abort' || event.type === 'error') { /** A turn that stops here keeps every token it streamed. */ land(); diff --git a/packages/chat/src/react/__tests__/follow.spec.tsx b/packages/chat/src/react/__tests__/follow.spec.tsx index d040bdc0be..6ecf47caf6 100644 --- a/packages/chat/src/react/__tests__/follow.spec.tsx +++ b/packages/chat/src/react/__tests__/follow.spec.tsx @@ -527,6 +527,31 @@ describe('useFollowChat', () => { expect(streams).toHaveLength(1); }); + it("hands what the stream changes beside the list to the caller's handlers, as they are now", async () => { + const { transport, streams } = createTransport(running); + const first = jest.fn(); + const second = jest.fn(); + const { rerender, loadMessages } = setup({ transport, beside: { onAttachment: first } }); + await waitFor(() => expect(streams).toHaveLength(1)); + act(() => streams[0].push(snapshot('Partial'))); + await settle(); + + const file = { messageId: 'task-1:assistant', file_id: 'file-1', filename: 'chart.png' }; + act(() => streams[0].push({ type: 'attachment', data: file } as ChatEvent)); + await settle(); + expect(first).toHaveBeenCalledWith(expect.objectContaining({ data: file })); + + rerender({ id: 'thread-1', transport, loadMessages, beside: { onAttachment: second } }); + act(() => + streams[0].push({ type: 'attachment', data: { ...file, file_id: 'file-2' } } as ChatEvent), + ); + await settle(); + expect(first).toHaveBeenCalledTimes(1); + expect(second).toHaveBeenCalledWith( + expect.objectContaining({ data: expect.objectContaining({ file_id: 'file-2' }) }), + ); + }); + it('leaves the conversation it was following when the id changes', async () => { const { transport, streams } = createTransport(running); const { result, rerender, loadMessages } = setup({ transport }); diff --git a/packages/chat/src/react/follow.ts b/packages/chat/src/react/follow.ts index 88f14fe4ac..f4b304ca86 100644 --- a/packages/chat/src/react/follow.ts +++ b/packages/chat/src/react/follow.ts @@ -6,6 +6,7 @@ import type { ChatTransport, ChatConnection, ChatFinishResult, + EventApplierInit, EventApplierEffect, } from '../core'; import { Chat, MAX_RETRIES, getReconnectDelay, createEventApplier } from '../core'; @@ -33,11 +34,43 @@ export type FollowChatOptions = { enabled?: boolean; /** Live status a step reports beside the message list. */ onEffect?: (effect: EventApplierEffect) => void; - /** Every stream event after it was applied: usage, titles, attachments. */ + /** + * The handlers for what the stream changes beside the message list: usage, attachments, the + * title and steer chips. A host that passes main chat's keeps the state main chat keeps, for + * live frames and for what a late joiner missed. Read on every call. + */ + beside?: FollowBeside; + /** Every stream event after it was applied. */ onEvent?: (event: ChatEvent) => void; onFinish?: (result: ChatFinishResult) => void; }; +export type FollowBeside = Pick; + +/** The same handlers, looked up when a frame arrives, so the caller's latest are the ones called. */ +const forwardBeside = (read: () => FollowBeside | undefined): Required => ({ + usage: { + bindResponse: (submission) => read()?.usage?.bindResponse(submission), + contextHandler: (data, submission) => read()?.usage?.contextHandler(data, submission), + usageHandler: (data, submission) => read()?.usage?.usageHandler(data, submission), + tapStream: (data, submission) => read()?.usage?.tapStream(data, submission), + tapContent: (text, submission) => read()?.usage?.tapContent(text, submission), + finalizeUsage: (data, submission) => read()?.usage?.finalizeUsage(data, submission), + backfillUsage: (entries, submission) => read()?.usage?.backfillUsage(entries, submission), + seedLiveFromContent: (content, submission) => + read()?.usage?.seedLiveFromContent(content, submission), + }, + onAttachment: (params) => read()?.onAttachment?.(params), + onTitle: (frame) => read()?.onTitle?.(frame), + steers: { + resolve: (...args) => read()?.steers?.resolve(...args) ?? [], + markLiveApplied: (steerId) => read()?.steers?.markLiveApplied(steerId), + update: (event) => read()?.steers?.update(event), + seed: (...args) => read()?.steers?.seed(...args), + settleApplied: (...args) => read()?.steers?.settleApplied(...args), + }, +}); + export type FollowedChat = { /** The chat, once the conversation's `id` has one. */ chat: Chat | undefined; @@ -103,12 +136,13 @@ export function useFollowChat({ live = false, enabled = true, onEffect, + beside, onEvent, onFinish, }: FollowChatOptions): FollowedChat { const queryClient = useQueryClient(); - const host = useRef({ transport, loadMessages, live, onEffect, onEvent, onFinish }); - host.current = { transport, loadMessages, live, onEffect, onEvent, onFinish }; + const host = useRef({ transport, loadMessages, live, onEffect, beside, onEvent, onFinish }); + host.current = { transport, loadMessages, live, onEffect, beside, onEvent, onFinish }; const [attempt, setAttempt] = useState(0); const [entry, setEntry] = useState(); const [load, setLoad] = useState({ id, attempt, state: 'loading' }); @@ -133,6 +167,7 @@ export function useFollowChat({ const applier = createEventApplier({ schedule: perFrame, onEffect: (effect) => host.current.onEffect?.(effect), + ...forwardBeside(() => host.current.beside), }); const chat = new Chat({ id,