LibreChat/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
Danny Avila ff522e0771
🧭 refactor: Build One Execution Context Where a Run Enters (#18255)
* 🧭 refactor: Build One Execution Context Where a Run Enters

Each consumer of a scheduled run's identity (the host token resolver, the
scheduled MCP authority and the resource-bearer host) read the trigger body
and the restored job for itself, and the remote agent API carried a separate
`unattended` flag through tool loading.

The entry point now builds one frozen ExecutionContext: the chat controller
from the verified schedule trigger, the resume host from the saved job, and
the remote controllers from the run envelope. The initializer and tool
loading receive it as an argument. The schedule consumers project their
identity from it, and `unattended` is derived from it at the tool-loading
boundary. No client or model input sets any field.

* 🧭 refactor: Decide Whether a Run Is Unattended in Its Execution Context
2026-11-18 07:39:34 -05:00

605 lines
22 KiB
JavaScript

const mockLogger = {
debug: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
info: jest.fn(),
};
let activeTenantContext;
const mockTenantStorageRun = jest.fn(async (context, callback) => {
activeTenantContext = context;
try {
return await callback();
} finally {
activeTenantContext = undefined;
}
});
const mockSaveMessage = jest.fn();
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockStampConvoLastResponse = jest.fn();
const mockIsAgentTriggerPrincipalActive = jest.fn();
const mockFilterPersistableAbortContent = jest.fn((content) => content);
const mockCheckAndIncrementPendingRequest = jest.fn();
const mockDecrementPendingRequest = jest.fn();
const mockGenerationJobManager = {
createJob: jest.fn(),
emitError: jest.fn(),
completeJob: jest.fn(),
beginProviderExecution: jest.fn(),
markProviderExecutionDrained: jest.fn(),
getResumeState: jest.fn(),
getJobStore: jest.fn(),
updateMetadata: jest.fn(),
claimGeneration: jest.fn(),
releaseGeneration: jest.fn(),
hasJob: jest.fn(),
steering: {
closeAndDrain: jest.fn(),
park: jest.fn(),
},
};
jest.mock('@librechat/data-schemas', () => ({
logger: mockLogger,
redactMessage: jest.requireActual('@librechat/data-schemas').redactMessage,
tenantStorage: {
run: (...args) => mockTenantStorageRun(...args),
},
}));
jest.mock('@librechat/api', () => ({
getTrustedNewConversationId: (...args) =>
jest.requireActual('@librechat/api').getTrustedNewConversationId(...args),
buildGenerationStartedResponse: (...args) =>
jest.requireActual('@librechat/api').buildGenerationStartedResponse(...args),
savePrivateTextMessage: (save, _req, ...args) => save(...args),
savePrivateTextErrorTurn: (...args) =>
jest.requireActual('@librechat/api').savePrivateTextErrorTurn(...args),
getUserFacingGenerationError: (...args) =>
jest.requireActual('@librechat/api').getUserFacingGenerationError(...args),
stampPreliminaryPrivateTextMessage: (_req, message) => message,
getAgentErrorMetadata: (...args) =>
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
applyForcedTemporaryRequest: jest.fn(),
resolveResumableRetention: jest.requireActual('@librechat/api').resolveResumableRetention,
planDisconnectSnapshot: (...args) =>
jest.requireActual('@librechat/api').planDisconnectSnapshot(...args),
createDisconnectSnapshotQueue: (...args) =>
jest.requireActual('@librechat/api').createDisconnectSnapshotQueue(...args),
announceDisconnectSnapshot: (...args) =>
jest.requireActual('@librechat/api').announceDisconnectSnapshot(...args),
settleExistingRowsBeforeErrorTurn: (...args) =>
jest.requireActual('@librechat/api').settleExistingRowsBeforeErrorTurn(...args),
sendEvent: jest.fn(),
persistedReasoningOverrideFields:
jest.requireActual('@librechat/api').persistedReasoningOverrideFields,
isScheduleFireRequest: jest.fn(() => false),
createScheduleExecutionContext: jest.fn(() => undefined),
exemptFromConcurrencyLimiter: jest.fn(() => false),
toPendingSteer: jest.fn((item) => item),
isSteerPreemptSupported: jest.fn(() => true),
isSteerTerminalContinuationSupported: jest.fn(() => false),
buildRecoveredSteerPayload: jest.fn(() => null),
deleteAgentCheckpoint: jest.fn(),
getViolationInfo: jest.fn(() => ({
type: 'concurrent',
limit: 2,
pendingRequests: 3,
score: 1,
})),
buildMessageFiles: jest.fn(() => []),
resolveTitleTiming: jest.fn(() => 'immediate'),
createConvoPersistenceSignal: jest.requireActual('@librechat/api').createConvoPersistenceSignal,
recoverTurnMessageReference: jest.requireActual('@librechat/api').recoverTurnMessageReference,
resolveConversationAnchor: jest.requireActual('@librechat/api').resolveConversationAnchor,
resolveRunCodeWorkspaces: jest.requireActual('@librechat/api').resolveRunCodeWorkspaces,
shouldPersistCodeWorkspaceInitializationError:
jest.requireActual('@librechat/api').shouldPersistCodeWorkspaceInitializationError,
resolvePersistableCodeEnvironmentDecision: (...args) =>
jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),
getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,
logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,
startAgentProjectContextResolution:
jest.requireActual('@librechat/api').startAgentProjectContextResolution,
assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions,
getChatProjectTurnFailure: jest.requireActual('@librechat/api').getChatProjectTurnFailure,
GenerationJobManager: mockGenerationJobManager,
getReferencedQuotes: jest.fn(() => null),
cleanupMCPRequestContext: jest.fn(),
createMCPRequestContext: jest.fn(() => ({
connections: new Map(),
pending: new Map(),
cleanupStarted: false,
})),
getMCPRequestContext: jest.fn(() => ({
connections: new Map(),
pending: new Map(),
cleanupStarted: false,
})),
filterPersistableAbortContent: (...args) => mockFilterPersistableAbortContent(...args),
cleanupMCPRequestContextForReq: jest.fn(),
decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
sanitizeMessageForTransmit: jest.fn((message) => message),
checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
getAgentStartupTelemetry: jest.fn(() => undefined),
acceptAgentStartupTelemetry: jest.fn(),
isUnpersistedPreliminaryParent: jest.fn(async () => false),
createMCPRuntimeRequestBody: ({ messageId, conversationId, parentMessageId }) => ({
messageId,
conversationId,
parentMessageId,
}),
parseAgentEventActorDetachedCompletion: jest.fn(() => undefined),
}));
jest.mock('~/server/cleanup', () => ({
disposeClient: jest.fn(),
}));
jest.mock('~/server/middleware', () => ({
handleAbortError: jest.fn(() => Promise.resolve()),
}));
jest.mock('~/cache', () => ({
logViolation: jest.fn(),
}));
jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
getMessages: (...args) => mockGetMessages(...args),
getConvo: (...args) => mockGetConvo(...args),
stampConvoLastResponse: (...args) => mockStampConvoLastResponse(...args),
isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
}));
const AgentController = require('../request');
describe('ResumableAgentController tenant context', () => {
/** The anchor is persisted and no response or error row exists yet. */
const anchorOnly = () =>
mockGetMessages.mockImplementation(async (filter) =>
filter.messageId === 'user-message' ? [{ _id: 'existing-anchor' }] : [],
);
beforeEach(() => {
jest.clearAllMocks();
activeTenantContext = undefined;
mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true });
mockDecrementPendingRequest.mockResolvedValue(undefined);
mockGetConvo.mockResolvedValue({ createdAt: '2026-07-31T00:00:00.000Z' });
mockGetMessages.mockResolvedValue([]);
mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);
mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);
mockGenerationJobManager.hasJob.mockResolvedValue(true);
mockGenerationJobManager.steering.closeAndDrain.mockResolvedValue([]);
mockGenerationJobManager.steering.park.mockResolvedValue(undefined);
});
/**
* Drives the controller far enough to register the `allSubscribersLeft` handler,
* fires it, and returns the tenant context that was active during `saveMessage`.
*/
const partialContextMeta = {
calibrationRatio: 1.2,
encoding: 'claude',
fading: { v: 1, budgetTokens: 50_000, masked: true },
};
let lastAllSubscribersLeftHandler;
const firePartialDisconnect = async (
user,
jobRecord = { createdAt: 1000, contextMeta: partialContextMeta },
{ body = {}, aggregatedContent = [{ type: 'text', text: 'Partial response' }] } = {},
) => {
mockGetConvo.mockResolvedValue({
conversationId: 'conversation-123',
user: user.id,
tenantId: user.tenantId,
createdAt: '2026-07-31T00:00:00.000Z',
});
let allSubscribersLeftHandler;
mockGenerationJobManager.getJobStore.mockReturnValue({
getJob: jest.fn().mockResolvedValue(jobRecord),
});
mockGenerationJobManager.createJob.mockResolvedValue({
createdAt: 1000,
metadata: {
providerExecutionId: 'provider-segment-1',
providerDrained: true,
},
readyPromise: Promise.resolve(),
abortController: new AbortController(),
emitter: {
on: jest.fn((event, handler) => {
if (event === 'allSubscribersLeft') {
allSubscribersLeftHandler = handler;
lastAllSubscribersLeftHandler = handler;
}
}),
},
});
mockGenerationJobManager.getResumeState.mockResolvedValue({
conversationId: 'conversation-123',
responseMessageId: 'response-message',
userMessage: {
messageId: 'user-message',
},
});
let tenantSeenBySave;
mockSaveMessage.mockImplementation(async () => {
tenantSeenBySave = activeTenantContext;
return {};
});
const initializeClient = jest.fn().mockRejectedValue(new Error('stop after setup'));
const req = {
user,
body: {
text: 'Continue the analysis',
messageId: 'user-message',
parentMessageId: 'parent-message',
conversationId: 'conversation-123',
endpointOption: {
endpoint: 'agents',
modelOptions: { model: 'gpt-4.1' },
},
...body,
},
config: {},
};
const res = {
headersSent: true,
json: jest.fn(),
status: jest.fn(() => res),
};
await AgentController(req, res, jest.fn(), initializeClient, null);
expect(allSubscribersLeftHandler).toEqual(expect.any(Function));
await allSubscribersLeftHandler(aggregatedContent);
return tenantSeenBySave;
};
it('carries the context meta the run published onto the partial response saved on disconnect', async () => {
await firePartialDisconnect({ id: 'user-123', tenantId: 'tenant-a' });
expect(mockStampConvoLastResponse).not.toHaveBeenCalled();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.any(Object),
expect.objectContaining({
messageId: 'response-message',
unfinished: true,
contextMeta: partialContextMeta,
}),
expect.any(Object),
);
});
it('leaves context meta off the partial response when the job record belongs to another epoch', async () => {
await firePartialDisconnect(
{ id: 'user-123', tenantId: 'tenant-a' },
{ createdAt: 2000, contextMeta: partialContextMeta },
);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).not.toHaveProperty('contextMeta');
});
it('restores the authenticated tenant before saving a partial response on disconnect', async () => {
const tenantSeenBySave = await firePartialDisconnect({ id: 'user-123', tenantId: 'tenant-a' });
expect(mockTenantStorageRun).toHaveBeenCalledWith(
{ tenantId: 'tenant-a', userId: 'user-123' },
expect.any(Function),
);
expect(tenantSeenBySave).toEqual({ tenantId: 'tenant-a', userId: 'user-123' });
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
});
it('saves the partial response without tenant context when the user has no tenant', async () => {
const tenantSeenBySave = await firePartialDisconnect({ id: 'user-123' });
expect(mockTenantStorageRun).not.toHaveBeenCalled();
expect(tenantSeenBySave).toBeUndefined();
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
});
/** A cancelled compaction's partial row is built here, not by sendCompletion,
* so it carries no marker unless the disconnect path stamps one: without it
* the row reads as an answer to the message it hangs off and keeps that
* message's rerun controls. */
it('stamps a partial response saved on disconnect with the compaction identity', async () => {
await firePartialDisconnect(
{ id: 'user-123' },
{ createdAt: 1000 },
{
body: { compact: true },
aggregatedContent: [
{
type: 'summary',
content: [{ type: 'text', text: 'Half a summary' }],
summarizing: true,
},
],
},
);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({
messageId: 'response-message',
unfinished: true,
error: false,
content: [{ type: 'summary', summarizing: true, initiatedBy: 'user' }],
});
});
/** The disconnect save runs while the generation is still live and the
* completing run overwrites the row, so it must not report a failure that
* has not happened: no typed failure is invented for a compaction whose
* snapshot carries no summary or error part. */
it('saves a non-outcome compaction partial on disconnect without a synthesized failure', async () => {
await firePartialDisconnect(
{ id: 'user-123' },
{ createdAt: 1000 },
{
body: { compact: true },
aggregatedContent: [{ type: 'think', think: 'Picking what to summarize' }],
},
);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({
unfinished: true,
error: false,
content: [{ type: 'think', think: 'Picking what to summarize' }],
});
expect(savedMessage.content).toHaveLength(1);
});
/** The settling path (completion, error, abort) owns the compaction's final
* row: a disconnect snapshot landing after it would reopen the settled
* turn as an unfinished response. */
it('skips the compaction partial save when the job record has settled', async () => {
await firePartialDisconnect(
{ id: 'user-123' },
{ createdAt: 1000, status: 'error' },
{
body: { compact: true },
aggregatedContent: [{ type: 'text', text: 'Partial response' }],
},
);
expect(mockSaveMessage).not.toHaveBeenCalled();
});
/** An ordinary turn's record cannot prove its response row landed (a
* terminal transition without a write leaves the same record), so its
* snapshot stays the fallback row, in its live shape. */
it.each([
['a settled record', {}],
['a pending terminal write', { terminalPersistencePending: true }],
['a reconciliation frame', { finalEvent: JSON.stringify({ final: true, reconcile: true }) }],
])('keeps an ordinary partial save as the fallback behind %s', async (_label, record) => {
await firePartialDisconnect(
{ id: 'user-123' },
{ createdAt: 1000, status: 'complete', ...record },
{ aggregatedContent: [{ type: 'text', text: 'Partial response' }] },
);
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({ unfinished: true, error: false });
});
/** The terminal write failed and settled for a reconciliation frame: the
* snapshot is the turn's only row, so it persists with the terminal
* outcome (the typed failure for content with no summary) and envelope. */
it('promotes a reconciled compaction snapshot to the terminal row', async () => {
anchorOnly();
await firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{
body: { compact: true },
aggregatedContent: [{ type: 'think', think: 'Picking what to summarize' }],
},
);
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({ unfinished: false, error: true });
expect(savedMessage.content).toEqual([
{ type: 'think', think: 'Picking what to summarize' },
expect.objectContaining({ type: 'error', initiatedBy: 'user' }),
]);
expect(mockStampConvoLastResponse).toHaveBeenCalledWith(
'user-123',
'conversation-123',
'response-message',
);
});
/** A reconciliation frame does not prove the row is missing: an error turn
* saved before it is a settled row the promotion must not overwrite. */
it('withholds a reconciled snapshot when a settled response row exists', async () => {
mockGetMessages.mockImplementation(async (filter) =>
filter.messageId === 'response-message'
? [{ _id: 'error-row', unfinished: false }]
: [{ _id: 'existing-anchor' }],
);
await firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{
body: { compact: true },
aggregatedContent: [{ type: 'text', text: 'Partial response' }],
},
);
expect(mockSaveMessage).not.toHaveBeenCalled();
});
it('promotes a compaction snapshot on a later disconnect after a live save', async () => {
mockGetMessages.mockImplementation(async (filter) =>
filter.messageId === 'user-message' ? [{ _id: 'existing-anchor' }] : [],
);
await firePartialDisconnect(
{ id: 'user-123' },
{ createdAt: 1000, status: 'running' },
{ body: { compact: true } },
);
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
expect(mockSaveMessage.mock.calls[0][1]).toMatchObject({ unfinished: true });
mockGenerationJobManager.getJobStore.mockReturnValue({
getJob: jest.fn().mockResolvedValue({
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
}),
});
mockGetMessages.mockImplementation(async (filter) =>
filter.messageId === 'user-message'
? [{ _id: 'existing-anchor' }]
: [{ _id: 'live-row', unfinished: true }],
);
await lastAllSubscribersLeftHandler([{ type: 'text', text: 'Partial response' }]);
expect(mockSaveMessage).toHaveBeenCalledTimes(2);
expect(mockSaveMessage.mock.calls[1][1]).toMatchObject({ unfinished: false, error: true });
});
it('logs a failed row read while planning the snapshot instead of rejecting the listener', async () => {
const readError = new Error('read failed');
mockGetMessages.mockImplementation(async (filter) => {
if (filter.messageId === 'user-message') {
return [{ _id: 'existing-anchor' }];
}
throw readError;
});
await expect(
firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{
body: { compact: true },
aggregatedContent: [{ type: 'text', text: 'Partial response' }],
},
),
).resolves.not.toThrow();
expect(mockSaveMessage).not.toHaveBeenCalled();
expect(mockLogger.error).toHaveBeenCalledWith(
'[ResumableAgentController] Error planning partial response:',
readError,
);
});
it('withholds a reconciled snapshot when the error turn saved its anchor-derived row', async () => {
mockGetMessages.mockImplementation(
async (filter) =>
({
'user-message': [{ _id: 'existing-anchor' }],
'user-message_': [{ _id: 'error-row', unfinished: false }],
})[filter.messageId] ?? [],
);
await firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{
body: { compact: true },
aggregatedContent: [{ type: 'text', text: 'Partial response' }],
},
);
expect(mockSaveMessage).not.toHaveBeenCalled();
});
/** Without a persisted anchor the promotion is withheld: the fallback must
* not recreate the orphan the absent-anchor abort suppressed. */
it('withholds a reconciled snapshot whose anchor was never persisted', async () => {
await firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{
body: { compact: true },
aggregatedContent: [{ type: 'text', text: 'Partial response' }],
},
);
expect(mockSaveMessage).not.toHaveBeenCalled();
});
/** An empty snapshot on a reconciled compaction still records the turn:
* the terminal synthesis appends the typed failure from nothing. */
it('synthesizes the terminal outcome for an empty reconciled compaction snapshot', async () => {
anchorOnly();
await firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'error',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{ body: { compact: true }, aggregatedContent: [] },
);
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({ unfinished: false, error: true });
expect(savedMessage.content).toEqual([
expect.objectContaining({ type: 'error', initiatedBy: 'user' }),
]);
});
/** A completed claim's promoted snapshot settles as a finished row, not an
* error. */
it('settles a completed reconciled snapshot as a finished row', async () => {
anchorOnly();
await firePartialDisconnect(
{ id: 'user-123' },
{
createdAt: 1000,
status: 'complete',
finalEvent: JSON.stringify({ final: true, reconcile: true }),
},
{
body: { compact: true },
aggregatedContent: [
{
type: 'summary',
content: [{ type: 'text', text: 'A finished checkpoint.' }],
boundary: { messageId: 'm', contentIndex: 0 },
},
],
},
);
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({ unfinished: false, error: false });
});
});