LibreChat/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
Danny Avila 6e964d5897 feat: Scheduled Chats — agent-centric scheduled runs creating real conversations
Squash of the full review-hardened branch (PR 14540, supersedes 14373) onto
latest dev, preserving the exact verified tree. History prior to this commit
lived on the pre-squash branch; every invariant below survived 25 Codex review
rounds plus two external audit rounds (R26) with regression tests that fail
without their fixes.

Feature:
- Schedules CRUD + side-panel UI (cadence dialog, run cards, Run Now), roles/
  permissions (SCHEDULES:USE), interface.schedules availability, per-user limits
  and capacity slots, timezone-aware cadence with DST-conservative floors and
  misfire grace.
- Engine: single-process claim/fire loop with leases, loopback POST dispatch
  (signed schedule-fire JWT claims, per-occurrence idempotency key inside the
  route's clientRequestId charset), overlap/balance/capacity/duplicate skip
  policies with auto-disable streaks (too_many_failures, insufficient_balance),
  reconciliation from retained terminal-job evidence, erasure sweep.
- Scheduled runs create real conversations through the resumable agents chat
  path: HITL pauses surface on the card (requires_action), resumes re-apply the
  fire boundary's admission policy (revision fence, enabled, global kill switch,
  SCHEDULES:USE, availability) before continuing a billed generation.

Correctness invariants (the audit surface):
- Settlement discipline: every persistence-producing write happens-before a
  run's terminal outcome write; Stop/complete/pause race through single-winner
  terminal CAS claims (dev's TerminalJobClaim substrate) with retained,
  completedAt-less evidence for scheduled fires plus an owner-intended outcome
  stamp (scheduleOutcome) the reconciler prefers over re-derived success —
  round-tripped through the Redis hash mapper.
- Swallowed generation failures (client error content parts) classify to
  error/skipped_balance instead of success on both initial and resumed paths;
  stale stamps are refreshed evidence-first when persistence plus the Mongo
  outcome write both fail.
- Abort honesty: delivery judged by generation ownership and the CAS's actual
  from-status; republication escalates to the transport's acknowledged variant;
  the Stop route settles only genuinely paused runs.
- Account deletion: one-way barrier (deletionRequestedAt) with auth-cache
  tombstone-before-stamp, boundary rechecks across all auth strategies, quiesce
  of scheduled + interactive work with durable per-stream abort fences
  (positive-evidence acknowledgement only), owner-side finalization markers for
  post-terminal billed writes, deferred-deletion sweep (explicitly ensured
  partial index) that completes cascades autonomously. Remote OpenAI-compatible/
  Responses requests are documented as outside the quiesce and tracked in issue
  14594.
- Store compatibility: finalization markers optional on the legacy IJobStore
  contract with coherent degradation (registration fails -> synchronous title
  fallback; count reads 0).
2026-08-18 07:46:18 -04:00

241 lines
8.7 KiB
JavaScript

const mockLogger = {
debug: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
info: jest.fn(),
};
let activeTenantContext;
const mockTenantStorageRun = jest.fn(async (context, callback) => {
activeTenantContext = context;
try {
return await callback();
} finally {
activeTenantContext = undefined;
}
});
const mockSaveMessage = jest.fn();
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockIsAgentTriggerPrincipalActive = jest.fn();
const mockFilterPersistableAbortContent = jest.fn((content) => content);
const mockCheckAndIncrementPendingRequest = jest.fn();
const mockDecrementPendingRequest = jest.fn();
const mockGenerationJobManager = {
// Owner-side finalization fence: the controller registers before its terminal
// CAS whenever post-terminal title work is possible, so the facade mock must
// mirror it or the turn stalls on an undefined call.
registerUserFinalization: jest.fn(async () => undefined),
clearUserFinalization: jest.fn(async () => undefined),
countUserFinalizations: jest.fn(async () => 0),
createJob: jest.fn(),
// The disconnect-partial save re-reads the job and refuses to write unless THIS
// generation is still live — a replacement turn must not have its history
// overwritten by a predecessor's teardown. Armed live for these tests.
getJob: jest.fn(async () => ({ createdAt: 1000, status: 'running' })),
emitError: jest.fn(),
completeJob: jest.fn(),
beginProviderExecution: jest.fn(),
markProviderExecutionDrained: jest.fn(),
getResumeState: jest.fn(),
updateMetadata: jest.fn(),
claimGeneration: jest.fn(),
releaseGeneration: jest.fn(),
hasJob: jest.fn(),
steering: {
closeAndDrain: jest.fn(),
park: jest.fn(),
},
};
jest.mock('@librechat/data-schemas', () => ({
logger: mockLogger,
tenantStorage: {
run: (...args) => mockTenantStorageRun(...args),
},
}));
jest.mock('@librechat/api', () => ({
// Scheduled-fire classification helpers the controller consults on every turn
// (and on its disconnect/teardown paths): an ordinary chat is never a fire.
isScheduleFireRequest: jest.fn(() => false),
exemptFromIpLimiter: jest.fn(() => false),
exemptFromUserLimiter: jest.fn(() => false),
exemptFromConcurrencyLimiter: jest.fn(() => false),
sendEvent: jest.fn(),
toPendingSteer: jest.fn((item) => item),
isSteerPreemptSupported: jest.fn(() => true),
buildRecoveredSteerPayload: jest.fn(() => null),
deleteAgentCheckpoint: jest.fn(),
getViolationInfo: jest.fn(() => ({
type: 'concurrent',
limit: 2,
pendingRequests: 3,
score: 1,
})),
buildMessageFiles: jest.fn(() => []),
resolveTitleTiming: jest.fn(() => 'immediate'),
resolveConversationAnchor: jest.requireActual('@librechat/api').resolveConversationAnchor,
GenerationJobManager: mockGenerationJobManager,
getReferencedQuotes: jest.fn(() => null),
cleanupMCPRequestContext: jest.fn(),
createMCPRequestContext: jest.fn(() => ({
connections: new Map(),
pending: new Map(),
cleanupStarted: false,
})),
getMCPRequestContext: jest.fn(() => ({
connections: new Map(),
pending: new Map(),
cleanupStarted: false,
})),
filterPersistableAbortContent: (...args) => mockFilterPersistableAbortContent(...args),
cleanupMCPRequestContextForReq: jest.fn(),
decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
sanitizeMessageForTransmit: jest.fn((message) => message),
checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
getAgentStartupTelemetry: jest.fn(() => undefined),
acceptAgentStartupTelemetry: jest.fn(),
isUnpersistedPreliminaryParent: jest.fn(async () => false),
}));
jest.mock('~/server/cleanup', () => ({
disposeClient: jest.fn(),
clientRegistry: null,
requestDataMap: {
set: jest.fn(),
},
}));
jest.mock('~/server/middleware', () => ({
handleAbortError: jest.fn(() => Promise.resolve()),
}));
// The controller imports the schedules service, whose module graph builds the app
// config service and its Keyv cache namespaces at import time from `@librechat/api`
// factories this spec's partial mock does not provide. Nothing here exercises
// scheduled runs, so stub the service the way the abort/resume suites do.
jest.mock('~/server/services/Schedules', () => ({
recordScheduleOutcome: jest.fn(async () => true),
isScheduleLive: jest.fn(async () => true),
clearScheduledJob: jest.fn(async () => undefined),
awaitStopAbortPersistence: jest.fn(async () => true),
}));
jest.mock('~/cache', () => ({
logViolation: jest.fn(),
}));
jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
getMessages: (...args) => mockGetMessages(...args),
getConvo: (...args) => mockGetConvo(...args),
isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
}));
const AgentController = require('../request');
describe('ResumableAgentController tenant context', () => {
beforeEach(() => {
jest.clearAllMocks();
activeTenantContext = undefined;
mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true });
mockDecrementPendingRequest.mockResolvedValue(undefined);
mockGetConvo.mockResolvedValue({ createdAt: '2026-07-31T00:00:00.000Z' });
mockGetMessages.mockResolvedValue([]);
mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);
mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);
mockGenerationJobManager.hasJob.mockResolvedValue(true);
mockGenerationJobManager.steering.closeAndDrain.mockResolvedValue([]);
mockGenerationJobManager.steering.park.mockResolvedValue(undefined);
});
/**
* Drives the controller far enough to register the `allSubscribersLeft` handler,
* fires it, and returns the tenant context that was active during `saveMessage`.
*/
const firePartialDisconnect = async (user) => {
let allSubscribersLeftHandler;
mockGenerationJobManager.createJob.mockResolvedValue({
createdAt: 1000,
metadata: {
providerExecutionId: 'provider-segment-1',
providerDrained: true,
},
readyPromise: Promise.resolve(),
abortController: new AbortController(),
emitter: {
on: jest.fn((event, handler) => {
if (event === 'allSubscribersLeft') {
allSubscribersLeftHandler = handler;
}
}),
},
});
mockGenerationJobManager.getResumeState.mockResolvedValue({
conversationId: 'conversation-123',
responseMessageId: 'response-message',
userMessage: {
messageId: 'user-message',
},
});
let tenantSeenBySave;
mockSaveMessage.mockImplementation(async () => {
tenantSeenBySave = activeTenantContext;
return {};
});
const initializeClient = jest.fn().mockRejectedValue(new Error('stop after setup'));
const req = {
user,
body: {
text: 'Continue the analysis',
messageId: 'user-message',
parentMessageId: 'parent-message',
conversationId: 'conversation-123',
endpointOption: {
endpoint: 'agents',
modelOptions: { model: 'gpt-4.1' },
},
},
config: {},
};
const res = {
headersSent: true,
json: jest.fn(),
status: jest.fn(() => res),
};
await AgentController(req, res, jest.fn(), initializeClient, null);
expect(allSubscribersLeftHandler).toEqual(expect.any(Function));
await allSubscribersLeftHandler([{ type: 'text', text: 'Partial response' }]);
return tenantSeenBySave;
};
it('restores the authenticated tenant before saving a partial response on disconnect', async () => {
const tenantSeenBySave = await firePartialDisconnect({ id: 'user-123', tenantId: 'tenant-a' });
expect(mockTenantStorageRun).toHaveBeenCalledWith(
{ tenantId: 'tenant-a', userId: 'user-123' },
expect.any(Function),
);
expect(tenantSeenBySave).toEqual({ tenantId: 'tenant-a', userId: 'user-123' });
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
});
it('saves the partial response without tenant context when the user has no tenant', async () => {
const tenantSeenBySave = await firePartialDisconnect({ id: 'user-123' });
expect(mockTenantStorageRun).not.toHaveBeenCalled();
expect(tenantSeenBySave).toBeUndefined();
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
});
});