LibreChat/api/server/controllers/agents/request.js
Danny Avila 2b10412c12 fix: deletion-fence marker protocol — generation-scoped, atomic, fail-closed, all terminal paths
One consolidated pass over the finalization-marker mechanism, per the fourth
external review round. The invariant it establishes: NO persistence-owning
terminal CAS runs without a durable, generation-qualified marker covering the
window it opens, and every consumer treats a pending terminal as unsettled.

- Generation-scoped markers. Entries were keyed (userId, streamId), so a
  Stop-superseded generation finishing late could clear the marker its
  replacement registered on the same conversation. Marker fields are now
  qualified by the generation's createdAt; clears must present the same
  identity, and an unqualified legacy clear cannot drop a qualified entry.
- Atomic Redis registration. HSET-then-EXPIRE loses the fresh marker when the
  user's existing hash expires between the two commands (or the process dies
  there); registration is now a single Lua script carrying both.
- Fail closed everywhere. Registration failure (after one retry) now REFUSES
  the terminal CAS instead of proceeding uncovered: the completion claim throws
  into the error path, the error path skips completeJob and leaves the job
  ACTIVE — deletion-visible by itself, recovered by the stale-running reaper —
  and abortJob returns a new retryable `fence_unavailable` failure the Stop
  route answers with 503 and the deletion quiesce treats as an unacknowledged
  stop (fence kept). The previous round's log-and-proceed is gone.
- Every terminal path enrolled. abortJob now owns its window (register before
  the abort CAS, clear in its finally — the Stop route's checkpoint prune and
  partial save run inside beforePublish, between CAS and publication); the
  interactive and background generation-error paths register before their
  completeJob; the resume controller's error finalization registers before its
  completeJob. A lost or thrown claim releases the marker after pending saves
  flush instead of holding the user's deletion behind the TTL.
- Every pending terminal unsettled. settleAbortFence defers on
  terminalPersistencePending for ALL statuses — including `aborted`, whose
  route-side persistence the previous guard missed.

Also: a direct Redis regression for scheduled stale-pause retention (the P2
test gap), and the PR description no longer claims to carry every commit.
2026-08-18 07:46:18 -04:00

2630 lines
112 KiB
JavaScript

const { logger, tenantStorage } = require('@librechat/data-schemas');
const { v5: uuidv5 } = require('uuid');
const {
Constants,
EModelEndpoint,
ErrorTypes,
ViolationTypes,
isEphemeralAgentId,
} = require('librechat-data-provider');
const {
toPendingSteer,
getViolationInfo,
buildMessageFiles,
getReferencedQuotes,
resolveTitleTiming,
GenerationJobManager,
filterPersistableAbortContent,
decrementPendingRequest,
sanitizeMessageForTransmit,
checkAndIncrementPendingRequest,
exemptFromConcurrencyLimiter,
isUnpersistedPreliminaryParent,
isScheduleFireRequest,
resolveConversationAnchor,
getAgentStartupTelemetry,
acceptAgentStartupTelemetry,
isSteerPreemptSupported,
buildRecoveredSteerPayload,
deleteAgentCheckpoint,
getAttachmentTitleText,
} = require('@librechat/api');
const { disposeClient } = require('~/server/cleanup');
const {
getMCPRequestContext,
cleanupMCPRequestContextForReq,
} = require('~/server/services/MCPRequestContext');
const { logViolation } = require('~/cache');
const { saveMessage, getMessages, getConvo, isAgentTriggerPrincipalActive } = require('~/models');
const {
GENERATION_PROTOCOL_HEADER,
GENERATION_PROTOCOL_V2,
negotiateNewGenerationProtocol,
negotiateExistingGenerationProtocol,
} = require('./protocol');
function sendGenerationJson(res, status, body, generationProtocolVersion) {
if (typeof res.set === 'function') {
res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
} else if (typeof res.setHeader === 'function') {
res.setHeader(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
}
return res.status(status).json({ ...body, generationProtocolVersion });
}
function getInitializationFailure(error) {
if (error?.code === ErrorTypes.RESOURCE_RECOVERY_REQUIRED) {
return {
status: 409,
code: ErrorTypes.RESOURCE_RECOVERY_REQUIRED,
error: error.message || 'Attached resources must be restored before retrying.',
};
}
const candidateStatus = error?.status ?? error?.statusCode;
if (!Number.isInteger(candidateStatus) || candidateStatus < 400 || candidateStatus >= 600) {
return null;
}
return {
status: candidateStatus,
...(typeof error?.code === 'string' ? { code: error.code } : {}),
error: error?.message || 'Failed to start generation',
};
}
function resolveConversationCreatedAt({ userId, conversationId, isNewConvo, conversation }) {
return resolveConversationAnchor({
isNewConversation: isNewConvo,
loadConversation: () =>
conversation !== undefined ? Promise.resolve(conversation) : getConvo(userId, conversationId),
onLoadError: (error) => {
logger.warn('[AgentController] Failed to resolve conversation timestamp anchor', {
conversationId,
error: error.message,
});
},
});
}
async function attachConversationCreatedAt(req, conversationId, conversationAnchorPromise) {
req.body.conversationId = conversationId;
const resolved = await conversationAnchorPromise;
req.conversationCreatedAt = resolved.createdAt;
if (resolved.conversation !== undefined) {
req.resolvedConversation = resolved.conversation ?? null;
}
}
function getPreliminaryResponseMessageId({ messageId, responseMessageId }) {
if (typeof responseMessageId === 'string' && responseMessageId.length > 0) {
return responseMessageId;
}
if (typeof messageId !== 'string' || messageId.length === 0) {
return null;
}
return `${messageId.replace(/_+$/, '')}_`;
}
function getPreliminaryUserMessage(
{ messageId, parentMessageId, text, quotes, files, manualSkills, alwaysAppliedSkills },
conversationId,
) {
if (typeof messageId !== 'string' || messageId.length === 0) {
return null;
}
/**
* Seed normalized quotes here too: if the user aborts before `sendMessage`
* reaches `onStart` (during init/tool loading), `abortMiddleware` falls back
* to this preliminary metadata, which must carry the excerpts so the stopped
* turn keeps its `MessageQuotes`.
*/
const referencedQuotes = getReferencedQuotes(quotes);
return {
messageId,
parentMessageId,
conversationId,
text,
...(referencedQuotes != null && { quotes: referencedQuotes }),
// Persist the turn's uploaded files on this AWAITED preliminary write so they land on
// job.metadata.userMessage BEFORE the run can reach its first interrupt. onStart's
// later writes are fire-and-forget, so a fast approval could otherwise read the job
// and resume an approved code/read-file tool without the paused turn's uploads.
...(Array.isArray(files) && files.length > 0 && { files }),
// Carry skill selections so a HITL-resumed turn's reconstructed `requestMessage`
// keeps its skill pills — the client's final handler replaces the user bubble from
// this object, and they'd otherwise vanish until a full reload refetches the row.
...(Array.isArray(manualSkills) && manualSkills.length > 0 && { manualSkills }),
...(Array.isArray(alwaysAppliedSkills) &&
alwaysAppliedSkills.length > 0 && { alwaysAppliedSkills }),
};
}
function getRequestModelSpec(req, endpointOption) {
const spec = endpointOption?.spec ?? req.body?.spec;
if (typeof spec !== 'string' || spec.length === 0) {
return;
}
const list = req.config?.modelSpecs?.list;
if (!Array.isArray(list)) {
return;
}
return list.find((modelSpec) => modelSpec?.name === spec);
}
function getModelSpecIconURL(modelSpec) {
return modelSpec?.iconURL ?? modelSpec?.preset?.iconURL ?? modelSpec?.preset?.endpoint ?? '';
}
function getEndpointIconURL(req, endpointOption) {
const iconURL =
endpointOption?.iconURL ?? getModelSpecIconURL(getRequestModelSpec(req, endpointOption));
return iconURL || undefined;
}
function getEndpointResponseModel(endpointOption) {
return endpointOption?.modelOptions?.model || endpointOption?.model_parameters?.model;
}
function getAgentResponseModel(req, endpointOption) {
const agentId = endpointOption?.agent_id || req.body?.agent_id;
if (typeof agentId === 'string' && agentId.length > 0 && !isEphemeralAgentId(agentId)) {
return agentId;
}
return getEndpointResponseModel(endpointOption);
}
async function finishResumableRequest(req, userId) {
try {
await cleanupMCPRequestContextForReq(req);
} finally {
// Mirrors the increment: only an AUTOMATIC fire skipped the counter, so only it
// must skip the decrement. Uses the decision captured at request start, not a
// re-check of the expiring token.
if (!exemptFromConcurrencyLimiter(req)) {
await decrementPendingRequest(userId);
}
}
}
const JOB_RECORD_WAIT_ATTEMPTS = 5;
const JOB_RECORD_WAIT_DELAY_MS = 60;
// A winner writes its job record within a few ms of claiming; if a losing duplicate still
// sees no job within this window of the claim, the winner is still starting (retry rather
// than hand back a stream that would 404). Past it, a missing job means the original
// already completed and was cleaned up (attach and let the client refetch).
const IDEMPOTENCY_STARTUP_GRACE_MS = 5000;
const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
/** New-chat retries do not carry a conversation id, so derive the stream id
* from their stable per-submission id. This keeps both the dedupe key and the
* Redis hash slot identical across a lost-response retry. */
const NEW_CONVERSATION_IDEMPOTENCY_NAMESPACE = 'd7f2518c-94b8-4fe8-97ad-2d4bdb2c9f43';
function isValidGenerationClaim(value, streamId, conversationId, requireStarted = false) {
return (
value != null &&
typeof value === 'object' &&
value.streamId === streamId &&
value.conversationId === conversationId &&
Number.isSafeInteger(value.claimedAt) &&
value.claimedAt >= 0 &&
typeof value.claimToken === 'string' &&
value.claimToken.length > 0 &&
value.claimToken.length <= 128 &&
(value.generationProtocolVersion == null ||
value.generationProtocolVersion === 1 ||
value.generationProtocolVersion === GENERATION_PROTOCOL_V2) &&
(value.startedAt == null || (Number.isSafeInteger(value.startedAt) && value.startedAt >= 0)) &&
(!requireStarted || value.startedAt != null)
);
}
/** Pre-bridge servers wrote the legacy global key without a claim token and,
* for a new conversation, chose a random stream before claiming it. Accept
* only that tightly bounded legacy shape: existing conversations must still
* match the requested stream exactly; new-chat claims may point to the old
* random stream only when streamId === conversationId. Ownership is verified
* against the live job before attachment. */
function isValidLegacyGenerationClaim(value, streamId, isNewConvo) {
return (
value != null &&
typeof value === 'object' &&
typeof value.streamId === 'string' &&
value.streamId.length > 0 &&
value.streamId.length <= 512 &&
value.conversationId === value.streamId &&
(isNewConvo || value.streamId === streamId) &&
Number.isSafeInteger(value.claimedAt) &&
value.claimedAt >= 0 &&
value.claimToken == null &&
value.startedAt == null &&
(value.generationProtocolVersion == null || value.generationProtocolVersion === 1)
);
}
/** Store corruption must not turn a user-scoped idempotency claim into a
* pointer to another user's/tenant's live stream. Missing tenant metadata is
* kept as the explicit legacy case, but missing ownership never authorizes. */
function liveJobBelongsToRequester(job, user) {
return (
job?.metadata?.userId === user.id &&
(job.metadata?.tenantId == null || job.metadata.tenantId === user.tenantId)
);
}
/**
* Poll briefly for a job record to appear. A deduped retry that loses the idempotency
* claim must not be handed the winner's stream until its job exists, or the client's
* subscribe 404s terminally. The winner writes the record a few ms after claiming.
*/
async function waitForJobRecord(streamId) {
for (let attempt = 0; attempt < JOB_RECORD_WAIT_ATTEMPTS; attempt++) {
const job = await GenerationJobManager.getJob(streamId);
if (job) {
return job;
}
await new Promise((resolve) => setTimeout(resolve, JOB_RECORD_WAIT_DELAY_MS));
}
return GenerationJobManager.getJob(streamId);
}
/** The claimed generation already reached durable/terminal history, but its
* conversation stream id now belongs to no job or to a newer submission. A
* success shape with that streamId would attach the stale submission to the
* replacement, so tell the client to refetch without opening SSE. */
function sendSettledGeneration(
res,
streamId,
conversationId,
startupTelemetry,
generationProtocolVersion,
) {
startupTelemetry?.end('deduplicated');
if (generationProtocolVersion < GENERATION_PROTOCOL_V2) {
return sendGenerationJson(
res,
200,
{ streamId, conversationId, status: 'resumed' },
generationProtocolVersion,
);
}
return sendGenerationJson(
res,
200,
{ conversationId, status: 'settled' },
generationProtocolVersion,
);
}
function rejectPreliminaryParentMessageId(res, generationProtocolVersion) {
return sendGenerationJson(
res,
409,
{
error:
'Cannot submit a follow-up while the selected parent response is still being saved. Please wait and try again.',
},
generationProtocolVersion,
);
}
/**
* Resumable Agent Controller - Generation runs independently of HTTP connection.
* Returns streamId immediately, client subscribes separately via SSE.
*/
const ResumableAgentController = async (req, res, next, initializeClient, addTitle) => {
const startupTelemetry = getAgentStartupTelemetry(req);
let generationProtocolVersion = negotiateNewGenerationProtocol(req, GenerationJobManager);
const {
text,
isRegenerate,
endpointOption,
conversationId: reqConversationId,
isContinued = false,
editedContent = null,
parentMessageId = null,
overrideParentMessageId = null,
responseMessageId: editedResponseMessageId = null,
scheduleId: bodyScheduleId = null,
scheduledFor: bodyScheduledFor = null,
scheduleConfigRevision: bodyScheduleConfigRevision = null,
} = req.body;
// Only honor schedule bookkeeping fields on a server-minted scheduled fire
// (scope-claim verified). A normal chat could otherwise pass an arbitrary
// scheduleId and corrupt another schedule's lastRun/counters via the hook.
// Prefer the decision the chat route captured right after auth: the short-lived
// fire token can expire during the slower chat middleware, so re-verifying here
// would wrongly demote a valid fire to an ordinary chat and orphan its run.
const isScheduledFire =
typeof req._isScheduledFire === 'boolean' ? req._isScheduledFire : isScheduleFireRequest(req);
// Capture the decision on req: the short-lived fire token expires mid-run, so
// cleanup must not re-verify it (an expired token would read as non-scheduled
// and wrongly decrement the concurrent-request counter it never incremented).
req._isScheduledFire = isScheduledFire;
const scheduleId = isScheduledFire ? bodyScheduleId : null;
const scheduledFor = isScheduledFire ? bodyScheduledFor : null;
const scheduleConfigRevision = isScheduledFire
? (bodyScheduleConfigRevision ?? undefined)
: undefined;
const userId = req.user.id;
const tenantId = req.user.tenantId;
const rawClientRequestId = req.body?.clientRequestId;
if (
rawClientRequestId != null &&
(typeof rawClientRequestId !== 'string' || !CLIENT_REQUEST_ID_PATTERN.test(rawClientRequestId))
) {
startupTelemetry?.end('rejected');
return sendGenerationJson(
res,
400,
{
code: 'INVALID_CLIENT_REQUEST_ID',
error: 'clientRequestId must be a 1-128 character identifier.',
},
generationProtocolVersion,
);
}
const clientRequestId = rawClientRequestId;
const rawExpectedPredecessorCreatedAt = req.body?.expectedPredecessorCreatedAt;
if (
rawExpectedPredecessorCreatedAt != null &&
(!Number.isSafeInteger(rawExpectedPredecessorCreatedAt) || rawExpectedPredecessorCreatedAt < 0)
) {
startupTelemetry?.end('rejected');
return sendGenerationJson(
res,
400,
{
code: 'INVALID_GENERATION_PREDECESSOR',
error: 'expectedPredecessorCreatedAt must be a non-negative safe integer.',
},
generationProtocolVersion,
);
}
const expectedPredecessorCreatedAt = rawExpectedPredecessorCreatedAt;
const legacyRecoveredSteerId =
clientRequestId?.startsWith('steer-recovery:') === true
? clientRequestId.slice('steer-recovery:'.length)
: undefined;
const explicitRecoveredSteerId = req.body?.recoverySteerId;
const invalidExplicitRecoveryId =
explicitRecoveredSteerId != null &&
(typeof explicitRecoveredSteerId !== 'string' ||
!CLIENT_REQUEST_ID_PATTERN.test(explicitRecoveredSteerId));
const mismatchedRecoveryIds =
explicitRecoveredSteerId != null &&
legacyRecoveredSteerId != null &&
explicitRecoveredSteerId !== legacyRecoveredSteerId;
if (invalidExplicitRecoveryId || mismatchedRecoveryIds) {
startupTelemetry?.end('rejected');
return sendGenerationJson(
res,
400,
{
code: 'INVALID_RECOVERY_REQUEST',
error: 'recoverySteerId must identify exactly one parked steer source.',
},
generationProtocolVersion,
);
}
const recoveredSteerId = explicitRecoveredSteerId ?? legacyRecoveredSteerId;
const isRecoveredSteerRequest = recoveredSteerId != null;
const recoveryUserMessageId = req.body?.overrideUserMessageId;
const recoveredSteerPayload = isRecoveredSteerRequest
? buildRecoveredSteerPayload(text, req.body?.files)
: undefined;
/** A recovered steer is handed off as a new ordinary user turn. Edit,
* regenerate, continue, and arbitrary override-id shapes can reuse an
* existing user row (or deliberately skip its save); consuming the parked
* source from one of those shapes would therefore erase the only durable
* copy of the recovered words without proving that a new user row contains
* them. The source steer id itself is the one permitted user-row override:
* retries intentionally upsert that stable recovery row while each
* generation attempt uses a fresh clientRequestId. */
if (
isRecoveredSteerRequest &&
(!clientRequestId ||
!recoveredSteerId ||
!recoveredSteerPayload ||
!!isRegenerate ||
!!isContinued ||
editedContent != null ||
overrideParentMessageId != null ||
editedResponseMessageId != null ||
(recoveryUserMessageId != null && recoveryUserMessageId !== recoveredSteerId) ||
!!req.body?.overrideConvoId)
) {
startupTelemetry?.end('rejected');
return sendGenerationJson(
res,
400,
{
code: 'INVALID_RECOVERY_REQUEST',
error: 'A recovered steer must be submitted as a new user turn.',
},
generationProtocolVersion,
);
}
if (isRecoveredSteerRequest && recoveryUserMessageId === recoveredSteerId) {
/** BaseClient treats a bare override id as an already-persisted row and
* skips its save. Recovery instead needs an idempotent upsert: preserve
* the source-derived row id while explicitly selecting save index zero. */
req.body.overrideUserMessageId = `${recoveredSteerId}${Constants.COMMON_DIVIDER}0`;
}
// Treat "new" as a placeholder that needs a real UUID (the frontend may send "new").
const isNewConvo = !reqConversationId || reqConversationId === 'new';
// A verified scheduled fire may supply its own new-conversation id so its run row can
// record the id BEFORE the request runs, making the occurrence's job findable by
// reconciliation even if the post-accept detail write fails. Gated on the fire flag so
// an ordinary chat can't pin an arbitrary conversation id.
const scheduledNewConversationId =
isScheduledFire && typeof req.body?.newConversationId === 'string'
? req.body.newConversationId
: null;
let conversationId = reqConversationId;
if (isNewConvo) {
// The server-minted scheduled id wins: it is already recorded on the run row, so
// reconciliation can only find the job under that id.
conversationId =
scheduledNewConversationId ??
(typeof clientRequestId === 'string' && clientRequestId.length > 0
? uuidv5(`${userId}:${clientRequestId}`, NEW_CONVERSATION_IDEMPOTENCY_NAMESPACE)
: crypto.randomUUID());
}
const conversationAnchorPromise = resolveConversationCreatedAt({
userId,
conversationId,
isNewConvo,
conversation: Object.prototype.hasOwnProperty.call(req, 'resolvedConversation')
? req.resolvedConversation
: undefined,
});
if (
await isUnpersistedPreliminaryParent({
userId,
conversationId: reqConversationId,
parentMessageId,
getMessages,
})
) {
startupTelemetry?.end('rejected');
return rejectPreliminaryParentMessageId(res, generationProtocolVersion);
}
/** When to generate the conversation title. `immediate` (default) fires title
* generation in parallel with the response, from the user's first message;
* `final` defers it until the full response completes (legacy behavior).
* Resolved from the agent's actual endpoint once the client is initialized. */
let titleTiming = 'immediate';
// streamId === conversationId always; both are resolved above so the conversation
// anchor can be prefetched.
const streamId = conversationId;
req.body.conversationId = conversationId;
// Idempotency: a lost/reset start-generation response makes the client re-POST the
// identical payload, which would otherwise start a second fully-billed generation.
// Claim the submission's clientRequestId before creating the job so a retry attaches
// to the original stream instead of spawning a duplicate. Runs before the concurrency
// check so a deduped retry is never counted against the limiter. Once a
// stable id is present, an ambiguous store outcome must fail closed.
let ownedIdempotencyClaim = null;
if (clientRequestId) {
let claim = null;
try {
claim = await GenerationJobManager.claimGeneration(
userId,
clientRequestId,
streamId,
conversationId,
generationProtocolVersion,
);
} catch (err) {
logger.error(
'[ResumableAgentController] Idempotency claim outcome is unknown; asking the client to retry',
err,
);
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership could not be confirmed. Please retry shortly.',
},
generationProtocolVersion,
);
}
if (claim?.existing != null) {
generationProtocolVersion = Math.min(
generationProtocolVersion,
claim.existing.generationProtocolVersion === GENERATION_PROTOCOL_V2
? GENERATION_PROTOCOL_V2
: 1,
);
}
const isLegacyTokenlessClaim =
claim?.source === 'legacy' && claim?.existing != null && claim.existing.claimToken == null;
const validClaim = isLegacyTokenlessClaim
? isValidLegacyGenerationClaim(claim.existing, streamId, isNewConvo)
: isValidGenerationClaim(claim?.existing, streamId, conversationId);
if (claim?.existing != null && !validClaim) {
logger.error('[ResumableAgentController] Invalid or miscorrelated idempotency claim');
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership could not be confirmed. Please retry shortly.',
},
generationProtocolVersion,
);
}
if (claim?.claimed && claim.existing?.claimToken) {
ownedIdempotencyClaim = claim.existing;
try {
const existingLiveGeneration = await GenerationJobManager.resumeClaimedGeneration(
userId,
clientRequestId,
streamId,
ownedIdempotencyClaim,
);
if (
existingLiveGeneration &&
isValidGenerationClaim(existingLiveGeneration, streamId, conversationId, true)
) {
// A fresh lease may have been negotiated under a different rollout
// cap than the still-live job it was atomically rebound to. The
// job's immutable protocol wins; echoing the fresh request's marker
// would make the client use v2-only recovery against a v1 run (or
// unnecessarily downgrade a v2 run).
generationProtocolVersion = Math.min(
generationProtocolVersion,
existingLiveGeneration.generationProtocolVersion === GENERATION_PROTOCOL_V2
? GENERATION_PROTOCOL_V2
: 1,
);
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
200,
{
streamId: existingLiveGeneration.streamId,
conversationId: existingLiveGeneration.conversationId,
generationCreatedAt: existingLiveGeneration.startedAt,
status: 'resumed',
},
generationProtocolVersion,
);
} else if (existingLiveGeneration) {
throw new Error('Live generation idempotency adoption returned invalid ownership');
}
} catch (err) {
logger.error('[ResumableAgentController] Live generation idempotency adoption failed', err);
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership changed. Please retry shortly.',
},
generationProtocolVersion,
);
}
} else if (claim?.existing) {
// A duplicate is confirmed. Attach to the original stream — and never fall through to
// a second generation, even if the job lookup hiccups.
const existingStreamId = claim.existing.streamId;
let liveJob;
try {
// Wait briefly for the winner to write the job record (it does so a few ms after
// claiming) so a still-live stream isn't handed back before its job exists.
liveJob = await waitForJobRecord(existingStreamId);
} catch (err) {
// Store hiccup while checking the job: ask the client to retry rather than starting
// a second generation for a request we know is a duplicate.
logger.error(
'[ResumableAgentController] Job lookup failed for an existing claim; asking the client to retry',
err,
);
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation is still starting. Please retry shortly.',
},
generationProtocolVersion,
);
}
const claimAgeMs = Date.now() - (claim.existing.claimedAt ?? 0);
if (!liveJob && claim.existing.startedAt != null) {
// createJob marked this claim in the same transaction that installed
// the job. A now-missing record therefore represents an already-owned
// generation (usually fast completion + cleanup), never an abandoned
// pre-create lease that may be taken over and billed again. There is no
// attachable stream; the settled response refetches persisted history.
return sendSettledGeneration(
res,
existingStreamId,
claim.existing.conversationId,
startupTelemetry,
generationProtocolVersion,
);
}
if (!liveJob && isLegacyTokenlessClaim && claimAgeMs >= IDEMPOTENCY_STARTUP_GRACE_MS) {
/** A legacy owner cannot be fenced (its value has no token), so it is
* never safe to take over. Return its original stream on the legacy
* attach/refetch path: this covers fast completion without starting a
* second billed generation, while an abandoned pre-create claim ages
* out under the old server's bounded TTL. */
return sendSettledGeneration(
res,
existingStreamId,
claim.existing.conversationId,
startupTelemetry,
generationProtocolVersion,
);
}
if (!liveJob && claimAgeMs < IDEMPOTENCY_STARTUP_GRACE_MS) {
// The winner claimed but has not written the job yet (still between claim and
// createJob). Handing back the stream now would 404 and tear down the client while
// the winner goes on to generate and bill with no UI attached — ask the client to
// retry via the readiness path instead.
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation is still starting. Please retry shortly.',
},
generationProtocolVersion,
);
}
if (liveJob) {
generationProtocolVersion = negotiateExistingGenerationProtocol(req, liveJob);
if (!liveJobBelongsToRequester(liveJob, req.user)) {
logger.error(
'[ResumableAgentController] Existing idempotency claim resolved to a foreign generation',
);
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership could not be confirmed. Please retry shortly.',
},
generationProtocolVersion,
);
}
const liveClientRequestId = liveJob.metadata?.idempotencyClientRequestId;
const startedAt = claim.existing.startedAt;
if (liveJob.metadata?.terminalPersistencePending === true) {
/** The terminal owner has claimed the outcome but has not yet
* finished the required persistence hook. Do not let a duplicate
* start refetch history until that single-winner publication is
* finalized (or stale-pending recovery publishes failure). */
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation is finalizing. Please retry shortly.',
},
generationProtocolVersion,
);
}
const terminalWithoutPayload =
['complete', 'error', 'aborted'].includes(liveJob.status) &&
!liveJob.finalEvent &&
!liveJob.error;
if (terminalWithoutPayload) {
/** A terminal CAS can precede its required DB save and durable FINAL
* by a narrow window. Returning an attachable/settled success here
* lets the retry refetch before persistence is complete. Keep the
* duplicate on the readiness path until the owner publishes its
* terminal payload (or cleanup makes the job disappear). */
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation is finalizing. Please retry shortly.',
},
generationProtocolVersion,
);
}
const replacedGeneration =
(startedAt != null && liveJob.createdAt !== startedAt) ||
(liveClientRequestId != null && liveClientRequestId !== clientRequestId);
if (replacedGeneration) {
// streamId === conversationId, so a later turn reuses the same route.
// Never pair this stale POST's optimistic submission with that newer
// job's SSE snapshot. If the replacement is still active, distinguish
// it from an ordinary settled retry so the client hands off to the
// authoritative B submission instead of going idle and starting C.
if (liveJob.status === 'running' || liveJob.status === 'requires_action') {
startupTelemetry?.end('deduplicated');
if (generationProtocolVersion < GENERATION_PROTOCOL_V2) {
return sendGenerationJson(
res,
409,
{ code: 'RUN_REPLACED' },
generationProtocolVersion,
);
}
return sendGenerationJson(
res,
200,
{
streamId: existingStreamId,
conversationId: claim.existing.conversationId,
generationCreatedAt: liveJob.createdAt,
status: 'replaced',
},
generationProtocolVersion,
);
}
return sendSettledGeneration(
res,
existingStreamId,
claim.existing.conversationId,
startupTelemetry,
generationProtocolVersion,
);
}
if (liveClientRequestId == null && !isLegacyTokenlessClaim) {
// A syntactically valid claim plus an uncorrelated live job is
// outcome-ambiguous (legacy/corrupt/partially written state). Attaching
// risks cross-wiring two submissions; starting risks double billing.
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership could not be confirmed. Please retry shortly.',
},
generationProtocolVersion,
);
}
logger.debug('[ResumableAgentController] Deduped retried start-generation request', {
userId,
clientRequestId,
streamId: existingStreamId,
});
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
200,
{
streamId: existingStreamId,
conversationId: claim.existing.conversationId,
generationCreatedAt: liveJob.createdAt,
status: 'resumed',
},
generationProtocolVersion,
);
}
// The creator held the claim beyond the startup grace but never made a
// job. Atomically take over its lease; createJob verifies this token in
// the same Redis transaction as job creation, so the abandoned winner
// can no longer wake up and start a second generation.
const takeover = await GenerationJobManager.takeoverGeneration(
userId,
clientRequestId,
existingStreamId,
claim.existing,
).catch((err) => {
logger.error('[ResumableAgentController] Stale idempotency takeover failed', err);
return null;
});
if (
!takeover?.claimed ||
!isValidGenerationClaim(takeover.existing, streamId, conversationId)
) {
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership changed. Please retry shortly.',
},
generationProtocolVersion,
);
}
ownedIdempotencyClaim = takeover.existing;
} else {
// A malformed/unreadable existing claim is outcome-ambiguous. Starting
// anyway would turn a store parsing failure into duplicate generation.
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
return sendGenerationJson(
res,
503,
{
code: 'SERVER_NOT_READY',
error: 'Generation ownership could not be confirmed. Please retry shortly.',
},
generationProtocolVersion,
);
}
}
// AUTOMATIC scheduled fires bypass the interactive concurrent-request limiter (like
// the message-rate limiters): a user with active chats or several schedules due at
// once would otherwise get 429s recorded as schedule errors, walking valid schedules
// toward auto-disable. Scheduler caps govern their concurrency.
//
// Run Now does NOT get that exemption. It is user-paced and can be fired across
// several schedules while the same user already has interactive generations running,
// so exempting it lets one user run unbounded concurrent generations — and unlike an
// automatic occurrence, a 429 here is surfaced to the clicking user rather than
// booked against the schedule (see the throttled branch in fireSchedule).
// Shared predicate, not a hand-rolled copy: the acquire and both release points
// live in different files, and re-deriving it at each is what let them drift.
const exemptFromConcurrency = exemptFromConcurrencyLimiter(req);
if (!exemptFromConcurrency) {
const { allowed, pendingRequests, limit } = await checkAndIncrementPendingRequest(userId);
if (!allowed) {
if (ownedIdempotencyClaim) {
await GenerationJobManager.releaseGeneration(
userId,
clientRequestId,
streamId,
ownedIdempotencyClaim,
).catch(() => {});
}
const violationInfo = getViolationInfo(pendingRequests, limit);
await logViolation(req, res, ViolationTypes.CONCURRENT, violationInfo, violationInfo.score);
startupTelemetry?.end('rejected');
return sendGenerationJson(res, 429, violationInfo, generationProtocolVersion);
}
}
startupTelemetry?.mark('request_admitted');
let client = null;
let jobCreatedAt;
let providerExecutionId;
try {
logger.debug(`[ResumableAgentController] Creating job`, {
streamId,
conversationId,
reqConversationId,
userId,
});
const endpointIconURL = getEndpointIconURL(req, endpointOption);
const responseModel = getAgentResponseModel(req, endpointOption);
const preliminaryUserMessage = getPreliminaryUserMessage(req.body, conversationId);
const preliminaryResponseMessageId = getPreliminaryResponseMessageId(req.body);
const job = await GenerationJobManager.createJob(streamId, userId, conversationId, {
startupTelemetry,
...(recoveredSteerId && { recoveredSteerId }),
...(recoveredSteerPayload && { recoveredSteerPayload }),
...(expectedPredecessorCreatedAt != null && { expectedPredecessorCreatedAt }),
...(ownedIdempotencyClaim?.claimToken && {
idempotencyClientRequestId: clientRequestId,
idempotencyClaimToken: ownedIdempotencyClaim.claimToken,
}),
initialMetadata: {
conversationId,
generationProtocolVersion,
endpoint: endpointOption.endpoint,
iconURL: endpointIconURL,
model: responseModel,
// Recorded HERE because this process owns the generation: the steer
// route may land on a different replica whose own SDK probe would
// answer for the wrong process during a rolling deploy.
preemptCapable: isSteerPreemptSupported(),
// Persist the originating agent so a HITL resume can refuse to rebuild this
// paused run on a different agent (see resume.js).
agent_id: endpointOption.agent_id ?? req.body?.agent_id,
// Persist temporary-chat state so a HITL resume keeps the resumed response
// non-persisted instead of trusting the resume request to re-send the flag.
isTemporary: req.body?.isTemporary,
// Persist schedule bookkeeping ATOMICALLY with the create, so a concurrent
// quiesce can identity-match and abort THIS job from the moment it exists, and
// so a HITL resume can record the outcome (the resume request carries no
// scheduleId). Only set for verified fires.
// scheduleManual records the fire's classification: a paused approval can be
// answered up to the abandonment window later, and the resume must apply the
// same admission rules this boundary does — which needs to know whether the
// occurrence was automatic or an explicit Run Now.
...(scheduleId
? {
scheduleId,
scheduledFor,
...(req._isManualScheduledFire === true ? { scheduleManual: '1' } : {}),
...(typeof scheduleConfigRevision === 'number'
? { scheduleConfigRevision: String(scheduleConfigRevision) }
: {}),
}
: {}),
responseMessageId: preliminaryResponseMessageId,
userMessage: preliminaryUserMessage,
},
});
startupTelemetry?.mark('job_created');
generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
jobCreatedAt = job.createdAt; // Capture creation time to detect job replacement
providerExecutionId = job.metadata?.providerExecutionId;
/** Authentication can precede a slow admission path. Recheck the durable
* account-deletion fence after the job is committed but before execution
* starts. This ordering closes both sides of the race for ordinary and
* trigger-scoped sessions: a fence that wins first rejects this run; a
* fence that starts after this read must observe the already-created job
* in account deletion's active-generation drain. */
if (!(await isAgentTriggerPrincipalActive(userId))) {
throw Object.assign(new Error('Account deletion is in progress'), {
code: 'ACCOUNT_DELETION_IN_PROGRESS',
status: 409,
});
}
if (
providerExecutionId &&
!(await GenerationJobManager.beginProviderExecution(
streamId,
jobCreatedAt,
providerExecutionId,
))
) {
throw Object.assign(new Error('Generation stopped before provider startup'), {
code: 'RUN_REPLACED',
status: 409,
});
}
acceptAgentStartupTelemetry(req, streamId);
startupTelemetry?.mark('metadata_persisted');
req._resumableStreamId = streamId;
getMCPRequestContext(req, undefined, { cleanupOnResponse: false });
let recoveredSteerCommitted = false;
const commitRecoveredSteer = async () => {
if (!recoveredSteerId || recoveredSteerCommitted) {
return;
}
if (client?.skipSaveUserMessage) {
throw new Error('Recovered steer cannot skip user message persistence');
}
const committed = await GenerationJobManager.steering.consumeRecovered(
streamId,
recoveredSteerId,
{ userId, tenantId: req.user?.tenantId },
jobCreatedAt,
);
if (!committed) {
throw new Error('Recovered steer could not be committed after message persistence');
}
recoveredSteerCommitted = true;
};
// Send JSON response IMMEDIATELY so client can connect to SSE stream
// This is critical: tool loading (MCP OAuth) may emit events that the client needs to receive
sendGenerationJson(
res,
200,
{ streamId, conversationId, generationCreatedAt: jobCreatedAt, status: 'started' },
generationProtocolVersion,
);
await attachConversationCreatedAt(req, conversationId, conversationAnchorPromise).then(() =>
startupTelemetry?.mark('conversation_resolved'),
);
// Note: We no longer use res.on('close') to abort since we send JSON immediately.
// The response closes normally after res.json(), which is not an abort condition.
// Abort handling is done through GenerationJobManager via the SSE stream connection.
// Track if partial response was already saved to avoid duplicates. Finalization
// sets it too, so a disconnect firing while the terminal saves run cannot
// overwrite the completed response with an `unfinished` partial.
let partialResponseSaved = false;
/** The ENTIRE in-flight disconnect-save operation, assigned SYNCHRONOUSLY at
* event dispatch — never from inside the async body, whose first await would
* leave a window where settlement sees no pending write while one is starting.
* Terminal paths (success, abort, error) await it before they save or settle. */
let partialSavePromise = null;
const savePartialOnDisconnect = async (aggregatedContent) => {
const persistableContent = filterPersistableAbortContent(aggregatedContent);
if (persistableContent.length === 0) {
logger.debug('[ResumableAgentController] No persistable content to save partial response');
return;
}
const resumeState = await GenerationJobManager.getResumeState(streamId, jobCreatedAt);
if (!resumeState?.userMessage) {
logger.debug('[ResumableAgentController] No user message to save partial response for');
return;
}
// Re-check liveness right before writing: this handler can resume from its
// awaits after the generation finished or was replaced, and a partial written
// then would mark a COMPLETED response unfinished (or write for a job that no
// longer exists). Only a still-live incarnation of THIS generation may save.
const liveJob = await GenerationJobManager.getJob(streamId).catch(() => null);
if (
!liveJob ||
liveJob.createdAt !== jobCreatedAt ||
(liveJob.status !== 'running' && liveJob.status !== 'requires_action') ||
partialResponseSaved
) {
return;
}
partialResponseSaved = true;
const responseConversationId = resumeState.conversationId || conversationId;
try {
const partialMessage = {
messageId: resumeState.responseMessageId || `${resumeState.userMessage.messageId}_`,
conversationId: responseConversationId,
parentMessageId: resumeState.userMessage.messageId,
sender: client?.sender ?? 'AI',
content: persistableContent,
unfinished: true,
error: false,
isCreatedByUser: false,
user: userId,
endpoint: endpointOption.endpoint,
iconURL: resumeState.iconURL || endpointIconURL,
model: resumeState.model || responseModel,
};
if (req.body?.agent_id) {
partialMessage.agent_id = req.body.agent_id;
}
const savePartialMessage = () =>
saveMessage(
{
userId,
isTemporary: req?.body?.isTemporary,
interfaceConfig: req?.config?.interfaceConfig,
},
partialMessage,
{
context: 'api/server/controllers/agents/request.js - partial response on disconnect',
},
);
const savedPartialMessage = tenantId
? await tenantStorage.run({ tenantId, userId }, savePartialMessage)
: await savePartialMessage();
if (!savedPartialMessage) {
throw new Error('Partial response could not be persisted after disconnect');
}
logger.debug(
`[ResumableAgentController] Saved partial response for ${streamId}, content parts: ${persistableContent.length}`,
);
} catch (error) {
logger.error('[ResumableAgentController] Error saving partial response:', error);
// Reset flag so we can try again if subscribers reconnect and leave again
partialResponseSaved = false;
}
};
/**
* Listen for all subscribers leaving to save partial response.
* This ensures the response is saved to DB even if all clients disconnect
* while generation continues.
*
* Note: The messageId used here falls back to `${userMessage.messageId}_` if the
* actual response messageId isn't available yet. The final response save will
* overwrite this with the complete response using the same messageId pattern.
*/
job.emitter.on('allSubscribersLeft', (aggregatedContent) => {
if (partialResponseSaved || !aggregatedContent || aggregatedContent.length === 0) {
return;
}
// Never-rejecting: settlement awaits completion, not success. Tracked rather
// than awaited here so a disconnect cannot hold the emitter, and RETURNED so a
// caller that does await the handler observes the same completion.
partialSavePromise = savePartialOnDisconnect(aggregatedContent).catch((error) => {
logger.error('[ResumableAgentController] Disconnect-partial save failed:', error);
});
return partialSavePromise;
});
/** @type {{ client: TAgentClient; userMCPAuthMap?: Record<string, Record<string, string>> }} */
const result = await initializeClient({
req,
res,
endpointOption,
// Use the job's abort controller signal - allows abort via GenerationJobManager.abortJob()
signal: job.abortController.signal,
jobCreatedAt,
checkpointNamespace: job.metadata?.checkpointNamespace,
});
startupTelemetry?.mark('client_initialized');
client = result.client;
/** Request-shape validation rejects every known edit/regenerate path, but
* the client owns the final persistence decision. Fail closed if a future
* or provider-specific path still derives skip-save for a recovered turn;
* consuming its parked source would otherwise erase the only durable copy
* of the user's words. Re-checked inside commitRecoveredSteer in case a
* client mutates the flag while sending. */
if (recoveredSteerId && client?.skipSaveUserMessage) {
throw new Error('Recovered steer cannot skip user message persistence');
}
if (job.abortController.signal.aborted) {
if (scheduleId) {
// A scheduled fire aborted during initialization (e.g. a delete/quiesce now
// identity-matches this job) must record its terminal outcome and preserve the
// aborted job when that write fails — otherwise the `started` ScheduleRun row
// consumes global capacity until the orphan sweep and the aborted evidence can
// be reaped before the reconciler sees it. Mirror the liveness-abort path
// (abortJob -> reconcile 'interrupted', not 'success').
// An interactive Stop can be what aborted this init: wait for the route's own
// writes (durable abortPersistedAt stamp) before settling, same barrier as the
// generation catch below. FAIL CLOSED: an unconfirmed barrier means the route
// may still be persisting, so don't settle — leave the run active and the
// aborted job preserved for the reconciler, which finalizes it once the
// owner-death fence lapses.
const persisted = await awaitStopAbortPersistence(scheduleId, new Date(scheduledFor)).catch(
(err) => {
logger.warn('[ResumableAgentController] Stop-abort persistence barrier failed', err);
return false;
},
);
const recorded = persisted
? await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: 'interrupted',
conversationId: streamId,
})
: false;
await GenerationJobManager.abortJob(streamId, {
preserveForReconcile: !recorded,
expectedCreatedAt: jobCreatedAt,
}).catch(() => undefined);
} else {
await GenerationJobManager.completeJob(
streamId,
'Request aborted during initialization',
jobCreatedAt,
).catch((completeErr) => {
logger.warn(
'[ResumableAgentController] completeJob failed after initialization abort',
completeErr,
);
});
}
startupTelemetry?.end('aborted');
try {
await finishResumableRequest(req, userId);
} finally {
if (client) {
disposeClient(client);
}
client = null;
if (providerExecutionId) {
await GenerationJobManager.markProviderExecutionDrained?.(
streamId,
jobCreatedAt,
providerExecutionId,
).catch((drainError) => {
logger.warn(
'[ResumableAgentController] Failed to record initialization-abort provider drain',
drainError,
);
});
}
}
return;
}
// Tag the client with THIS generation's identity so HITL terminal side-effects
// (pause CAS, checkpoint prune) can tell whether a newer request has since replaced
// this job on the same conversationId before acting on it.
client.jobCreatedAt = jobCreatedAt;
// Resolve title timing from the public agents endpoint first, then fall
// back to the agent's actual backing provider/custom endpoint.
titleTiming = resolveTitleTiming({
appConfig: req.config,
endpoint: [endpointOption?.endpoint, client?.options?.agent?.endpoint],
});
if (client?.sender) {
void GenerationJobManager.updateMetadata(
streamId,
{ sender: client.sender },
jobCreatedAt,
).catch((err) => {
logger.warn('[ResumableAgentController] Failed to persist response sender', err);
});
}
// Store reference to client's contentParts - graph will be set when run is created
if (client?.contentParts) {
GenerationJobManager.setContentParts(streamId, client.contentParts, jobCreatedAt);
}
let userMessage;
/** BaseClient's background user-message/conversation save (already caught inside
* the client, so it never rejects). Scheduled-run settlement awaits it. */
let userMessagePromise;
const getReqData = (data = {}) => {
if (data.userMessage) {
userMessage = data.userMessage;
}
if (data.userMessagePromise) {
userMessagePromise = data.userMessagePromise;
}
// conversationId is pre-generated, no need to update from callback
};
/**
* Barrier before a scheduled run's terminal settlement. Settlement is what drops
* the run out of the active set — the exact signal deletion drains wait on — so
* every write that could land after it must be flushed first: the background
* user-message/conversation save, any disconnect-partial save, and (for an
* interactive Stop) the abort route's checkpoint prune + partial-response save,
* awaited via the durable `abortPersistedAt` stamp. Without this, account
* deletion can observe zero active runs, destroy the user's data, and have one
* of these pending writes recreate messages for the deleted account.
*
* Returns whether settlement is CLEARED. False only when a requested stop-abort
* barrier could not be confirmed (route still persisting, or its state
* unreadable) — the caller must then NOT settle, leaving the run active for the
* reconciler to finalize once the owner-death fence lapses. Failing open here
* would be the original bug with extra steps.
*/
const awaitPendingPersistence = async ({ stopAbort = false } = {}) => {
if (userMessagePromise) {
await userMessagePromise;
}
if (partialSavePromise) {
await partialSavePromise;
}
// Titles are BILLED work: recordCollectedUsage writes a balance upsert and a
// transaction row. An immediate-mode title still in flight (or just aborted and
// unwinding) must land before settlement, or an account-deletion drain that
// confirms on the settle has the title's usage writes recreate rows for the
// deleted account. The promise carries its own .catch, so it never rejects.
if (immediateTitlePromise) {
await immediateTitlePromise;
}
if (stopAbort && scheduleId && scheduledFor) {
return awaitStopAbortPersistence(scheduleId, new Date(scheduledFor)).catch((err) => {
logger.warn('[ResumableAgentController] Stop-abort persistence barrier failed', err);
return false;
});
}
return true;
};
let immediateTitlePromise = null;
let trailingWritePromise = null;
let backgroundClientCleanupScheduled = false;
let terminalClaim = null;
let terminalClaimFinished = false;
let terminalPersistenceChecked = false;
let terminalWasAborted = false;
let preemptIncomplete = false;
/** A pause-row write failure is terminalized through the exact action/epoch
* barrier. Once that path starts, neither generic background error handler
* may call completeJob: the pause may already have been replaced by a newer
* action or generation by the time the persistence failure is observed. */
let pausePersistenceFailed = false;
const finishOwnedTerminalClaim = async () => {
if (!terminalClaim || terminalClaimFinished) {
return;
}
try {
await GenerationJobManager.finishTerminalJob(terminalClaim);
} finally {
terminalClaimFinished = true;
}
};
/** Runs inside BaseClient immediately before it can start the completed
* response write. A lost claim returns false, and BaseClient skips that
* stale `unfinished:false` write entirely. The fallback invocation below
* supports test/custom clients that do not derive from BaseClient. */
/** OWNER-SIDE persistence acknowledgement for account deletion. A post-terminal
* title persists BILLED work (a balance upsert + a transaction insert) after the
* terminal CAS has already dropped this job out of the active set, so an empty
* active-set enumeration is not settlement. The marker is registered BEFORE that
* CAS — registering after it leaves a window where a deletion quiesce sees neither
* an active job nor a fence and cascades mid-title — and cleared once the title
* work settles (or immediately when no post-terminal title runs). */
let userFinalizationRegistered = false;
const registerUserFinalizationFence = async () => {
if (userFinalizationRegistered) {
return true;
}
// Generation-qualified (jobCreatedAt): a predecessor's late clear on the same
// conversation must never drop THIS generation's marker. One retry absorbs a
// transient store blip; a persistent failure is the caller's fail-closed signal.
for (let attempt = 1; attempt <= 2 && !userFinalizationRegistered; attempt++) {
userFinalizationRegistered = await GenerationJobManager.registerUserFinalization(
userId,
streamId,
req.user?.tenantId,
jobCreatedAt,
)
.then(() => true)
.catch((err) => {
logger.warn(
`[ResumableAgentController] Finalization registration failed (attempt ${attempt}/2): ${streamId}`,
err,
);
return false;
});
}
return userFinalizationRegistered;
};
const clearUserFinalization = () => {
if (!userFinalizationRegistered) {
return;
}
userFinalizationRegistered = false;
void GenerationJobManager.clearUserFinalization(
userId,
streamId,
req.user?.tenantId,
jobCreatedAt,
).catch(() => undefined);
};
const claimBeforeResponsePersistence = async () => {
if (terminalPersistenceChecked) {
return terminalClaim != null;
}
terminalPersistenceChecked = true;
if (client?.pendingApproval) {
// AgentClient installed a durable pause-persistence barrier in the
// running→requires_action CAS. BaseClient must not start its ordinary
// `unfinished:false` response write; the HITL branch persists the
// partial row as unfinished before releasing that barrier.
return false;
}
terminalWasAborted = job.abortController.signal.aborted;
/**
* A preempt boundary that had nothing to inject (cancelled/stale
* request) ends the turn with a genuinely truncated answer — the SDK
* reports it via preempt stats and the halt reason. Persist it with
* the same honest `unfinished` contract an abort gets, never as a
* silent completion.
*/
const preemptStats = client?.run?.getPreemptStats?.();
preemptIncomplete =
(preemptStats?.emptyBoundaries ?? 0) > 0 ||
client?.run?.getHaltReason?.() === 'preempt_incomplete';
// Fence BEFORE the CAS — for EVERY persistence-owning claim, not just titled
// turns: the CAS drops this job out of the active set while the response save
// (and the background user-message/convo saves) are still in flight, so a
// deletion quiesce landing in that window would otherwise see neither an active
// job nor a marker and cascade while this request can still recreate messages.
// Registration is best-effort (the store contract requires marker support, so a
// false here is a transient blip): the title branches re-check with a
// synchronous-title fallback, and a response save under a missed marker is the
// pre-marker window, logged loudly rather than failing the user's turn.
// FAIL CLOSED: without the marker, the terminal CAS would make this job
// invisible to account-deletion quiescing while the response save is still
// ahead — exactly the race the marker fences. Throwing routes the turn to the
// generation-error path, whose own fenced settlement (below) either registers
// its marker or leaves the job ACTIVE for the stale reaper — deletion-visible
// either way.
if (!(await registerUserFinalizationFence())) {
throw new Error('Finalization marker unavailable before terminal persistence');
}
terminalClaim = await GenerationJobManager.claimTerminalJob(
streamId,
terminalWasAborted ? 'aborted' : 'complete',
undefined,
jobCreatedAt,
{
persistencePending: true,
// The terminal CAS necessarily precedes the schedule-outcome write, and the
// claim is frozen, so a scheduled fire must be claimed RETAINED (terminal
// WITHOUT `completedAt`): that record is the only evidence a reconciler could
// read if the outcome write never lands. The settlement below reaps it via
// `clearScheduledJob` as soon as the outcome IS durable. The intended outcome
// rides along for the non-aborted case, because `complete` alone cannot tell
// the reconciler a swallowed provider failure (or a mid-run balance refusal)
// from a clean finish — see classifyScheduleOutcome.
...(scheduleId
? {
preserveForReconcile: true,
...(terminalWasAborted
? {}
: {
scheduleOutcome: classifyScheduleOutcome(
client?.completionError,
'Run ended in error',
),
}),
}
: {}),
},
);
return terminalClaim != null;
};
const disposeBackgroundClient = () => {
if (backgroundClientCleanupScheduled) {
return;
}
backgroundClientCleanupScheduled = true;
if (immediateTitlePromise) {
immediateTitlePromise.finally(() => {
if (client) {
disposeClient(client);
}
});
} else if (client) {
disposeClient(client);
}
};
// Start background generation immediately. The stream layer buffers and persists events
// until an SSE subscriber attaches, so generation no longer waits on subscriber readiness.
const startGeneration = async () => {
/** Immediate-mode title generation runs in parallel with the response, so
* the conversation row may not exist when the title resolves. `convoReady`
* resolves once the response (and thus the conversation) has been saved,
* gating the title's `saveConvo`. Declared here so both the success tail
* and the catch block can settle it and gate `disposeClient` on the title. */
let titleEventPromise = null;
let acceptsTitleEvents = true;
let resolveConvoReady;
const convoReady = new Promise((resolve) => {
resolveConvoReady = resolve;
});
/** Dedicated controller so a user Stop (or a replaced stream) cancels the
* in-flight title — kept separate from `job.abortController`, which
* `completeJob` also aborts on *successful* completion and would otherwise
* cancel a title that is merely slower than a short response. */
const titleAbortController = new AbortController();
/** Separate from `titleAbortController`: a user Stop cancels the in-flight
* title model call but keeps a title that already finished generating.
* Only a superseded/failed stream aborts this to discard such a title so it
* cannot clobber the conversation now owned by the newer run. */
const titleDiscardController = new AbortController();
const abortTitleOnJobAbort = () => titleAbortController.abort();
if (job.abortController.signal.aborted) {
titleAbortController.abort();
} else {
job.abortController.signal.addEventListener('abort', abortTitleOnJobAbort, { once: true });
}
const titleEligible =
addTitle && parentMessageId === Constants.NO_PARENT && isNewConvo && !req.body?.isTemporary;
const emitTitleEvent = ({ conversationId: titleConversationId, title }) => {
titleEventPromise = (async () => {
if (!acceptsTitleEvents || titleAbortController.signal.aborted) {
return;
}
const currentJob = await GenerationJobManager.getJob(streamId);
if (!currentJob || currentJob.createdAt !== jobCreatedAt) {
return;
}
if (titleAbortController.signal.aborted) {
return;
}
await GenerationJobManager.emitChunk(
streamId,
{
event: 'title',
data: {
conversationId: titleConversationId,
title,
},
},
{ expectedCreatedAt: jobCreatedAt },
);
})().catch((err) => {
logger.error('[ResumableAgentController] Error emitting title event', err);
});
return titleEventPromise;
};
try {
const onStart = (userMsg, respMsgId, _isNewConvo) => {
userMessage = userMsg;
// Store userMessage and responseMessageId upfront for resume capability
GenerationJobManager.updateMetadata(
streamId,
{
responseMessageId: respMsgId,
userMessage: {
messageId: userMsg.messageId,
parentMessageId: userMsg.parentMessageId,
conversationId: userMsg.conversationId,
text: userMsg.text,
quotes: userMsg.quotes,
// Persist the turn's uploaded files here (authoritative job metadata) so a
// HITL resume sources them from the job, not the user DB row — which the
// approval prompt can race (the row save may still be in flight when a fast
// /resume reads it). Without this an approved tool run can rebuild without the
// paused turn's files.
...(Array.isArray(req.body?.files) &&
req.body.files.length > 0 && { files: req.body.files }),
// Skill selections aren't on `userMsg` yet at onStart (BaseClient adds them
// later), so source them from the request — otherwise this update overwrites
// the preliminary metadata and a HITL-resumed turn loses its skill pills.
...(Array.isArray(req.body?.manualSkills) &&
req.body.manualSkills.length > 0 && { manualSkills: req.body.manualSkills }),
...(Array.isArray(req.body?.alwaysAppliedSkills) &&
req.body.alwaysAppliedSkills.length > 0 && {
alwaysAppliedSkills: req.body.alwaysAppliedSkills,
}),
},
},
jobCreatedAt,
).catch((err) => {
logger.error('[ResumableAgentController] Failed to persist start metadata', err);
});
GenerationJobManager.emitChunk(
streamId,
{
created: true,
// Skill selections aren't on `userMessage` yet at onStart (BaseClient adds
// them later), so attach them from the request — this is the message
// `trackUserMessage` persists as the authoritative job.metadata.userMessage,
// and it's what the live client renders the user bubble from.
message: {
...userMessage,
// Carry files so trackUserMessage (the authoritative writer) persists them on
// job.metadata.userMessage for a HITL resume (see the updateMetadata above).
...(Array.isArray(req.body?.files) &&
req.body.files.length > 0 && { files: req.body.files }),
...(Array.isArray(req.body?.manualSkills) &&
req.body.manualSkills.length > 0 && { manualSkills: req.body.manualSkills }),
...(Array.isArray(req.body?.alwaysAppliedSkills) &&
req.body.alwaysAppliedSkills.length > 0 && {
alwaysAppliedSkills: req.body.alwaysAppliedSkills,
}),
},
streamId,
},
{ expectedCreatedAt: jobCreatedAt },
).catch((err) => {
logger.error('[ResumableAgentController] Failed to queue created event', err);
});
};
const messageOptions = {
user: userId,
onStart,
getReqData,
isContinued,
isRegenerate,
editedContent,
conversationId,
parentMessageId,
abortController: job.abortController,
overrideParentMessageId,
isEdited: !!editedContent,
beforeResponsePersistence: claimBeforeResponsePersistence,
userMCPAuthMap: result.userMCPAuthMap,
responseMessageId: editedResponseMessageId,
progressOptions: {
res: {
write: () => true,
end: () => {},
headersSent: false,
writableEnded: false,
},
},
};
const sendPromise = client.sendMessage(text, messageOptions);
if (titleEligible && titleTiming === 'immediate') {
immediateTitlePromise = addTitle(req, {
text: text || getAttachmentTitleText(req.body.files),
conversationId,
client,
immediate: true,
convoReady,
signal: titleAbortController.signal,
discardSignal: titleDiscardController.signal,
onTitleGenerated: emitTitleEvent,
}).catch((err) => {
logger.error('[ResumableAgentController] Error in immediate title generation', err);
});
}
const response = await sendPromise;
// HITL: the turn paused for human review (see AgentClient.handleRunInterrupt).
// The job is already `requires_action` with the pending action persisted and
// emitted to the client; the resume route owns finishing this turn. Settle and
// verify the required unfinished history, then tear down without publishing a
// terminal event or completing a successfully persisted paused job.
if (client?.pendingApproval) {
if (response?.databasePromise) {
try {
await response.databasePromise;
} catch (dbErr) {
logger.error(
'[ResumableAgentController] Error settling databasePromise on HITL pause',
dbErr,
);
}
delete response.databasePromise;
}
const pauseActionId = client.pendingApproval.actionId;
const pauseCreatedAt = client.jobCreatedAt ?? jobCreatedAt;
// The barrier claim supersedes the old liveness probe: the user can approve the
// instant the pending-action SSE lands and resume.js can then claim + finalize,
// saving the COMPLETED response, while we are still settling `databasePromise`.
// Only the barrier owner may write the pre-pause response as unfinished, so a
// resumed/replaced generation can no longer clobber the completed row.
const ownsPausePersistence = await GenerationJobManager.approvals.ownsPausePersistence(
streamId,
pauseActionId,
pauseCreatedAt,
);
if (ownsPausePersistence) {
try {
/** BaseClient awaits its first user/conversation write before the
* pause hook, but deliberately swallows a failed/falsy user save
* and may still record the id locally. Re-save idempotently for
* every ordinary user turn before exposing the approval. */
if (!client?.skipSaveUserMessage) {
if (!userMessage) {
throw new Error('User message was unavailable before HITL pause');
}
if (
typeof client.saveMessageToDatabase === 'function' &&
typeof client.getSaveOptions === 'function'
) {
/** Retry through BaseClient so a failure before its original
* saveConvo is repaired along with the message row. Direct
* saveMessage alone cannot recreate that conversation. */
const savedUserTurn = await client.saveMessageToDatabase(
userMessage,
client.getSaveOptions(),
userId,
);
if (!savedUserTurn?.message) {
throw new Error('User message could not be persisted before HITL pause');
}
if (!client.skipSaveConvo && !savedUserTurn.conversation) {
throw new Error('Conversation could not be persisted before HITL pause');
}
} else {
// Custom clients used by integrations/tests may not inherit BaseClient.
const savedUserMessage = await saveMessage(
{
userId,
isTemporary: req?.body?.isTemporary,
interfaceConfig: req?.config?.interfaceConfig,
},
userMessage,
{
context:
'api/server/controllers/agents/request.js - user message before HITL pause',
},
);
if (!savedUserMessage) {
throw new Error('User message could not be persisted before HITL pause');
}
}
}
if (!response?.messageId) {
throw new Error('Response message was unavailable before HITL pause');
}
const savedResponseMessage = await saveMessage(
{
userId,
isTemporary: req?.body?.isTemporary,
interfaceConfig: req?.config?.interfaceConfig,
},
{
...response,
endpoint: endpointOption.endpoint,
unfinished: true,
user: userId,
},
{
context:
'api/server/controllers/agents/request.js - HITL pause (persist unfinished)',
},
);
if (!savedResponseMessage) {
throw new Error('Paused response could not be persisted as unfinished');
}
await commitRecoveredSteer();
} catch (pausePersistenceError) {
pausePersistenceFailed = true;
let failed;
try {
failed = await GenerationJobManager.failPausePersistence(
streamId,
pauseActionId,
pausePersistenceError?.message ?? 'Pause persistence failed',
pauseCreatedAt,
// RETAIN a scheduled fire's failed-pause error: on the ordinary short
// completed TTL this evidence evaporates during any outage longer than
// the TTL, and the reconciler then misreads the vanished run as
// `interrupted` instead of `error`. Reaped by the catch below once the
// outcome write lands, or by the reconciler's error branch.
scheduleId
? {
preserveForReconcile: true,
scheduleOutcome: {
status: 'error',
error: pausePersistenceError?.message ?? 'Pause persistence failed',
},
}
: undefined,
);
} catch (failError) {
logger.error(
`[ResumableAgentController] Failed to terminalize pause persistence error for ${streamId}`,
failError,
);
}
if (failed === true) {
/** Namespaced checkpoints belong exclusively to this epoch,
* so the exact pause-failure CAS winner can safely remove the
* now-unresumable graph state. Legacy shared namespaces are
* left to their guarded/TTL cleanup path. */
const checkpointNamespace = job.metadata?.checkpointNamespace;
if (typeof checkpointNamespace === 'string' && checkpointNamespace !== '') {
try {
await deleteAgentCheckpoint(
conversationId,
req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
undefined,
{ checkpointNamespace },
);
} catch (checkpointError) {
logger.error(
`[ResumableAgentController] Failed to prune checkpoint after pause persistence error for ${streamId}`,
checkpointError,
);
}
}
} else if (failed === false) {
logger.warn(
`[ResumableAgentController] Skipping stale pause persistence failure — ${streamId} no longer owns its barrier`,
);
}
throw pausePersistenceError;
}
const released = await GenerationJobManager.approvals.finishPausePersistence(
streamId,
pauseActionId,
pauseCreatedAt,
);
if (!released) {
logger.warn(
`[ResumableAgentController] Pause persistence barrier changed before release: ${streamId}`,
);
}
} else {
logger.debug(
`[ResumableAgentController] Skipping stale pause persistence — ${streamId} no longer owns its barrier`,
);
}
// UNBLOCK the title BEFORE the persistence barrier below: `addTitle` waits
// on `convoReady` before persisting, so awaiting `immediateTitlePromise`
// with `convoReady` still pending deadlocks this branch. The conversation
// row was saved (and repaired if needed) above, so a title that already
// finished generating may persist now; the abort only cancels a
// still-in-flight title model call.
titleAbortController.abort();
acceptsTitleEvents = false;
resolveConvoReady();
// Every write launched during this segment must land before the pause is
// recorded: a deletion drain treats a recorded pause as settleable and can
// cascade immediately after. That covers the disconnect-partial save, the
// background user-message save, AND the immediate-mode title — a title is
// billed work (balance upsert + transaction insert), and its aborted task
// unwinding through usage persistence after the cascade would recreate rows
// for a deleted account. All never-rejecting by construction.
await awaitPendingPersistence();
// handleRunInterrupt already released the concurrency slot the moment it paused
// (so a fast /resume isn't 429'd); only release here if that didn't happen.
// Always run the MCP request-context cleanup.
await cleanupMCPRequestContextForReq(req);
// Mirrors the increment: only an AUTOMATIC fire skipped the counter. A Run Now
// holds a real slot, so it must release here like an interactive turn.
if (!exemptFromConcurrency && !client?.pendingRequestReleased) {
await decrementPendingRequest(userId);
}
if (client) {
disposeClient(client);
}
// Record a scheduled fire's pause as `requires_action` NOW (not on the
// next reconcile sweep): overlap/capacity checks key on `started`, so a
// run left `started` while paused would wrongly block a run-now or the
// next occurrence. A failed write just falls back to reconcile marking it.
if (scheduleId) {
await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: 'requires_action',
conversationId: streamId,
});
}
logger.debug(
`[ResumableAgentController] Turn paused for approval; awaiting resume: ${streamId}`,
);
startupTelemetry?.end('paused');
return;
}
// BaseClient invokes this before starting its response write. Custom
// clients/tests may return a database promise directly, so keep the
// controller-side fallback before awaiting that promise.
await claimBeforeResponsePersistence();
const endpoint = endpointOption.endpoint;
response.endpoint = endpoint;
const databasePromise = response.databasePromise;
delete response.databasePromise;
const { conversation: convoData = {} } = await databasePromise;
const conversation = { ...convoData };
conversation.title =
conversation && !conversation.title ? null : conversation?.title || 'New Chat';
if (!terminalClaim) {
/** Stop/replacement won before the response persistence hook. The
* BaseClient contract skipped its completed response write; cancel
* title work and leave terminal publication/persistence to the
* actual winner. */
titleAbortController.abort();
titleDiscardController.abort();
job.abortController.signal.removeEventListener('abort', abortTitleOnJobAbort);
acceptsTitleEvents = false;
resolveConvoReady();
// The RUN is identified by (scheduleId, scheduledFor), not by who owns the
// conversationId now, so the winner settling this generation must not leave
// the occurrence `started` — holding a global capacity slot and blocking
// account deletion until the orphan sweep. This turn produced no persisted
// response of its own, so it settles as interrupted, never success. FAIL
// CLOSED on an unconfirmed stop-abort barrier: leave the run active for the
// reconciler rather than settling while the abort route may still persist.
if (scheduleId) {
const cleared = await awaitPendingPersistence({ stopAbort: true });
const recorded = cleared
? await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: 'interrupted',
conversationId: streamId,
})
: false;
if (recorded) {
await clearScheduledJob(streamId, { scheduleId, scheduledFor }).catch((err) =>
logger.warn('[ResumableAgentController] Failed to clear reconciled job', err),
);
}
}
// Release the marker once this loser's remaining background saves land —
// holding it to the TTL would fence the user's deletion pointlessly, while
// clearing before the saves flush would reopen the fenced window.
await awaitPendingPersistence().catch(() => undefined);
clearUserFinalization();
await finishResumableRequest(req, userId);
disposeBackgroundClient();
startupTelemetry?.end(job.abortController.signal.aborted ? 'aborted' : 'replaced');
return;
}
if (req.body.files && Array.isArray(client.options.attachments)) {
const files = buildMessageFiles(req.body.files, client.options.attachments);
if (files.length > 0) {
userMessage.files = files;
}
delete userMessage.image_urls;
}
const shouldGenerateTitle =
addTitle &&
parentMessageId === Constants.NO_PARENT &&
isNewConvo &&
!terminalWasAborted &&
!preemptIncomplete;
/** Whether a scheduled run already ran its deferred title pre-settlement,
* so the post-settle tail must not start a second one. */
let scheduledTitleAwaited = false;
// FINALIZATION owns the response record from here: stop any new
// disconnect-partial save (a subscriber leaving mid-teardown would mark the
// completed response `unfinished`), and let an already-launched one land
// BEFORE the final saves so the completed record is the last writer.
partialResponseSaved = true;
if (partialSavePromise) {
await partialSavePromise;
}
// Save user message BEFORE sending final event to avoid race condition
// where client refetch happens before database is updated
const reqCtx = {
userId: req?.user?.id,
isTemporary: req?.body?.isTemporary,
interfaceConfig: req?.config?.interfaceConfig,
};
if (!client.skipSaveUserMessage) {
if (!userMessage) {
throw new Error('User message was unavailable before terminal persistence');
}
const savedUserMessage = await saveMessage(reqCtx, userMessage, {
context: 'api/server/controllers/agents/request.js - resumable user message',
});
if (!savedUserMessage) {
throw new Error('User message could not be persisted before terminal publication');
}
}
// Only consume the parked recovery source after the explicit user-row
// write above succeeds. `response.databasePromise` alone is insufficient:
// BaseClient intentionally swallows a failed first user-message save.
await commitRecoveredSteer();
// CRITICAL: Save response message BEFORE emitting final event.
// This prevents race conditions where the client sends a follow-up message
// before the response is saved to the database, causing orphaned parentMessageIds.
/** BaseClient can add the id to savedMessageIds even when its model-layer
* save resolved falsy. Re-save the terminal row idempotently and require
* the returned durable row before publishing the normal FINAL. */
const responseIsUnfinished = terminalWasAborted || preemptIncomplete;
const savedResponseMessage = await saveMessage(
reqCtx,
{
...response,
user: userId,
unfinished: responseIsUnfinished,
},
{
context: responseIsUnfinished
? 'api/server/controllers/agents/request.js - terminal response unfinished'
: 'api/server/controllers/agents/request.js - resumable response end',
},
);
if (!savedResponseMessage) {
throw new Error(
responseIsUnfinished
? 'Terminal response could not be persisted as unfinished'
: 'Response message could not be persisted before terminal publication',
);
}
// If the user stopped this turn — or an empty preempt boundary truncated
// it, which persists under the same honest `unfinished` contract — cancel
// the title BEFORE unblocking its persistence wait; otherwise resolving
// `convoReady` lets the title task resume and save before the later abort runs.
if (terminalWasAborted || preemptIncomplete) {
titleAbortController.abort();
} else {
job.abortController.signal.removeEventListener('abort', abortTitleOnJobAbort);
}
// The conversation row now exists and this stream is authoritative; allow
// any in-flight immediate title generation to persist (saveConvo uses noUpsert).
resolveConvoReady();
acceptsTitleEvents = false;
if (titleEventPromise) {
await titleEventPromise;
}
// The pre-CAS fence above covers the window the terminal transition opened.
// Re-assert it here for the paths that only now become known to run
// post-terminal title work, and degrade to a SYNCHRONOUS title when the store
// could not record the fence at all: settling with neither an active job nor a
// marker would let a deletion attempt cascade mid-title once Redis recovers.
// Excluded: the scheduled SUCCESS path, whose deferred title is awaited before
// settlement (never post-terminal).
const deferredTitlePostTerminal =
shouldGenerateTitle &&
titleTiming !== 'immediate' &&
!(scheduleId && !terminalWasAborted);
const immediateTitlePending = titleTiming === 'immediate' && immediateTitlePromise != null;
let fallbackTitleAwaited = false;
if (deferredTitlePostTerminal || immediateTitlePending) {
const fenced = await registerUserFinalizationFence();
if (!fenced) {
if (immediateTitlePending) {
await immediateTitlePromise.catch(() => undefined);
} else {
fallbackTitleAwaited = true;
await addTitle(req, {
text,
response: { ...response },
client,
}).catch((err) => {
logger.error('[ResumableAgentController] Error in title generation', err);
});
}
}
}
// No early release even without title work: the marker now also fences the
// response/background persistence this controller still owes (the tail block
// clears it once awaitPendingPersistence confirms those writes landed).
let terminalPublicationStarted = false;
try {
const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);
const finalEvent = {
final: true,
conversation,
title: conversation.title,
requestMessage: sanitizeMessageForTransmit(userMessage),
responseMessage: {
...response,
...((terminalWasAborted || preemptIncomplete) && { unfinished: true }),
},
...(pendingSteers.length > 0 && { pendingSteers }),
};
logger.debug(
terminalWasAborted
? `[ResumableAgentController] Emitting ABORTED FINAL event`
: `[ResumableAgentController] Emitting FINAL event`,
{
streamId,
wasAbortedBeforeComplete: terminalWasAborted,
userMessageId: userMessage?.messageId,
responseMessageId: response?.messageId,
conversationId: conversation?.conversationId,
},
);
terminalPublicationStarted = true;
const publication = await GenerationJobManager.publishTerminalClaim(
terminalClaim,
finalEvent,
);
let terminalOutcome = 'completed_without_delta';
if (publication.persistenceFailed) {
terminalOutcome = 'error';
} else if (terminalWasAborted) {
terminalOutcome = 'aborted';
}
startupTelemetry?.end(terminalOutcome);
} catch (terminalError) {
/** A failure while constructing the payload happened after this
* controller's terminal CAS but before the manager could durably
* settle it. Publish conservative reconciliation immediately. Once
* publication starts, the manager either stores the payload or owns
* its bounded recovery marker, so retrying with a different payload
* here would only risk duplicate delivery. */
if (!terminalPublicationStarted) {
try {
await GenerationJobManager.publishTerminalClaim(terminalClaim, null);
} catch (reconcileError) {
logger.warn(
'[ResumableAgentController] Failed to publish terminal persistence reconciliation',
reconcileError,
);
}
}
throw terminalError;
} finally {
// Pair every successful claim even when final-event construction or
// transport publication throws. Cleanup is epoch/runtime guarded.
await finishOwnedTerminalClaim();
}
// Settle the scheduled RUN row after publication but before this request ends.
// The terminal claim was taken with `preserveForReconcile` for every scheduled
// fire (see `claimBeforeResponsePersistence`), so the completedAt-less terminal
// job survives the finish above and is this occurrence's reconcile evidence
// until the outcome write below lands — dying in between can only make the
// reconciler re-derive the same outcome, never mislabel a finished run.
if (scheduleId) {
let scheduleOutcomeRecorded;
if (!terminalWasAborted) {
// Deferred-mode titles normally start AFTER this settle, but a title is
// billed work (a balance upsert + a transaction insert), so a scheduled
// run must finish it BEFORE settlement — an account-deletion drain
// confirming on the settle would otherwise have the title's usage writes
// recreate rows for the deleted account. Immediate-mode titles are
// awaited inside awaitPendingPersistence below.
if (shouldGenerateTitle && titleTiming !== 'immediate') {
scheduledTitleAwaited = true;
await addTitle(req, {
text,
response: { ...response },
client,
}).catch((err) => {
logger.error('[ResumableAgentController] Error in title generation', err);
});
}
await awaitPendingPersistence();
// Classified, never a bare `success`: sendCompletion swallows a provider,
// tool, or model failure into an ERROR content part instead of throwing (so
// the interactive UX can show it), which means this branch is also reached by
// runs that produced nothing but an error. See classifyScheduleOutcome.
scheduleOutcomeRecorded = await recordScheduleOutcome({
scheduleId,
scheduledFor,
...classifyScheduleOutcome(client?.completionError, 'Run ended in error'),
conversationId: conversation?.conversationId,
});
} else {
// Record the abort so the run doesn't linger as `started` (blocking
// run-now/overlap until the 30-minute orphan cutoff) — but AFTER every
// pending persistence write, including the Stop route's (settlement is what
// deletion drains confirm on). An UNCONFIRMED barrier means the route may
// still be persisting: FAIL CLOSED and leave the run active — the
// reconciler finalizes it from the retained aborted job once the
// owner-death fence lapses.
const cleared = await awaitPendingPersistence({ stopAbort: true });
scheduleOutcomeRecorded = cleared
? await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: 'interrupted',
conversationId: conversation?.conversationId,
})
: false;
}
// Only reap the retained job once the outcome is durable. Reconcile never
// rescans a terminal run, so nothing else would ever clear it — and while the
// outcome write is still missing it is the ONLY evidence, read as
// success/interrupted from the retained status (never as `error`, which would
// count a user stop toward failure auto-disable). Identity- and
// generation-fenced, so a replacement generation is never destroyed.
if (scheduleOutcomeRecorded) {
await clearScheduledJob(streamId, { scheduleId, scheduledFor }).catch((err) =>
logger.warn('[ResumableAgentController] Failed to clear reconciled job', err),
);
}
}
await finishResumableRequest(req, userId);
// The marker outlives every write this controller still owes: title work AND
// the background user-message/convo saves (awaitPendingPersistence never
// rejects by construction). Clearing before those land would reopen the
// window the marker fences — a deletion quiesce confirming on an empty
// active set while a pending save can still recreate rows.
const releaseUserFinalization = () =>
awaitPendingPersistence().then(
() => clearUserFinalization(),
() => clearUserFinalization(),
);
if (titleTiming === 'immediate') {
// Title was fired in parallel above (if eligible); a stopped turn already
// aborted it before `resolveConvoReady`. Defer disposal until it settles
// so the run/req aren't torn down mid-generation.
if (immediateTitlePromise) {
immediateTitlePromise.finally(() => {
void releaseUserFinalization();
if (client) {
disposeClient(client);
}
});
} else if (client) {
void releaseUserFinalization();
disposeClient(client);
}
} else if (shouldGenerateTitle) {
trailingWritePromise = addTitle(req, {
text: text || getAttachmentTitleText(req.body.files),
response: { ...response },
client,
})
.catch((err) => {
logger.error('[ResumableAgentController] Error in title generation', err);
})
.finally(() => {
void releaseUserFinalization();
if (client) {
disposeClient(client);
}
});
} else {
// No post-terminal title ran on this path (the scheduled success branch —
// or the registration-failure fallback — awaited it while the job was
// still active); release the marker once the remaining saves land.
await releaseUserFinalization();
if (client) {
disposeClient(client);
}
}
} catch (error) {
// Any failure (user Stop, or a preflight/quota failure before the run is
// even created) must cancel the title and unblock its waits: the title's
// `_waitForRun` would otherwise never resolve, deferring client disposal
// until the 45s title timeout, and no title should persist for a failed turn.
titleAbortController.abort();
titleDiscardController.abort();
job.abortController.signal.removeEventListener('abort', abortTitleOnJobAbort);
acceptsTitleEvents = false;
resolveConvoReady();
// Abort classification, mirroring resume.js: the SIGNAL is authoritative; an
// abort-shaped error alone is not. A genuine failure can mention 'abort'
// (driver errors like 'transaction aborted') with nothing having finalized
// the job — routing it here recorded `interrupted` (skipping the failure
// streak and auto-disable policy), emitted no error event, and left the job
// `running` with no controller driving it. abortJob flips the job terminal
// BEFORE any signal fires, so a job still running under this generation is
// proof no abort finalized it.
const abortShaped = error?.name === 'AbortError' || error?.message?.includes('abort');
let wasAborted = job.abortController.signal.aborted;
if (!wasAborted && abortShaped) {
try {
const liveJob = await GenerationJobManager.getJob(streamId);
wasAborted = !(
liveJob &&
liveJob.createdAt === jobCreatedAt &&
liveJob.status === 'running'
);
} catch (readErr) {
logger.warn(
'[ResumableAgentController] Abort classification read failed; treating as failure',
readErr,
);
}
}
let errorScheduleOutcomeRecorded = true;
let errorScheduleOutcome = null;
if (scheduleId) {
// SETTLE LAST: flush every pending persistence write (the background
// user-message/conversation save, a disconnect-partial save, and — for an
// interactive Stop — the abort route's own writes) before the outcome write
// drops this run out of the active set that deletion drains confirm on.
// An UNCONFIRMED stop barrier means the route may still be persisting:
// FAIL CLOSED, skip the settle, and leave the run active — the reconciler
// finalizes it from the preserved aborted job once the fence lapses.
const cleared = await awaitPendingPersistence({ stopAbort: wasAborted });
// The balance middleware's structured refusal is the OWNER's credits, not a
// schedule fault: settle as `skipped_balance` so it walks the
// insufficient_balance streak instead of too_many_failures.
const balanceRefusal = !wasAborted && isBalanceViolationError(error);
let outcomeStatus = 'error';
if (wasAborted) {
outcomeStatus = 'interrupted';
} else if (balanceRefusal) {
outcomeStatus = 'skipped_balance';
}
errorScheduleOutcome = {
status: outcomeStatus,
...(outcomeStatus === 'error' && { error: error.message || 'Generation failed' }),
};
// EVIDENCE FIRST: a won complete-claim stamped its outcome BEFORE the
// persistence that just failed, so the retained job still says `success`.
// Refresh the stamp before the Mongo outcome write — the job store is a
// different failure domain, so this usually survives exactly the outage
// that fails the write below, and the reconciler then reproduces the
// truthful classification instead of re-deriving success.
if (terminalClaim && !wasAborted) {
await GenerationJobManager.updateScheduleOutcome(
streamId,
jobCreatedAt,
errorScheduleOutcome,
);
}
errorScheduleOutcomeRecorded = cleared
? await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: outcomeStatus,
conversationId: streamId,
error: outcomeStatus === 'error' ? error.message : undefined,
})
: false;
}
// Once this controller owns terminal persistence, no competing error
// transition can win. Settle its pending marker with conservative
// reconciliation on any required-write/final-construction failure,
// then release exactly that claim.
if (terminalClaim && !terminalClaimFinished) {
try {
await GenerationJobManager.publishTerminalClaim(terminalClaim, null);
} catch (publishError) {
logger.warn(
'[ResumableAgentController] Failed to publish terminal persistence reconciliation',
publishError,
);
} finally {
await finishOwnedTerminalClaim().catch((finishError) => {
logger.warn(
'[ResumableAgentController] Failed to finish terminal persistence claim',
finishError,
);
});
}
logger.error(
`[ResumableAgentController] Terminal persistence failed for ${streamId}:`,
error,
);
startupTelemetry?.end('error', error);
} else if (pausePersistenceFailed) {
// failPausePersistence owns the only legal requires_action -> error
// transition for this exact action/epoch. Never fall through to
// completeJob, which could race a newer action or replacement job.
logger.error(
`[ResumableAgentController] Pause persistence failed for ${streamId}:`,
error,
);
startupTelemetry?.end('error', error);
} else if (wasAborted) {
logger.debug(`[ResumableAgentController] Generation aborted for ${streamId}`);
startupTelemetry?.end('aborted');
// abortJob already handled the terminal event and cleanup.
} else {
logger.error(`[ResumableAgentController] Generation error for ${streamId}:`, error);
const generationError = error.message || 'Generation failed';
try {
// FAIL CLOSED, same as the completion claim: the error terminal also drops
// the job out of the active set while pending saves are still landing.
// Unfenced, SKIP the terminal CAS — an active job is deletion-visible by
// itself, and the stale-running reaper recovers it.
if (!(await registerUserFinalizationFence())) {
logger.error(
`[ResumableAgentController] Leaving ${streamId} active for the stale reaper — ` +
'error terminal could not be fenced against account deletion',
);
} else {
// completeJob first wins running -> error and atomically parks
// steers, then publishes. A competing abort/pause emits nothing.
// MUST be awaited: when recordScheduleOutcome exhausted its retries this is
// the only write that retains the job as reconcile evidence. Unawaited,
// request cleanup (or process exit) can outrun it and the run is left
// unreconcilable.
await GenerationJobManager.completeJob(streamId, generationError, jobCreatedAt, {
preserveForReconcile: Boolean(scheduleId) && !errorScheduleOutcomeRecorded,
// Ride the classification on the claim itself so the retained evidence
// can never disagree with it (a balance refusal claims an `error`
// terminal but must walk the insufficient_balance streak).
...(errorScheduleOutcome != null && { scheduleOutcome: errorScheduleOutcome }),
});
}
} catch (completeErr) {
logger.warn(
'[ResumableAgentController] completeJob failed during generation-error cleanup',
completeErr,
);
} finally {
startupTelemetry?.end('error', error);
}
}
// One reap for every branch above: a schedule delete (or a preserved terminal
// claim) leaves an already-terminal job behind, and the outcome recorded above
// makes the run terminal, so reconcile never rescans it — nothing else would
// ever clear that job. Skipped when the outcome write failed: the job is then
// the only reconcile evidence. Identity- and generation-fenced.
if (scheduleId && errorScheduleOutcomeRecorded) {
await clearScheduledJob(streamId, { scheduleId, scheduledFor }).catch((err) =>
logger.warn('[ResumableAgentController] Failed to clear reconciled job', err),
);
}
// Release the persistence marker only after every pending save has landed
// (idempotent re-await for the scheduled branch, which already flushed).
// Left registered, the TTL would fence this user's deletion pointlessly.
await awaitPendingPersistence({ stopAbort: wasAborted }).catch(() => undefined);
clearUserFinalization();
try {
await finishResumableRequest(req, userId);
} finally {
disposeBackgroundClient();
}
// Don't continue to title generation after error/abort
return;
}
};
// Start generation and handle any unhandled errors
void startGeneration()
.catch(async (err) => {
logger.error(
`[ResumableAgentController] Unhandled error in background generation: ${err.message}`,
);
startupTelemetry?.end('error', err);
if (!pausePersistenceFailed) {
await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
(completeErr) => {
logger.warn(
'[ResumableAgentController] completeJob failed during background-error cleanup',
completeErr,
);
},
);
}
try {
await finishResumableRequest(req, userId);
} finally {
disposeBackgroundClient();
}
})
.finally(async () => {
await Promise.allSettled([immediateTitlePromise, trailingWritePromise].filter(Boolean));
if (providerExecutionId) {
await GenerationJobManager.markProviderExecutionDrained?.(
streamId,
jobCreatedAt,
providerExecutionId,
);
}
})
.catch((drainError) => {
logger.warn(
'[ResumableAgentController] Failed to record completed provider drain',
drainError,
);
});
} catch (error) {
logger.error('[ResumableAgentController] Initialization error:', error);
const initializationFailure = getInitializationFailure(error);
try {
if (!res.headersSent) {
if (error?.code === 'GENERATION_PREDECESSOR_MISMATCH') {
const currentJob = error.currentJob;
const currentStatus = currentJob?.status;
const predecessorVerified =
currentJob != null &&
Number.isSafeInteger(currentJob.createdAt) &&
currentJob.createdAt >= 0 &&
currentJob.verified !== false;
sendGenerationJson(
res,
409,
{
status: 'predecessor_mismatch',
code: 'GENERATION_PREDECESSOR_MISMATCH',
error: predecessorVerified
? 'A newer generation became current before this request could start.'
: 'The prior generation could not be verified. Please retry.',
streamId,
conversationId: currentJob?.conversationId ?? conversationId,
generationCreatedAt: currentJob?.createdAt,
predecessorVerified,
active:
typeof currentJob?.active === 'boolean'
? currentJob.active
: currentStatus === 'running' || currentStatus === 'requires_action',
},
generationProtocolVersion,
);
} else if (error?.code === 'RECOVERY_PAYLOAD_MISMATCH') {
sendGenerationJson(
res,
409,
{
code: 'RECOVERY_PAYLOAD_MISMATCH',
error: 'The queued message changed before it could be recovered. Please retry.',
},
generationProtocolVersion,
);
} else if (initializationFailure) {
sendGenerationJson(
res,
initializationFailure.status,
initializationFailure,
generationProtocolVersion,
);
} else {
sendGenerationJson(
res,
500,
{ error: error.message || 'Failed to start generation' },
generationProtocolVersion,
);
}
}
} catch (notificationError) {
logger.warn(
'[ResumableAgentController] Failed to send initialization error response',
notificationError,
);
} finally {
startupTelemetry?.end(
error?.code === 'GENERATION_PREDECESSOR_MISMATCH' ? 'deduplicated' : 'error',
error,
);
}
// A scheduled fire whose run row is already `started` (inserted before the
// early 200) must be terminalized here — an init failure after acceptance
// would otherwise leave it running until orphan reconciliation. Before
// completeJob deletes the job (same ordering rationale as the success path).
// If the outcome write fails (Mongo down across its retries), preserve the
// completed job so reconcile records the failure instead of losing the count.
let initScheduleOutcomeRecorded = true;
if (scheduleId) {
initScheduleOutcomeRecorded = await recordScheduleOutcome({
scheduleId,
scheduledFor,
status: 'error',
error: error.message,
conversationId: streamId,
});
}
// Finalize THIS failed job before releasing the idempotency claim. Releasing first would
// let the client's retry win the same key and createJob() the same streamId while we are
// still here. The generation guard is defense-in-depth around that ordering. A
// completeJob() rejection (store hiccup) must NOT skip the
// release + pending-request decrement below, or the retry stays wedged behind the claim
// and the concurrency slot leaks — so swallow its error. (A failed completeJob did not
// finalize anything, so releasing afterward can't let it abort a later replacement.)
if (jobCreatedAt != null) {
const initializationError = initializationFailure
? JSON.stringify(initializationFailure)
: error.message || 'Failed to start generation';
await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt, {
preserveForReconcile: Boolean(scheduleId) && !initScheduleOutcomeRecorded,
}).catch((completeErr) => {
logger.warn(
'[ResumableAgentController] completeJob failed during init-error cleanup',
completeErr,
);
});
}
if (ownedIdempotencyClaim) {
await GenerationJobManager.releaseGeneration(
userId,
clientRequestId,
streamId,
ownedIdempotencyClaim,
).catch(() => {});
}
await finishResumableRequest(req, userId);
if (client) {
disposeClient(client);
}
if (jobCreatedAt != null && providerExecutionId) {
await GenerationJobManager.markProviderExecutionDrained?.(
streamId,
jobCreatedAt,
providerExecutionId,
).catch((drainError) => {
logger.warn(
'[ResumableAgentController] Failed to record initialization-error provider drain',
drainError,
);
});
}
}
};
module.exports = ResumableAgentController;