diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index e5b452f406..9999c05e52 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -489,6 +489,105 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { expect(issue?.executionRunId).toBeNull(); }); + it("does not promote a scheduled retry after the issue is cancelled", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const sourceRunId = randomUUID(); + const now = new Date("2026-04-20T15:00:00.000Z"); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: { + heartbeat: { + wakeOnDemand: true, + maxConcurrentRuns: 1, + }, + }, + permissions: {}, + }); + + await db.insert(heartbeatRuns).values({ + id: sourceRunId, + companyId, + agentId, + invocationSource: "assignment", + triggerDetail: "system", + status: "failed", + error: "upstream overload", + errorCode: "adapter_failed", + finishedAt: now, + contextSnapshot: { + issueId, + wakeReason: "issue_assigned", + }, + updatedAt: now, + createdAt: now, + }); + + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Retry promotion cancellation", + status: "todo", + priority: "medium", + assigneeAgentId: agentId, + executionRunId: sourceRunId, + executionAgentNameKey: "codexcoder", + executionLockedAt: now, + issueNumber: 1, + identifier: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}-3`, + }); + + const scheduled = await heartbeat.scheduleBoundedRetry(sourceRunId, { + now, + random: () => 0.5, + }); + expect(scheduled.outcome).toBe("scheduled"); + if (scheduled.outcome !== "scheduled") return; + + await db.update(issues).set({ + status: "cancelled", + updatedAt: now, + }).where(eq(issues.id, issueId)); + + const promotion = await heartbeat.promoteDueScheduledRetries(scheduled.dueAt); + expect(promotion).toEqual({ promoted: 0, runIds: [] }); + + const oldRetry = await db + .select({ + status: heartbeatRuns.status, + errorCode: heartbeatRuns.errorCode, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, scheduled.run.id)) + .then((rows) => rows[0] ?? null); + expect(oldRetry).toEqual({ + status: "cancelled", + errorCode: "issue_cancelled", + }); + + const issue = await db + .select({ executionRunId: issues.executionRunId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issue?.executionRunId).toBeNull(); + }); + it("exhausts bounded retries after the hard cap", async () => { const companyId = randomUUID(); const agentId = randomUUID(); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 0387cee943..0d2c715f4d 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3550,6 +3550,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const issue = await db .select({ id: issues.id, + status: issues.status, assigneeAgentId: issues.assigneeAgentId, executionRunId: issues.executionRunId, }) @@ -3557,15 +3558,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where(and(eq(issues.id, dueRunIssueId), eq(issues.companyId, dueRun.companyId))) .then((rows) => rows[0] ?? null); - if (issue && issue.assigneeAgentId !== dueRun.agentId) { - const reason = "Cancelled because the issue was reassigned before the scheduled retry became due"; + if (issue && (issue.assigneeAgentId !== dueRun.agentId || issue.status === "cancelled")) { + const issueCancelled = issue.status === "cancelled"; + const reason = issueCancelled + ? "Cancelled because the issue was cancelled before the scheduled retry became due" + : "Cancelled because the issue was reassigned before the scheduled retry became due"; const cancelled = await db .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: now, error: reason, - errorCode: "issue_reassigned", + errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned", updatedAt: now, }) .where( @@ -3608,9 +3612,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Scheduled retry cancelled because issue ownership changed before it became due", + message: issueCancelled + ? "Scheduled retry cancelled because issue was cancelled before it became due" + : "Scheduled retry cancelled because issue ownership changed before it became due", payload: { issueId: issue.id, + issueStatus: issue.status, scheduledRetryAttempt: cancelled.scheduledRetryAttempt, scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null, scheduledRetryReason: cancelled.scheduledRetryReason, @@ -6305,6 +6312,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .select({ id: issues.id, companyId: issues.companyId, + status: issues.status, assigneeAgentId: issues.assigneeAgentId, executionRunId: issues.executionRunId, executionAgentNameKey: issues.executionAgentNameKey, @@ -6331,22 +6339,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const cancelStaleScheduledRetry = async (scheduledRun: typeof heartbeatRuns.$inferSelect) => { + const issueCancelled = issue.status === "cancelled"; if ( scheduledRun.status !== "scheduled_retry" || - scheduledRun.agentId === issue.assigneeAgentId + (scheduledRun.agentId === issue.assigneeAgentId && !issueCancelled) ) { return false; } const now = new Date(); - const reason = "Cancelled because the issue was reassigned before the scheduled retry became due"; + const reason = issueCancelled + ? "Cancelled because the issue was cancelled before the scheduled retry became due" + : "Cancelled because the issue was reassigned before the scheduled retry became due"; const cancelled = await tx .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: now, error: reason, - errorCode: "issue_reassigned", + errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned", updatedAt: now, }) .where(and(eq(heartbeatRuns.id, scheduledRun.id), eq(heartbeatRuns.status, "scheduled_retry"))) @@ -6379,6 +6390,33 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where(and(eq(issues.id, issue.id), eq(issues.executionRunId, scheduledRun.id))); } + const [eventSeq] = await tx + .select({ maxSeq: sql`max(${heartbeatRunEvents.seq})` }) + .from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.runId, cancelled.id)); + + await tx.insert(heartbeatRunEvents).values({ + companyId: cancelled.companyId, + runId: cancelled.id, + agentId: cancelled.agentId, + seq: Number(eventSeq?.maxSeq ?? 0) + 1, + eventType: "lifecycle", + stream: "system", + level: "warn", + message: issueCancelled + ? "Scheduled retry cancelled because issue was cancelled before it became due" + : "Scheduled retry cancelled because issue ownership changed before it became due", + payload: { + issueId: issue.id, + issueStatus: issue.status, + scheduledRetryAttempt: cancelled.scheduledRetryAttempt, + scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null, + scheduledRetryReason: cancelled.scheduledRetryReason, + previousRetryAgentId: cancelled.agentId, + currentAssigneeAgentId: issue.assigneeAgentId, + }, + }); + return true; };