Cancel retries for cancelled issues

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
2026-04-24 17:27:10 -05:00
parent ba28810e4c
commit 32e4e30955
2 changed files with 144 additions and 7 deletions

View File

@@ -489,6 +489,105 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
expect(issue?.executionRunId).toBeNull();
});
it("does not promote a scheduled retry after the issue is cancelled", async () => {
const companyId = randomUUID();
const agentId = randomUUID();
const issueId = randomUUID();
const sourceRunId = randomUUID();
const now = new Date("2026-04-20T15:00:00.000Z");
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
id: agentId,
companyId,
name: "CodexCoder",
role: "engineer",
status: "active",
adapterType: "codex_local",
adapterConfig: {},
runtimeConfig: {
heartbeat: {
wakeOnDemand: true,
maxConcurrentRuns: 1,
},
},
permissions: {},
});
await db.insert(heartbeatRuns).values({
id: sourceRunId,
companyId,
agentId,
invocationSource: "assignment",
triggerDetail: "system",
status: "failed",
error: "upstream overload",
errorCode: "adapter_failed",
finishedAt: now,
contextSnapshot: {
issueId,
wakeReason: "issue_assigned",
},
updatedAt: now,
createdAt: now,
});
await db.insert(issues).values({
id: issueId,
companyId,
title: "Retry promotion cancellation",
status: "todo",
priority: "medium",
assigneeAgentId: agentId,
executionRunId: sourceRunId,
executionAgentNameKey: "codexcoder",
executionLockedAt: now,
issueNumber: 1,
identifier: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}-3`,
});
const scheduled = await heartbeat.scheduleBoundedRetry(sourceRunId, {
now,
random: () => 0.5,
});
expect(scheduled.outcome).toBe("scheduled");
if (scheduled.outcome !== "scheduled") return;
await db.update(issues).set({
status: "cancelled",
updatedAt: now,
}).where(eq(issues.id, issueId));
const promotion = await heartbeat.promoteDueScheduledRetries(scheduled.dueAt);
expect(promotion).toEqual({ promoted: 0, runIds: [] });
const oldRetry = await db
.select({
status: heartbeatRuns.status,
errorCode: heartbeatRuns.errorCode,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, scheduled.run.id))
.then((rows) => rows[0] ?? null);
expect(oldRetry).toEqual({
status: "cancelled",
errorCode: "issue_cancelled",
});
const issue = await db
.select({ executionRunId: issues.executionRunId })
.from(issues)
.where(eq(issues.id, issueId))
.then((rows) => rows[0] ?? null);
expect(issue?.executionRunId).toBeNull();
});
it("exhausts bounded retries after the hard cap", async () => {
const companyId = randomUUID();
const agentId = randomUUID();

View File

@@ -3550,6 +3550,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const issue = await db
.select({
id: issues.id,
status: issues.status,
assigneeAgentId: issues.assigneeAgentId,
executionRunId: issues.executionRunId,
})
@@ -3557,15 +3558,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
.where(and(eq(issues.id, dueRunIssueId), eq(issues.companyId, dueRun.companyId)))
.then((rows) => rows[0] ?? null);
if (issue && issue.assigneeAgentId !== dueRun.agentId) {
const reason = "Cancelled because the issue was reassigned before the scheduled retry became due";
if (issue && (issue.assigneeAgentId !== dueRun.agentId || issue.status === "cancelled")) {
const issueCancelled = issue.status === "cancelled";
const reason = issueCancelled
? "Cancelled because the issue was cancelled before the scheduled retry became due"
: "Cancelled because the issue was reassigned before the scheduled retry became due";
const cancelled = await db
.update(heartbeatRuns)
.set({
status: "cancelled",
finishedAt: now,
error: reason,
errorCode: "issue_reassigned",
errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned",
updatedAt: now,
})
.where(
@@ -3608,9 +3612,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
eventType: "lifecycle",
stream: "system",
level: "warn",
message: "Scheduled retry cancelled because issue ownership changed before it became due",
message: issueCancelled
? "Scheduled retry cancelled because issue was cancelled before it became due"
: "Scheduled retry cancelled because issue ownership changed before it became due",
payload: {
issueId: issue.id,
issueStatus: issue.status,
scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
scheduledRetryReason: cancelled.scheduledRetryReason,
@@ -6305,6 +6312,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
.select({
id: issues.id,
companyId: issues.companyId,
status: issues.status,
assigneeAgentId: issues.assigneeAgentId,
executionRunId: issues.executionRunId,
executionAgentNameKey: issues.executionAgentNameKey,
@@ -6331,22 +6339,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
}
const cancelStaleScheduledRetry = async (scheduledRun: typeof heartbeatRuns.$inferSelect) => {
const issueCancelled = issue.status === "cancelled";
if (
scheduledRun.status !== "scheduled_retry" ||
scheduledRun.agentId === issue.assigneeAgentId
(scheduledRun.agentId === issue.assigneeAgentId && !issueCancelled)
) {
return false;
}
const now = new Date();
const reason = "Cancelled because the issue was reassigned before the scheduled retry became due";
const reason = issueCancelled
? "Cancelled because the issue was cancelled before the scheduled retry became due"
: "Cancelled because the issue was reassigned before the scheduled retry became due";
const cancelled = await tx
.update(heartbeatRuns)
.set({
status: "cancelled",
finishedAt: now,
error: reason,
errorCode: "issue_reassigned",
errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned",
updatedAt: now,
})
.where(and(eq(heartbeatRuns.id, scheduledRun.id), eq(heartbeatRuns.status, "scheduled_retry")))
@@ -6379,6 +6390,33 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
.where(and(eq(issues.id, issue.id), eq(issues.executionRunId, scheduledRun.id)));
}
const [eventSeq] = await tx
.select({ maxSeq: sql<number | null>`max(${heartbeatRunEvents.seq})` })
.from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, cancelled.id));
await tx.insert(heartbeatRunEvents).values({
companyId: cancelled.companyId,
runId: cancelled.id,
agentId: cancelled.agentId,
seq: Number(eventSeq?.maxSeq ?? 0) + 1,
eventType: "lifecycle",
stream: "system",
level: "warn",
message: issueCancelled
? "Scheduled retry cancelled because issue was cancelled before it became due"
: "Scheduled retry cancelled because issue ownership changed before it became due",
payload: {
issueId: issue.id,
issueStatus: issue.status,
scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
scheduledRetryReason: cancelled.scheduledRetryReason,
previousRetryAgentId: cancelled.agentId,
currentAssigneeAgentId: issue.assigneeAgentId,
},
});
return true;
};