From b0d71bdcac9670d0fd9564ee18fbb4b26fa220e3 Mon Sep 17 00:00:00 2001 From: vudc Date: Mon, 18 May 2026 23:35:53 +0700 Subject: [PATCH 1/9] feat: add resume workflow run functionality and UI components --- .../src/components/run-resume-action.tsx | 110 +++++++++++++++++ apps/dashboard/src/lib/api.ts | 12 ++ apps/dashboard/src/lib/status.ts | 9 ++ apps/dashboard/src/routes/runs/$runId.tsx | 10 +- openworkflow/flaky-payment.run.ts | 32 +++++ openworkflow/flaky-payment.ts | 95 +++++++++++++++ packages/openworkflow/client/client.ts | 17 +++ packages/openworkflow/core/backend.ts | 7 ++ packages/openworkflow/core/workflow-run.ts | 24 ++++ packages/openworkflow/postgres/backend.ts | 46 +++++++ packages/openworkflow/sqlite/backend.ts | 67 +++++++++++ .../openworkflow/worker/execution.test.ts | 113 ++++++++++++++++++ 12 files changed, 541 insertions(+), 1 deletion(-) create mode 100644 apps/dashboard/src/components/run-resume-action.tsx create mode 100644 openworkflow/flaky-payment.run.ts create mode 100644 openworkflow/flaky-payment.ts diff --git a/apps/dashboard/src/components/run-resume-action.tsx b/apps/dashboard/src/components/run-resume-action.tsx new file mode 100644 index 00000000..452d4c20 --- /dev/null +++ b/apps/dashboard/src/components/run-resume-action.tsx @@ -0,0 +1,110 @@ +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import { resumeWorkflowRunServerFn } from "@/lib/api"; +import { isRunResumableStatus } from "@/lib/status"; +import type { WorkflowRunStatus } from "openworkflow/internal"; +import { useState } from "react"; + +interface RunResumeActionProps { + runId: string; + status: WorkflowRunStatus; + onResumed?: (() => Promise) | (() => void); +} + +function getErrorMessage(error: unknown): string { + if (error instanceof Error && error.message) { + return error.message; + } + + return "Unable to resume workflow run"; +} + +export function RunResumeAction({ + runId, + status, + onResumed, +}: RunResumeActionProps) { + const [isOpen, setIsOpen] = useState(false); + const [isResuming, setIsResuming] = useState(false); + const [error, setError] = useState(null); + + if (!isRunResumableStatus(status)) { + return null; + } + + async function resumeRun() { + setIsResuming(true); + setError(null); + + try { + await resumeWorkflowRunServerFn({ + data: { + workflowRunId: runId, + }, + }); + await onResumed?.(); + setIsOpen(false); + } catch (caughtError) { + setError(getErrorMessage(caughtError)); + } finally { + setIsResuming(false); + } + } + + return ( + { + setIsOpen(nextOpen); + if (!nextOpen) { + setError(null); + } + }} + > + + + + + Resume this failed run? + + Completed steps stay cached and won't re-run. The failing step will + be retried with a fresh retry budget. Previous failed attempts will + be discarded. + + + + {error &&

{error}

} + + + Cancel + { + void resumeRun(); + }} + disabled={isResuming} + > + {isResuming ? "Resuming..." : "Resume Run"} + + + + + ); +} diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index d399ef81..e604f7ee 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -90,6 +90,18 @@ export const cancelWorkflowRunServerFn = createServerFn({ method: "POST" }) return backend.cancelWorkflowRun({ workflowRunId: data.workflowRunId }); }); +/** + * Resume a failed workflow run by ID. Flips the run back to `pending` and + * drops failed step attempts so the failing step starts with a fresh retry + * budget; completed steps stay cached. + */ +export const resumeWorkflowRunServerFn = createServerFn({ method: "POST" }) + .inputValidator(z.object({ workflowRunId: z.string() })) + .handler(async ({ data }): Promise => { + const backend = await getBackend(); + return backend.resumeWorkflowRun({ workflowRunId: data.workflowRunId }); + }); + /** * List step attempts for a workflow run. */ diff --git a/apps/dashboard/src/lib/status.ts b/apps/dashboard/src/lib/status.ts index af762394..be95d797 100644 --- a/apps/dashboard/src/lib/status.ts +++ b/apps/dashboard/src/lib/status.ts @@ -169,6 +169,11 @@ const CANCELABLE_RUN_STATUSES: ReadonlySet = new Set([ "sleeping", ]); +/** Run statuses that can be resumed from the dashboard. */ +const RESUMABLE_RUN_STATUSES: ReadonlySet = new Set([ + "failed", +]); + const fallbackStatusConfig = STATUS_CONFIG.pending; export function getRunStatusConfig(status: string): StatusConfig { @@ -198,3 +203,7 @@ export function getStatusStatIconClass(status: string): string { export function isRunCancelableStatus(status: string): boolean { return CANCELABLE_RUN_STATUSES.has(status as WorkflowRunStatus); } + +export function isRunResumableStatus(status: string): boolean { + return RESUMABLE_RUN_STATUSES.has(status as WorkflowRunStatus); +} diff --git a/apps/dashboard/src/routes/runs/$runId.tsx b/apps/dashboard/src/routes/runs/$runId.tsx index e9bf8115..288ba772 100644 --- a/apps/dashboard/src/routes/runs/$runId.tsx +++ b/apps/dashboard/src/routes/runs/$runId.tsx @@ -1,6 +1,7 @@ import { AppLayout } from "@/components/app-layout"; import { CursorPaginationControls } from "@/components/cursor-pagination-controls"; import { RunCancelAction } from "@/components/run-cancel-action"; +import { RunResumeAction } from "@/components/run-resume-action"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; @@ -292,7 +293,14 @@ function RunDetailsPage() { /> )} -
+
+ { + await router.invalidate(); + }} + /> workflow marked failed."); +console.log( + "Then open http://localhost:3000, click 'Resume Run', and watch it complete.\n", +); + +const handle = await ow.runWorkflow(flakyPayment.spec, { + cartId: "cart_demo_1", + amountCents: 4200, +}); + +console.log(`Run id: ${handle.workflowRun.id}`); +console.log("Waiting for first terminal state..."); + +try { + const result = await handle.result(); + console.log(`Workflow completed: ${JSON.stringify(result, null, 2)}`); +} catch (error) { + console.log("\nWorkflow failed (as expected on first pass):"); + console.log(error instanceof Error ? error.message : String(error)); + console.log( + "\nNow click 'Resume Run' on the run detail page in the dashboard.", + ); + console.log( + "Leave the worker running so the in-memory attempt counter persists.", + ); +} + +await backend.stop(); diff --git a/openworkflow/flaky-payment.ts b/openworkflow/flaky-payment.ts new file mode 100644 index 00000000..d4d719ec --- /dev/null +++ b/openworkflow/flaky-payment.ts @@ -0,0 +1,95 @@ +import { defineWorkflow } from "openworkflow"; + +interface FlakyPaymentInput { + cartId: string; + amountCents: number; +} + +interface FlakyPaymentOutput { + cartId: string; + authorizationId: string; + receiptId: string; + attemptsForReserve: number; +} + +const RESERVE_MAX_ATTEMPTS = 3; + +// Module-scoped counter. Persists across the failure-then-resume cycle as long +// as the worker process is alive, so the step succeeds on the first attempt +// after `ow.resumeWorkflowRun` is called. +let reserveAttempt = 0; + +/** + * Demo workflow for the Resume feature. + * + * Flow: + * 1. First run: "reserve-funds" throws on every attempt; the step's retry + * budget (RESERVE_MAX_ATTEMPTS) is exhausted, so the workflow run ends in + * `failed`. The downstream steps never run. + * 2. Click "Resume Run" in the dashboard (or call `ow.resumeWorkflowRun(id)`). + * The failed step_attempt rows are dropped and the run is requeued. + * 3. On the next worker tick, "validate-cart" is served from the cache (not + * re-executed); "reserve-funds" runs once more, the counter is now past + * RESERVE_MAX_ATTEMPTS so it returns successfully, and "confirm-payment" + * plus "send-receipt" proceed to completion. + * + * Note: this relies on `reserveAttempt` persisting in the worker process. If + * you restart the worker between the failure and the resume, the counter + * resets, so resume will fail again and you'll need to resume once more. + */ +export const flakyPayment = defineWorkflow( + { name: "flaky-payment" }, + async ({ input, step, run }) => { + console.log(`[run ${run.id}] flaky-payment for cart ${input.cartId}`); + + await step.run({ name: "validate-cart" }, () => { + if (input.amountCents <= 0) { + throw new Error("amountCents must be positive"); + } + }); + + const { authorizationId, attempts } = await step.run( + { + name: "reserve-funds", + retryPolicy: { + maximumAttempts: RESERVE_MAX_ATTEMPTS, + initialInterval: "500ms", + }, + }, + () => { + reserveAttempt++; + console.log(`reserve-funds attempt ${String(reserveAttempt)}`); + + if (reserveAttempt <= RESERVE_MAX_ATTEMPTS) { + throw new Error( + `simulated upstream 503 (attempt ${String(reserveAttempt)})`, + ); + } + + console.log( + `reserve-funds recovered on attempt ${String(reserveAttempt)} (after resume)`, + ); + return { + authorizationId: `auth_${input.cartId}_${String(Date.now())}`, + attempts: reserveAttempt, + }; + }, + ); + + const receiptId = await step.run({ name: "confirm-payment" }, () => { + console.log(`confirming with ${authorizationId}`); + return `rcpt_${input.cartId}`; + }); + + await step.run({ name: "send-receipt" }, () => { + console.log(`receipt ${receiptId} mailed`); + }); + + return { + cartId: input.cartId, + authorizationId, + receiptId, + attemptsForReserve: attempts, + }; + }, +); diff --git a/packages/openworkflow/client/client.ts b/packages/openworkflow/client/client.ts index e5f1f9fd..ac3f7930 100644 --- a/packages/openworkflow/client/client.ts +++ b/packages/openworkflow/client/client.ts @@ -175,6 +175,23 @@ export class OpenWorkflow { await this.backend.cancelWorkflowRun({ workflowRunId }); } + /** + * Resume a failed workflow run. The run's status flips back to `pending` + * so the next worker tick picks it up. Already-completed steps are served + * from history without re-executing; failed step attempts are discarded so + * the failing step starts with a fresh retry budget. + * @param workflowRunId - The ID of the failed workflow run to resume + * @returns Promise + * @throws {Error} If the run does not exist or is not in `failed` status + * @example + * ```ts + * await ow.resumeWorkflowRun("123"); + * ``` + */ + async resumeWorkflowRun(workflowRunId: string): Promise { + await this.backend.resumeWorkflowRun({ workflowRunId }); + } + /** * Send a signal to all workflows currently waiting on the given signal * string. If no workflow is waiting, the signal is silently dropped. diff --git a/packages/openworkflow/core/backend.ts b/packages/openworkflow/core/backend.ts index efc4b07f..ad59acb4 100644 --- a/packages/openworkflow/core/backend.ts +++ b/packages/openworkflow/core/backend.ts @@ -47,6 +47,9 @@ export interface Backend { cancelWorkflowRun( params: Readonly, ): Promise; + resumeWorkflowRun( + params: Readonly, + ): Promise; // Step Attempts createStepAttempt( @@ -143,6 +146,10 @@ export interface CancelWorkflowRunParams { workflowRunId: string; } +export interface ResumeWorkflowRunParams { + workflowRunId: string; +} + export interface CreateStepAttemptParams { workflowRunId: string; workerId: string; diff --git a/packages/openworkflow/core/workflow-run.ts b/packages/openworkflow/core/workflow-run.ts index ae967f3c..942c3b8a 100644 --- a/packages/openworkflow/core/workflow-run.ts +++ b/packages/openworkflow/core/workflow-run.ts @@ -58,6 +58,30 @@ export function resolveCancelWorkflowRunConflict( throw new Error("Failed to cancel workflow run"); } +/** + * Resolve the outcome when a resumeWorkflowRun UPDATE affected no rows. The + * UPDATE is gated on `status = 'failed'`, so a zero-row outcome means either + * the run doesn't exist or it isn't in a resumable state. + * @param workflowRunId - ID of the workflow run (used in error messages) + * @param existing - Current workflow run, or null if not found + * @returns Never; always throws describing why the resume is impossible + * @throws {Error} If the run does not exist or is not in `failed` status + */ +export function resolveResumeWorkflowRunConflict( + workflowRunId: string, + existing: Readonly | null, +): never { + if (!existing) { + // eslint-disable-next-line functional/no-throw-statements + throw new Error(`Workflow run ${workflowRunId} does not exist`); + } + + // eslint-disable-next-line functional/no-throw-statements + throw new Error( + `Cannot resume workflow run ${workflowRunId} with status ${existing.status}; only failed runs can be resumed`, + ); +} + /** * WorkflowRun represents a single execution instance of a workflow. */ diff --git a/packages/openworkflow/postgres/backend.ts b/packages/openworkflow/postgres/backend.ts index 77292a78..e4d128ac 100644 --- a/packages/openworkflow/postgres/backend.ts +++ b/packages/openworkflow/postgres/backend.ts @@ -5,6 +5,7 @@ import { Backend, WorkflowRunCounts, CancelWorkflowRunParams, + ResumeWorkflowRunParams, ClaimWorkflowRunParams, CreateStepAttemptParams, CreateWorkflowRunParams, @@ -37,6 +38,7 @@ import { StepAttempt } from "../core/step-attempt.js"; import { computeFailedWorkflowRunUpdate } from "../core/workflow-definition.js"; import { resolveCancelWorkflowRunConflict, + resolveResumeWorkflowRunConflict, WorkflowRun, } from "../core/workflow-run.js"; import { @@ -763,6 +765,50 @@ export class BackendPostgres implements Backend { return updated; } + async resumeWorkflowRun( + params: ResumeWorkflowRunParams, + ): Promise { + return await this.pg.begin(async (sql): Promise => { + const tx = sql as unknown as Postgres; + const workflowRunsTable = this.workflowRunsTable(tx); + const stepAttemptsTable = this.stepAttemptsTable(tx); + + const [updated] = await tx` + UPDATE ${workflowRunsTable} + SET + "status" = 'pending', + "worker_id" = NULL, + "error" = NULL, + "finished_at" = NULL, + "available_at" = NOW(), + "updated_at" = NOW() + WHERE "namespace_id" = ${this.namespaceId} + AND "id" = ${params.workflowRunId} + AND "status" = 'failed' + RETURNING * + `; + + if (!updated) { + const existing = await this.getWorkflowRun({ + workflowRunId: params.workflowRunId, + }); + resolveResumeWorkflowRunConflict(params.workflowRunId, existing); + } + + // Drop the prior failed attempts so the next worker pass starts the + // failed step with a fresh retry budget and the existing completed + // attempts remain in cache (replay skips re-execution). + await tx` + DELETE FROM ${stepAttemptsTable} + WHERE "namespace_id" = ${this.namespaceId} + AND "workflow_run_id" = ${params.workflowRunId} + AND "status" = 'failed' + `; + + return updated; + }); + } + private async wakeParentWorkflowRun( childWorkflowRun: Readonly, ): Promise { diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index fa610357..fa3005a0 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -4,6 +4,7 @@ import { DEFAULT_RUN_IDEMPOTENCY_PERIOD_MS, Backend, CancelWorkflowRunParams, + ResumeWorkflowRunParams, ClaimWorkflowRunParams, CreateStepAttemptParams, CreateWorkflowRunParams, @@ -37,6 +38,7 @@ import { StepAttempt } from "../core/step-attempt.js"; import { computeFailedWorkflowRunUpdate } from "../core/workflow-definition.js"; import { resolveCancelWorkflowRunConflict, + resolveResumeWorkflowRunConflict, WorkflowRun, } from "../core/workflow-run.js"; import { @@ -742,6 +744,71 @@ export class BackendSqlite implements Backend { return updated; } + async resumeWorkflowRun( + params: ResumeWorkflowRunParams, + ): Promise { + const currentTime = now(); + + try { + this.db.exec("BEGIN IMMEDIATE"); + + const updateStmt = this.db.prepare(` + UPDATE "workflow_runs" + SET + "status" = 'pending', + "worker_id" = NULL, + "error" = NULL, + "finished_at" = NULL, + "available_at" = ?, + "updated_at" = ? + WHERE "namespace_id" = ? + AND "id" = ? + AND "status" = 'failed' + `); + + const updateResult = updateStmt.run( + currentTime, + currentTime, + this.namespaceId, + params.workflowRunId, + ); + + if (updateResult.changes === 0) { + this.db.exec("ROLLBACK"); + const existing = await this.getWorkflowRun({ + workflowRunId: params.workflowRunId, + }); + resolveResumeWorkflowRunConflict(params.workflowRunId, existing); + } + + this.db + .prepare( + ` + DELETE FROM "step_attempts" + WHERE "namespace_id" = ? + AND "workflow_run_id" = ? + AND "status" = 'failed' + `, + ) + .run(this.namespaceId, params.workflowRunId); + + this.db.exec("COMMIT"); + } catch (error) { + try { + this.db.exec("ROLLBACK"); + } catch { + // ignore + } + throw error; + } + + const updated = await this.getWorkflowRun({ + workflowRunId: params.workflowRunId, + }); + requireRow(updated, "resume workflow run"); + return updated; + } + /** * Return positional placeholders for {@link RUNNING_WORKFLOW_RUN_OWNED_WHERE} * in the order the fragment expects: namespace, run id, worker id. diff --git a/packages/openworkflow/worker/execution.test.ts b/packages/openworkflow/worker/execution.test.ts index 5a30ab58..1fe71707 100644 --- a/packages/openworkflow/worker/execution.test.ts +++ b/packages/openworkflow/worker/execution.test.ts @@ -2994,6 +2994,119 @@ describe("StepExecutor", () => { expect(status).toBe("failed"); sendSignalSpy.mockRestore(); }); + + test("resumeWorkflowRun re-runs the failed step without re-executing completed steps", async () => { + const backend = await createTestBackend(); + const client = new OpenWorkflow({ backend }); + + let validateRuns = 0; + let flakyRuns = 0; + let shouldFail = true; + + const workflow = client.defineWorkflow( + { name: `resume-from-failure-${randomUUID()}` }, + async ({ step }) => { + const validated = await step.run({ name: "validate" }, () => { + validateRuns++; + return "ok"; + }); + + const flaky = await step.run( + { name: "flaky", retryPolicy: { maximumAttempts: 2 } }, + () => { + flakyRuns++; + if (shouldFail) { + throw new Error("simulated upstream failure"); + } + return "recovered"; + }, + ); + + return { validated, flaky }; + }, + ); + + const worker = client.newWorker({ concurrency: 1 }); + const handle = await workflow.run(); + + const failedStatus = await tickUntilTerminal( + backend, + worker, + handle.workflowRun.id, + 40, + 25, + ); + expect(failedStatus).toBe("failed"); + expect(validateRuns).toBe(1); + expect(flakyRuns).toBe(2); + + const stepsBeforeResume = await backend.listStepAttempts({ + workflowRunId: handle.workflowRun.id, + limit: 100, + }); + const failedBefore = stepsBeforeResume.data.filter( + (s) => s.status === "failed", + ); + expect(failedBefore.length).toBe(2); + + shouldFail = false; + await backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }); + + const resumedRun = await backend.getWorkflowRun({ + workflowRunId: handle.workflowRun.id, + }); + expect(resumedRun?.status).toBe("pending"); + expect(resumedRun?.error).toBeNull(); + expect(resumedRun?.finishedAt).toBeNull(); + expect(resumedRun?.workerId).toBeNull(); + + const stepsAfterResume = await backend.listStepAttempts({ + workflowRunId: handle.workflowRun.id, + limit: 100, + }); + expect( + stepsAfterResume.data.some((s) => s.status === "failed"), + ).toBe(false); + expect( + stepsAfterResume.data.some( + (s) => s.stepName === "validate" && s.status === "completed", + ), + ).toBe(true); + + const finalStatus = await tickUntilTerminal( + backend, + worker, + handle.workflowRun.id, + 40, + 25, + ); + expect(finalStatus).toBe("completed"); + // "validate" was cached from the original run, not re-executed + expect(validateRuns).toBe(1); + // "flaky" ran 2 times on the original run + 1 on resume + expect(flakyRuns).toBe(3); + + const result = await handle.result(); + expect(result).toEqual({ validated: "ok", flaky: "recovered" }); + }); + + test("resumeWorkflowRun throws when the run is not in failed status", async () => { + const backend = await createTestBackend(); + const client = new OpenWorkflow({ backend }); + + const workflow = client.defineWorkflow( + { name: `resume-invalid-${randomUUID()}` }, + async ({ step }) => { + return await step.run({ name: "noop" }, () => "ok"); + }, + ); + + const handle = await workflow.run(); + + await expect( + backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }), + ).rejects.toThrow(/Cannot resume workflow run.*pending/); + }); }); describe("executeWorkflow", () => { From 3be38eb076264b7e19ac735ec796ce669ee7e61e Mon Sep 17 00:00:00 2001 From: Duong Cong Vu <32474388+vudc@users.noreply.github.com> Date: Mon, 18 May 2026 23:50:01 +0700 Subject: [PATCH 2/9] fix(openworkflow): return the resumed run from client resumeWorkflowRun Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packages/openworkflow/client/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/openworkflow/client/client.ts b/packages/openworkflow/client/client.ts index ac3f7930..411f4814 100644 --- a/packages/openworkflow/client/client.ts +++ b/packages/openworkflow/client/client.ts @@ -181,15 +181,15 @@ export class OpenWorkflow { * from history without re-executing; failed step attempts are discarded so * the failing step starts with a fresh retry budget. * @param workflowRunId - The ID of the failed workflow run to resume - * @returns Promise + * @returns The updated workflow run * @throws {Error} If the run does not exist or is not in `failed` status * @example * ```ts * await ow.resumeWorkflowRun("123"); * ``` */ - async resumeWorkflowRun(workflowRunId: string): Promise { - await this.backend.resumeWorkflowRun({ workflowRunId }); + async resumeWorkflowRun(workflowRunId: string): Promise { + return await this.backend.resumeWorkflowRun({ workflowRunId }); } /** From a0ac0834ce99db65053bddba6711276578e90a7b Mon Sep 17 00:00:00 2001 From: vudc Date: Wed, 22 Jul 2026 22:14:05 +0700 Subject: [PATCH 3/9] fix(openworkflow): address review feedback on resume workflow run Backend correctness: - Delete every non-successful step attempt on resume, not just failed ones. Sleep, signal-wait and child-workflow attempts sit in 'running' and could survive a resume, making replay treat the step as in-flight instead of re-executing it. - Clear started_at so a resumed run reports its new lifecycle, matching rescheduleWorkflowRunAfterFailedStepAttempt. - Restructure the SQLite transaction so the conflict lookup happens outside it, removing the double ROLLBACK on an already-closed transaction. - Read the conflicting run through the transaction in Postgres instead of the outer connection, and use the shared withTransaction helper. Tests: - Add a resumeWorkflowRun() block to the shared backend testsuite so both SQLite and Postgres are covered; the SQLite path had none. - Cover the non-existent-run branch of resolveResumeWorkflowRunConflict. - Assert started_at is cleared and only successful attempts survive. Remove the flaky-payment demo, which drove failures from module-scoped mutable state in a workflow handler. --- openworkflow/flaky-payment.run.ts | 32 ---- openworkflow/flaky-payment.ts | 95 ------------ packages/openworkflow/postgres/backend.ts | 30 ++-- packages/openworkflow/sqlite/backend.ts | 60 ++++---- .../openworkflow/testing/backend.testsuite.ts | 145 ++++++++++++++++++ .../openworkflow/worker/execution.test.ts | 16 +- 6 files changed, 212 insertions(+), 166 deletions(-) delete mode 100644 openworkflow/flaky-payment.run.ts delete mode 100644 openworkflow/flaky-payment.ts diff --git a/openworkflow/flaky-payment.run.ts b/openworkflow/flaky-payment.run.ts deleted file mode 100644 index 8810c151..00000000 --- a/openworkflow/flaky-payment.run.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { backend, ow } from "./client.js"; -import { flakyPayment } from "./flaky-payment.js"; - -console.log("Running flaky-payment workflow..."); -console.log("Expected: 3 failed attempts -> workflow marked failed."); -console.log( - "Then open http://localhost:3000, click 'Resume Run', and watch it complete.\n", -); - -const handle = await ow.runWorkflow(flakyPayment.spec, { - cartId: "cart_demo_1", - amountCents: 4200, -}); - -console.log(`Run id: ${handle.workflowRun.id}`); -console.log("Waiting for first terminal state..."); - -try { - const result = await handle.result(); - console.log(`Workflow completed: ${JSON.stringify(result, null, 2)}`); -} catch (error) { - console.log("\nWorkflow failed (as expected on first pass):"); - console.log(error instanceof Error ? error.message : String(error)); - console.log( - "\nNow click 'Resume Run' on the run detail page in the dashboard.", - ); - console.log( - "Leave the worker running so the in-memory attempt counter persists.", - ); -} - -await backend.stop(); diff --git a/openworkflow/flaky-payment.ts b/openworkflow/flaky-payment.ts deleted file mode 100644 index d4d719ec..00000000 --- a/openworkflow/flaky-payment.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { defineWorkflow } from "openworkflow"; - -interface FlakyPaymentInput { - cartId: string; - amountCents: number; -} - -interface FlakyPaymentOutput { - cartId: string; - authorizationId: string; - receiptId: string; - attemptsForReserve: number; -} - -const RESERVE_MAX_ATTEMPTS = 3; - -// Module-scoped counter. Persists across the failure-then-resume cycle as long -// as the worker process is alive, so the step succeeds on the first attempt -// after `ow.resumeWorkflowRun` is called. -let reserveAttempt = 0; - -/** - * Demo workflow for the Resume feature. - * - * Flow: - * 1. First run: "reserve-funds" throws on every attempt; the step's retry - * budget (RESERVE_MAX_ATTEMPTS) is exhausted, so the workflow run ends in - * `failed`. The downstream steps never run. - * 2. Click "Resume Run" in the dashboard (or call `ow.resumeWorkflowRun(id)`). - * The failed step_attempt rows are dropped and the run is requeued. - * 3. On the next worker tick, "validate-cart" is served from the cache (not - * re-executed); "reserve-funds" runs once more, the counter is now past - * RESERVE_MAX_ATTEMPTS so it returns successfully, and "confirm-payment" - * plus "send-receipt" proceed to completion. - * - * Note: this relies on `reserveAttempt` persisting in the worker process. If - * you restart the worker between the failure and the resume, the counter - * resets, so resume will fail again and you'll need to resume once more. - */ -export const flakyPayment = defineWorkflow( - { name: "flaky-payment" }, - async ({ input, step, run }) => { - console.log(`[run ${run.id}] flaky-payment for cart ${input.cartId}`); - - await step.run({ name: "validate-cart" }, () => { - if (input.amountCents <= 0) { - throw new Error("amountCents must be positive"); - } - }); - - const { authorizationId, attempts } = await step.run( - { - name: "reserve-funds", - retryPolicy: { - maximumAttempts: RESERVE_MAX_ATTEMPTS, - initialInterval: "500ms", - }, - }, - () => { - reserveAttempt++; - console.log(`reserve-funds attempt ${String(reserveAttempt)}`); - - if (reserveAttempt <= RESERVE_MAX_ATTEMPTS) { - throw new Error( - `simulated upstream 503 (attempt ${String(reserveAttempt)})`, - ); - } - - console.log( - `reserve-funds recovered on attempt ${String(reserveAttempt)} (after resume)`, - ); - return { - authorizationId: `auth_${input.cartId}_${String(Date.now())}`, - attempts: reserveAttempt, - }; - }, - ); - - const receiptId = await step.run({ name: "confirm-payment" }, () => { - console.log(`confirming with ${authorizationId}`); - return `rcpt_${input.cartId}`; - }); - - await step.run({ name: "send-receipt" }, () => { - console.log(`receipt ${receiptId} mailed`); - }); - - return { - cartId: input.cartId, - authorizationId, - receiptId, - attemptsForReserve: attempts, - }; - }, -); diff --git a/packages/openworkflow/postgres/backend.ts b/packages/openworkflow/postgres/backend.ts index e4d128ac..88534353 100644 --- a/packages/openworkflow/postgres/backend.ts +++ b/packages/openworkflow/postgres/backend.ts @@ -768,8 +768,7 @@ export class BackendPostgres implements Backend { async resumeWorkflowRun( params: ResumeWorkflowRunParams, ): Promise { - return await this.pg.begin(async (sql): Promise => { - const tx = sql as unknown as Postgres; + return await this.withTransaction(async (tx): Promise => { const workflowRunsTable = this.workflowRunsTable(tx); const stepAttemptsTable = this.stepAttemptsTable(tx); @@ -779,6 +778,7 @@ export class BackendPostgres implements Backend { "status" = 'pending', "worker_id" = NULL, "error" = NULL, + "started_at" = NULL, "finished_at" = NULL, "available_at" = NOW(), "updated_at" = NOW() @@ -789,20 +789,30 @@ export class BackendPostgres implements Backend { `; if (!updated) { - const existing = await this.getWorkflowRun({ - workflowRunId: params.workflowRunId, - }); - resolveResumeWorkflowRunConflict(params.workflowRunId, existing); + const [existing] = await tx` + SELECT * + FROM ${workflowRunsTable} + WHERE "namespace_id" = ${this.namespaceId} + AND "id" = ${params.workflowRunId} + LIMIT 1 + `; + + resolveResumeWorkflowRunConflict( + params.workflowRunId, + existing ?? null, + ); } - // Drop the prior failed attempts so the next worker pass starts the - // failed step with a fresh retry budget and the existing completed - // attempts remain in cache (replay skips re-execution). + // Drop every attempt that did not succeed. Failed attempts go so the + // failing step gets a fresh retry budget; still-'running' attempts + // (sleep, signal-wait, child workflow) go so replay does not mistake + // them for in-flight work. Successful attempts stay and are replayed + // from cache without re-executing. await tx` DELETE FROM ${stepAttemptsTable} WHERE "namespace_id" = ${this.namespaceId} AND "workflow_run_id" = ${params.workflowRunId} - AND "status" = 'failed' + AND "status" NOT IN ('completed', 'succeeded') `; return updated; diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index fa3005a0..c7d22b3d 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -275,7 +275,8 @@ export class BackendSqlite implements Backend { `); const row = stmt.get(this.namespaceId, params.workflowRunId) as - WorkflowRunRow | undefined; + | WorkflowRunRow + | undefined; return Promise.resolve(row ? rowToWorkflowRun(row) : null); } @@ -394,7 +395,8 @@ export class BackendSqlite implements Backend { LIMIT 1 `); const row = stmt.get(this.namespaceId, params.stepAttemptId) as - { data: string | null } | undefined; + | { data: string | null } + | undefined; if (!row) return Promise.resolve(undefined); return Promise.resolve( @@ -748,16 +750,17 @@ export class BackendSqlite implements Backend { params: ResumeWorkflowRunParams, ): Promise { const currentTime = now(); + let resumed = false; + this.db.exec("BEGIN IMMEDIATE"); try { - this.db.exec("BEGIN IMMEDIATE"); - const updateStmt = this.db.prepare(` UPDATE "workflow_runs" SET "status" = 'pending', "worker_id" = NULL, "error" = NULL, + "started_at" = NULL, "finished_at" = NULL, "available_at" = ?, "updated_at" = ? @@ -773,38 +776,40 @@ export class BackendSqlite implements Backend { params.workflowRunId, ); - if (updateResult.changes === 0) { - this.db.exec("ROLLBACK"); - const existing = await this.getWorkflowRun({ - workflowRunId: params.workflowRunId, - }); - resolveResumeWorkflowRunConflict(params.workflowRunId, existing); + resumed = updateResult.changes > 0; + + if (resumed) { + // Drop every attempt that did not succeed. Failed attempts go so the + // failing step gets a fresh retry budget; still-'running' attempts + // (sleep, signal-wait, child workflow) go so replay does not mistake + // them for in-flight work. Successful attempts stay and are replayed + // from cache without re-executing. + this.db + .prepare( + ` + DELETE FROM "step_attempts" + WHERE "namespace_id" = ? + AND "workflow_run_id" = ? + AND "status" NOT IN ('completed', 'succeeded') + `, + ) + .run(this.namespaceId, params.workflowRunId); } - this.db - .prepare( - ` - DELETE FROM "step_attempts" - WHERE "namespace_id" = ? - AND "workflow_run_id" = ? - AND "status" = 'failed' - `, - ) - .run(this.namespaceId, params.workflowRunId); - this.db.exec("COMMIT"); } catch (error) { - try { - this.db.exec("ROLLBACK"); - } catch { - // ignore - } + this.db.exec("ROLLBACK"); throw error; } const updated = await this.getWorkflowRun({ workflowRunId: params.workflowRunId, }); + + if (!resumed) { + resolveResumeWorkflowRunConflict(params.workflowRunId, updated); + } + requireRow(updated, "resume workflow run"); return updated; } @@ -1080,7 +1085,8 @@ export class BackendSqlite implements Backend { `); const row = stmt.get(this.namespaceId, params.stepAttemptId) as - StepAttemptRow | undefined; + | StepAttemptRow + | undefined; return Promise.resolve(row ? rowToStepAttempt(row) : null); } diff --git a/packages/openworkflow/testing/backend.testsuite.ts b/packages/openworkflow/testing/backend.testsuite.ts index 22e5291b..be12cbd7 100644 --- a/packages/openworkflow/testing/backend.testsuite.ts +++ b/packages/openworkflow/testing/backend.testsuite.ts @@ -2573,6 +2573,151 @@ export function testBackend(options: TestBackendOptions): void { }); }); + describe("resumeWorkflowRun()", () => { + test("flips a failed run back to pending and clears failure fields", async () => { + const backend = await setup(); + + await createPendingWorkflowRun(backend); + const failedId = await claimAndFailNextPendingRun(backend); + + const resumed = await backend.resumeWorkflowRun({ + workflowRunId: failedId, + }); + + expect(resumed.status).toBe("pending"); + expect(resumed.error).toBeNull(); + expect(resumed.workerId).toBeNull(); + expect(resumed.startedAt).toBeNull(); + expect(resumed.finishedAt).toBeNull(); + expect(resumed.availableAt).not.toBeNull(); + expect(deltaSeconds(resumed.availableAt)).toBeLessThan(1); + + await teardown(backend); + }); + + test("drops every unsuccessful step attempt but keeps successful ones", async () => { + const backend = await setup(); + + const claimed = await createClaimedWorkflowRun(backend); + const workerId = claimed.workerId!; // eslint-disable-line @typescript-eslint/no-non-null-assertion + + const completed = await backend.createStepAttempt({ + workflowRunId: claimed.id, + workerId, + stepName: "completed-step", + kind: "function", + config: {}, + context: null, + }); + await backend.completeStepAttempt({ + workflowRunId: claimed.id, + stepAttemptId: completed.id, + workerId, + output: { ok: true }, + }); + + const failed = await backend.createStepAttempt({ + workflowRunId: claimed.id, + workerId, + stepName: "failed-step", + kind: "function", + config: {}, + context: null, + }); + await backend.failStepAttempt({ + workflowRunId: claimed.id, + stepAttemptId: failed.id, + workerId, + error: { message: "boom" }, + }); + + // left in 'running': mirrors a sleep/signal-wait/child-workflow + // attempt that never reached a terminal state before the run failed + await backend.createStepAttempt({ + workflowRunId: claimed.id, + workerId, + stepName: "running-step", + kind: "sleep", + config: {}, + context: { + kind: "sleep", + resumeAt: new Date(Date.now() + 60_000).toISOString(), + }, + }); + + await backend.failWorkflowRun({ + workflowRunId: claimed.id, + workerId, + error: { message: "run failed" }, + retryPolicy: { + ...DEFAULT_WORKFLOW_RETRY_POLICY, + maximumAttempts: 1, + }, + }); + + const failedRun = await backend.getWorkflowRun({ + workflowRunId: claimed.id, + }); + expect(failedRun?.status).toBe("failed"); + + await backend.resumeWorkflowRun({ workflowRunId: claimed.id }); + + const attempts = await backend.listStepAttempts({ + workflowRunId: claimed.id, + limit: 100, + }); + expect(attempts.data.map((a) => a.stepName)).toEqual([ + "completed-step", + ]); + expect(attempts.data[0]?.status).toBe("completed"); + + await teardown(backend); + }); + + test("throws when resuming a run that is not failed", async () => { + const backend = await setup(); + + const created = await createPendingWorkflowRun(backend); + + await expect( + backend.resumeWorkflowRun({ workflowRunId: created.id }), + ).rejects.toThrow(/Cannot resume workflow run .* with status pending/); + + await teardown(backend); + }); + + test("throws when resuming a non-existent workflow run", async () => { + const backend = await setup(); + + const nonExistentId = randomUUID(); + + await expect( + backend.resumeWorkflowRun({ workflowRunId: nonExistentId }), + ).rejects.toThrow(`Workflow run ${nonExistentId} does not exist`); + + await teardown(backend); + }); + + test("a resumed run is claimable by workers again", async () => { + const backend = await setup(); + + await createPendingWorkflowRun(backend); + const failedId = await claimAndFailNextPendingRun(backend); + + await backend.resumeWorkflowRun({ workflowRunId: failedId }); + + const claimed = await backend.claimWorkflowRun({ + workerId: randomUUID(), + leaseDurationMs: 100, + }); + + expect(claimed?.id).toBe(failedId); + expect(claimed?.status).toBe("running"); + + await teardown(backend); + }); + }); + describe("sendSignal()", () => { test("returns empty when no active waiters", async () => { const result = await backend.sendSignal({ diff --git a/packages/openworkflow/worker/execution.test.ts b/packages/openworkflow/worker/execution.test.ts index 1fe71707..5ef85392 100644 --- a/packages/openworkflow/worker/execution.test.ts +++ b/packages/openworkflow/worker/execution.test.ts @@ -3057,6 +3057,7 @@ describe("StepExecutor", () => { }); expect(resumedRun?.status).toBe("pending"); expect(resumedRun?.error).toBeNull(); + expect(resumedRun?.startedAt).toBeNull(); expect(resumedRun?.finishedAt).toBeNull(); expect(resumedRun?.workerId).toBeNull(); @@ -3064,9 +3065,12 @@ describe("StepExecutor", () => { workflowRunId: handle.workflowRun.id, limit: 100, }); + // only successful attempts survive; failed and still-running rows are gone expect( - stepsAfterResume.data.some((s) => s.status === "failed"), - ).toBe(false); + stepsAfterResume.data.every( + (s) => s.status === "completed" || s.status === "succeeded", + ), + ).toBe(true); expect( stepsAfterResume.data.some( (s) => s.stepName === "validate" && s.status === "completed", @@ -3107,6 +3111,14 @@ describe("StepExecutor", () => { backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }), ).rejects.toThrow(/Cannot resume workflow run.*pending/); }); + + test("resumeWorkflowRun throws when the run does not exist", async () => { + const backend = await createTestBackend(); + + await expect( + backend.resumeWorkflowRun({ workflowRunId: randomUUID() }), + ).rejects.toThrow(/does not exist/); + }); }); describe("executeWorkflow", () => { From c18fa2890b90fb368efce9549fbf2af60f3a924f Mon Sep 17 00:00:00 2001 From: vudc Date: Thu, 23 Jul 2026 11:00:07 +0700 Subject: [PATCH 4/9] fix(openworkflow, dashboard): address resume review findings Correctness: - Narrow the step-attempt cleanup to failed attempts plus inert running attempts (function, signal-send). Running sleep, signal-wait and workflow attempts are now preserved: deleting them orphaned live child runs via ON DELETE SET NULL, dropped already-delivered signals, and restarted durable timers from zero. - Reset the run-level attempts counter on resume so a resumed run gets the fresh retry budget the UI promises. - Gate resume on an unexpired deadline and reject deadline-expired runs with a clear error instead of destroying their failure history for a run the next tick would immediately re-fail. Dashboard: - Extract the shared RunActionDialog so resume and cancel stop tripping the zero-tolerance jscpd duplication gate. - Use .validator instead of the removed .inputValidator, matching the sibling server functions. Tests: - Rewrite the attempt-cleanup test to assert in-flight attempts survive, and add a child-workflow test proving the linked child is not orphaned (both fail against the previous broad DELETE). - Add deadline-rejection and attempts-reset coverage. - Give the end-to-end resume test a 20s wait budget so the flaky step's 1s retry backoff fits inside it. --- .../src/components/run-action-dialog.tsx | 118 ++++++++++++++++ .../src/components/run-cancel-action.tsx | 105 +++----------- .../src/components/run-resume-action.tsx | 103 ++------------ apps/dashboard/src/lib/api.ts | 2 +- packages/openworkflow/core/workflow-run.ts | 20 ++- packages/openworkflow/postgres/backend.ts | 18 ++- packages/openworkflow/sqlite/backend.ts | 19 ++- .../openworkflow/testing/backend.testsuite.ts | 132 ++++++++++++++++-- .../openworkflow/worker/execution.test.ts | 4 +- 9 files changed, 317 insertions(+), 204 deletions(-) create mode 100644 apps/dashboard/src/components/run-action-dialog.tsx diff --git a/apps/dashboard/src/components/run-action-dialog.tsx b/apps/dashboard/src/components/run-action-dialog.tsx new file mode 100644 index 00000000..0a15be45 --- /dev/null +++ b/apps/dashboard/src/components/run-action-dialog.tsx @@ -0,0 +1,118 @@ +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import type { ReactNode } from "react"; +import { useState } from "react"; + +type ButtonVariant = React.ComponentProps["variant"]; + +interface RunActionDialogProps { + triggerLabel: string; + triggerVariant?: ButtonVariant; + title: string; + description: ReactNode; + cancelLabel: string; + confirmLabel: string; + pendingLabel: string; + confirmVariant?: ButtonVariant; + fallbackErrorMessage: string; + action: () => Promise; + onDone?: (() => Promise) | (() => void); +} + +function getErrorMessage(error: unknown, fallback: string): string { + if (error instanceof Error && error.message) { + return error.message; + } + + return fallback; +} + +// Confirmation dialog shared by the run action buttons (cancel, resume). +export function RunActionDialog({ + triggerLabel, + triggerVariant = "default", + title, + description, + cancelLabel, + confirmLabel, + pendingLabel, + confirmVariant, + fallbackErrorMessage, + action, + onDone, +}: RunActionDialogProps) { + const [isOpen, setIsOpen] = useState(false); + const [isPending, setIsPending] = useState(false); + const [error, setError] = useState(null); + + async function runAction() { + setIsPending(true); + setError(null); + + try { + await action(); + await onDone?.(); + setIsOpen(false); + } catch (caughtError) { + setError(getErrorMessage(caughtError, fallbackErrorMessage)); + } finally { + setIsPending(false); + } + } + + return ( + { + setIsOpen(nextOpen); + if (!nextOpen) { + setError(null); + } + }} + > + + + + + {title} + {description} + + + {error &&

{error}

} + + + + {cancelLabel} + + { + void runAction(); + }} + disabled={isPending} + > + {isPending ? pendingLabel : confirmLabel} + + + + + ); +} diff --git a/apps/dashboard/src/components/run-cancel-action.tsx b/apps/dashboard/src/components/run-cancel-action.tsx index 77b94021..d8dd2727 100644 --- a/apps/dashboard/src/components/run-cancel-action.tsx +++ b/apps/dashboard/src/components/run-cancel-action.tsx @@ -1,18 +1,7 @@ -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; -import { Button } from "@/components/ui/button"; +import { RunActionDialog } from "@/components/run-action-dialog"; import { cancelWorkflowRunServerFn } from "@/lib/api"; import { isRunCancelableStatus } from "@/lib/status"; import type { WorkflowRunStatus } from "openworkflow/internal"; -import { useState } from "react"; interface RunCancelActionProps { runId: string; @@ -20,92 +9,30 @@ interface RunCancelActionProps { onCanceled?: (() => Promise) | (() => void); } -function getErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) { - return error.message; - } - - return "Unable to cancel workflow run"; -} - export function RunCancelAction({ runId, status, onCanceled, }: RunCancelActionProps) { - const [isOpen, setIsOpen] = useState(false); - const [isCanceling, setIsCanceling] = useState(false); - const [error, setError] = useState(null); - if (!isRunCancelableStatus(status)) { return null; } - async function cancelRun() { - setIsCanceling(true); - setError(null); - - try { - await cancelWorkflowRunServerFn({ - data: { - workflowRunId: runId, - }, - }); - await onCanceled?.(); - setIsOpen(false); - } catch (caughtError) { - setError(getErrorMessage(caughtError)); - } finally { - setIsCanceling(false); - } - } - return ( - { - setIsOpen(nextOpen); - if (!nextOpen) { - setError(null); - } - }} - > - - - - - Cancel this run? - - This will stop any future progress for this workflow run. - - - - {error &&

{error}

} - - - - Keep Running - - { - void cancelRun(); - }} - disabled={isCanceling} - > - {isCanceling ? "Canceling..." : "Cancel Run"} - - - - + + cancelWorkflowRunServerFn({ data: { workflowRunId: runId } }) + } + onDone={onCanceled} + /> ); } diff --git a/apps/dashboard/src/components/run-resume-action.tsx b/apps/dashboard/src/components/run-resume-action.tsx index 452d4c20..5795a691 100644 --- a/apps/dashboard/src/components/run-resume-action.tsx +++ b/apps/dashboard/src/components/run-resume-action.tsx @@ -1,18 +1,7 @@ -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; -import { Button } from "@/components/ui/button"; +import { RunActionDialog } from "@/components/run-action-dialog"; import { resumeWorkflowRunServerFn } from "@/lib/api"; import { isRunResumableStatus } from "@/lib/status"; import type { WorkflowRunStatus } from "openworkflow/internal"; -import { useState } from "react"; interface RunResumeActionProps { runId: string; @@ -20,91 +9,29 @@ interface RunResumeActionProps { onResumed?: (() => Promise) | (() => void); } -function getErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) { - return error.message; - } - - return "Unable to resume workflow run"; -} - export function RunResumeAction({ runId, status, onResumed, }: RunResumeActionProps) { - const [isOpen, setIsOpen] = useState(false); - const [isResuming, setIsResuming] = useState(false); - const [error, setError] = useState(null); - if (!isRunResumableStatus(status)) { return null; } - async function resumeRun() { - setIsResuming(true); - setError(null); - - try { - await resumeWorkflowRunServerFn({ - data: { - workflowRunId: runId, - }, - }); - await onResumed?.(); - setIsOpen(false); - } catch (caughtError) { - setError(getErrorMessage(caughtError)); - } finally { - setIsResuming(false); - } - } - return ( - { - setIsOpen(nextOpen); - if (!nextOpen) { - setError(null); - } - }} - > - - - - - Resume this failed run? - - Completed steps stay cached and won't re-run. The failing step will - be retried with a fresh retry budget. Previous failed attempts will - be discarded. - - - - {error &&

{error}

} - - - Cancel - { - void resumeRun(); - }} - disabled={isResuming} - > - {isResuming ? "Resuming..." : "Resume Run"} - - - - + + resumeWorkflowRunServerFn({ data: { workflowRunId: runId } }) + } + onDone={onResumed} + /> ); } diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index e604f7ee..59dba33a 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -96,7 +96,7 @@ export const cancelWorkflowRunServerFn = createServerFn({ method: "POST" }) * budget; completed steps stay cached. */ export const resumeWorkflowRunServerFn = createServerFn({ method: "POST" }) - .inputValidator(z.object({ workflowRunId: z.string() })) + .validator(z.object({ workflowRunId: z.string() })) .handler(async ({ data }): Promise => { const backend = await getBackend(); return backend.resumeWorkflowRun({ workflowRunId: data.workflowRunId }); diff --git a/packages/openworkflow/core/workflow-run.ts b/packages/openworkflow/core/workflow-run.ts index 942c3b8a..c74ad47b 100644 --- a/packages/openworkflow/core/workflow-run.ts +++ b/packages/openworkflow/core/workflow-run.ts @@ -60,12 +60,12 @@ export function resolveCancelWorkflowRunConflict( /** * Resolve the outcome when a resumeWorkflowRun UPDATE affected no rows. The - * UPDATE is gated on `status = 'failed'`, so a zero-row outcome means either - * the run doesn't exist or it isn't in a resumable state. + * UPDATE is gated on `status = 'failed'` AND an unexpired deadline, so a + * zero-row outcome means the run doesn't exist, isn't in a resumable state, + * or has already blown its deadline (in which case resuming it is futile). * @param workflowRunId - ID of the workflow run (used in error messages) * @param existing - Current workflow run, or null if not found - * @returns Never; always throws describing why the resume is impossible - * @throws {Error} If the run does not exist or is not in `failed` status + * @throws {Error} If the run is missing, not `failed`, or past its deadline */ export function resolveResumeWorkflowRunConflict( workflowRunId: string, @@ -76,6 +76,15 @@ export function resolveResumeWorkflowRunConflict( throw new Error(`Workflow run ${workflowRunId} does not exist`); } + if (existing.status === "failed") { + // The UPDATE also gates on the deadline, so a still-`failed` run that did + // not resume is one whose deadline has already elapsed. + // eslint-disable-next-line functional/no-throw-statements + throw new Error( + `Cannot resume workflow run ${workflowRunId}; its deadline has already passed`, + ); + } + // eslint-disable-next-line functional/no-throw-statements throw new Error( `Cannot resume workflow run ${workflowRunId} with status ${existing.status}; only failed runs can be resumed`, @@ -128,7 +137,8 @@ export type SchemaOutput = TSchema extends StandardSchemaV1 * error message. */ export type ValidationResult = - { success: true; value: T } | { success: false; error: string }; + | { success: true; value: T } + | { success: false; error: string }; /** * Validate input against a Standard Schema. Pure async function that validates diff --git a/packages/openworkflow/postgres/backend.ts b/packages/openworkflow/postgres/backend.ts index 88534353..f473c5d2 100644 --- a/packages/openworkflow/postgres/backend.ts +++ b/packages/openworkflow/postgres/backend.ts @@ -778,6 +778,7 @@ export class BackendPostgres implements Backend { "status" = 'pending', "worker_id" = NULL, "error" = NULL, + "attempts" = 0, "started_at" = NULL, "finished_at" = NULL, "available_at" = NOW(), @@ -785,6 +786,7 @@ export class BackendPostgres implements Backend { WHERE "namespace_id" = ${this.namespaceId} AND "id" = ${params.workflowRunId} AND "status" = 'failed' + AND ("deadline_at" IS NULL OR "deadline_at" > NOW()) RETURNING * `; @@ -803,16 +805,20 @@ export class BackendPostgres implements Backend { ); } - // Drop every attempt that did not succeed. Failed attempts go so the - // failing step gets a fresh retry budget; still-'running' attempts - // (sleep, signal-wait, child workflow) go so replay does not mistake - // them for in-flight work. Successful attempts stay and are replayed - // from cache without re-executing. + // Drop failed attempts so the failing step gets a fresh retry budget, + // plus inert running attempts whose kinds hold no external state + // (function, signal-send). Running sleep, signal-wait and workflow + // attempts are preserved: replay resumes them, and deleting them would + // orphan linked child runs and already-delivered signals. Successful + // attempts stay and are replayed from cache. await tx` DELETE FROM ${stepAttemptsTable} WHERE "namespace_id" = ${this.namespaceId} AND "workflow_run_id" = ${params.workflowRunId} - AND "status" NOT IN ('completed', 'succeeded') + AND ( + "status" = 'failed' + OR ("status" = 'running' AND "kind" IN ('function', 'signal-send')) + ) `; return updated; diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index c7d22b3d..699c8a88 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -760,6 +760,7 @@ export class BackendSqlite implements Backend { "status" = 'pending', "worker_id" = NULL, "error" = NULL, + "attempts" = 0, "started_at" = NULL, "finished_at" = NULL, "available_at" = ?, @@ -767,6 +768,7 @@ export class BackendSqlite implements Backend { WHERE "namespace_id" = ? AND "id" = ? AND "status" = 'failed' + AND ("deadline_at" IS NULL OR "deadline_at" > ?) `); const updateResult = updateStmt.run( @@ -774,23 +776,28 @@ export class BackendSqlite implements Backend { currentTime, this.namespaceId, params.workflowRunId, + currentTime, ); resumed = updateResult.changes > 0; if (resumed) { - // Drop every attempt that did not succeed. Failed attempts go so the - // failing step gets a fresh retry budget; still-'running' attempts - // (sleep, signal-wait, child workflow) go so replay does not mistake - // them for in-flight work. Successful attempts stay and are replayed - // from cache without re-executing. + // Drop failed attempts so the failing step gets a fresh retry budget, + // plus inert running attempts whose kinds hold no external state + // (function, signal-send). Running sleep, signal-wait and workflow + // attempts are preserved: replay resumes them, and deleting them + // would orphan linked child runs and already-delivered signals. + // Successful attempts stay and are replayed from cache. this.db .prepare( ` DELETE FROM "step_attempts" WHERE "namespace_id" = ? AND "workflow_run_id" = ? - AND "status" NOT IN ('completed', 'succeeded') + AND ( + "status" = 'failed' + OR ("status" = 'running' AND "kind" IN ('function', 'signal-send')) + ) `, ) .run(this.namespaceId, params.workflowRunId); diff --git a/packages/openworkflow/testing/backend.testsuite.ts b/packages/openworkflow/testing/backend.testsuite.ts index be12cbd7..f8749568 100644 --- a/packages/openworkflow/testing/backend.testsuite.ts +++ b/packages/openworkflow/testing/backend.testsuite.ts @@ -2589,13 +2589,15 @@ export function testBackend(options: TestBackendOptions): void { expect(resumed.workerId).toBeNull(); expect(resumed.startedAt).toBeNull(); expect(resumed.finishedAt).toBeNull(); + // run-level retry budget is reset so the resumed run gets fresh retries + expect(resumed.attempts).toBe(0); expect(resumed.availableAt).not.toBeNull(); expect(deltaSeconds(resumed.availableAt)).toBeLessThan(1); await teardown(backend); }); - test("drops every unsuccessful step attempt but keeps successful ones", async () => { + test("keeps successful and in-flight attempts, drops failed and inert running ones", async () => { const backend = await setup(); const claimed = await createClaimedWorkflowRun(backend); @@ -2631,12 +2633,23 @@ export function testBackend(options: TestBackendOptions): void { error: { message: "boom" }, }); - // left in 'running': mirrors a sleep/signal-wait/child-workflow - // attempt that never reached a terminal state before the run failed + // Inert running function attempt (e.g. a worker that died mid-step): + // safe to drop because nothing references it. await backend.createStepAttempt({ workflowRunId: claimed.id, workerId, - stepName: "running-step", + stepName: "running-fn", + kind: "function", + config: {}, + context: null, + }); + + // In-flight durable wait: must be preserved so replay resumes it + // instead of restarting the timer from zero. + await backend.createStepAttempt({ + workflowRunId: claimed.id, + workerId, + stepName: "running-sleep", kind: "sleep", config: {}, context: { @@ -2666,10 +2679,113 @@ export function testBackend(options: TestBackendOptions): void { workflowRunId: claimed.id, limit: 100, }); - expect(attempts.data.map((a) => a.stepName)).toEqual([ - "completed-step", - ]); - expect(attempts.data[0]?.status).toBe("completed"); + const survivingNames = attempts.data.map((a) => a.stepName); + expect(survivingNames).toHaveLength(2); + expect(survivingNames).toContain("completed-step"); + expect(survivingNames).toContain("running-sleep"); + + await teardown(backend); + }); + + test("preserves an in-flight child-workflow attempt so the child is not orphaned", async () => { + const backend = await setup(); + + const parent = await createClaimedWorkflowRun(backend); + const workerId = parent.workerId!; // eslint-disable-line @typescript-eslint/no-non-null-assertion + + // Running kind='workflow' attempt with a child run linked back to it. + const workflowAttempt = await backend.createStepAttempt({ + workflowRunId: parent.id, + workerId, + stepName: "invoke-child", + kind: "workflow", + config: {}, + context: { kind: "workflow", timeoutAt: null }, + }); + + const child = await backend.createWorkflowRun({ + workflowName: randomUUID(), + version: null, + idempotencyKey: null, + input: null, + config: {}, + context: null, + parentStepAttemptNamespaceId: workflowAttempt.namespaceId, + parentStepAttemptId: workflowAttempt.id, + availableAt: null, + deadlineAt: null, + }); + expect(child.parentStepAttemptId).toBe(workflowAttempt.id); + + await backend.failWorkflowRun({ + workflowRunId: parent.id, + workerId, + error: { message: "sibling failed" }, + retryPolicy: { + ...DEFAULT_WORKFLOW_RETRY_POLICY, + maximumAttempts: 1, + }, + }); + + await backend.resumeWorkflowRun({ workflowRunId: parent.id }); + + // The running workflow attempt must survive the resume... + const attempts = await backend.listStepAttempts({ + workflowRunId: parent.id, + limit: 100, + }); + expect(attempts.data.some((a) => a.id === workflowAttempt.id)).toBe( + true, + ); + + // ...so the child's parent pointer is not nulled by ON DELETE SET NULL. + const childAfter = await backend.getWorkflowRun({ + workflowRunId: child.id, + }); + expect(childAfter?.parentStepAttemptId).toBe(workflowAttempt.id); + + await teardown(backend); + }); + + test("throws and preserves history when the deadline has passed", async () => { + const backend = await setup(); + + const created = await backend.createWorkflowRun({ + workflowName: randomUUID(), + version: null, + idempotencyKey: null, + input: null, + config: {}, + context: null, + parentStepAttemptNamespaceId: null, + parentStepAttemptId: null, + availableAt: null, + deadlineAt: new Date(Date.now() - 1000), + }); + + // Claiming triggers the deadline sweep, which flips the run to failed; + // the run itself is then excluded from the claim, so this returns null. + await backend.claimWorkflowRun({ + workerId: randomUUID(), + leaseDurationMs: 100, + }); + + const failedRun = await backend.getWorkflowRun({ + workflowRunId: created.id, + }); + expect(failedRun?.status).toBe("failed"); + expect(failedRun?.error).not.toBeNull(); + + await expect( + backend.resumeWorkflowRun({ workflowRunId: created.id }), + ).rejects.toThrow(/deadline has already passed/); + + // Resume must not have destroyed the run's failure diagnostics. + const afterResume = await backend.getWorkflowRun({ + workflowRunId: created.id, + }); + expect(afterResume?.status).toBe("failed"); + expect(afterResume?.error).not.toBeNull(); await teardown(backend); }); diff --git a/packages/openworkflow/worker/execution.test.ts b/packages/openworkflow/worker/execution.test.ts index 5ef85392..78048224 100644 --- a/packages/openworkflow/worker/execution.test.ts +++ b/packages/openworkflow/worker/execution.test.ts @@ -3035,6 +3035,7 @@ describe("StepExecutor", () => { handle.workflowRun.id, 40, 25, + { maxWaitMs: 20_000 }, ); expect(failedStatus).toBe("failed"); expect(validateRuns).toBe(1); @@ -3083,6 +3084,7 @@ describe("StepExecutor", () => { handle.workflowRun.id, 40, 25, + { maxWaitMs: 20_000 }, ); expect(finalStatus).toBe("completed"); // "validate" was cached from the original run, not re-executed @@ -3092,7 +3094,7 @@ describe("StepExecutor", () => { const result = await handle.result(); expect(result).toEqual({ validated: "ok", flaky: "recovered" }); - }); + }, 30_000); test("resumeWorkflowRun throws when the run is not in failed status", async () => { const backend = await createTestBackend(); From e69a02f75d6022a1a176ddb6ab97b4cc2afba36c Mon Sep 17 00:00:00 2001 From: vudc Date: Mon, 27 Jul 2026 01:40:07 +0700 Subject: [PATCH 5/9] style(openworkflow): match prettier 3.9.6 union formatting An earlier local prettier 3.8.3 pass expanded short union types that the pinned 3.9.6 keeps on one line, breaking the format check in CI. Reformat with the pinned version. --- packages/openworkflow/core/workflow-run.ts | 3 +-- packages/openworkflow/sqlite/backend.ts | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/openworkflow/core/workflow-run.ts b/packages/openworkflow/core/workflow-run.ts index c74ad47b..7575ab59 100644 --- a/packages/openworkflow/core/workflow-run.ts +++ b/packages/openworkflow/core/workflow-run.ts @@ -137,8 +137,7 @@ export type SchemaOutput = TSchema extends StandardSchemaV1 * error message. */ export type ValidationResult = - | { success: true; value: T } - | { success: false; error: string }; + { success: true; value: T } | { success: false; error: string }; /** * Validate input against a Standard Schema. Pure async function that validates diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index 699c8a88..7162a21a 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -275,8 +275,7 @@ export class BackendSqlite implements Backend { `); const row = stmt.get(this.namespaceId, params.workflowRunId) as - | WorkflowRunRow - | undefined; + WorkflowRunRow | undefined; return Promise.resolve(row ? rowToWorkflowRun(row) : null); } @@ -395,8 +394,7 @@ export class BackendSqlite implements Backend { LIMIT 1 `); const row = stmt.get(this.namespaceId, params.stepAttemptId) as - | { data: string | null } - | undefined; + { data: string | null } | undefined; if (!row) return Promise.resolve(undefined); return Promise.resolve( @@ -1092,8 +1090,7 @@ export class BackendSqlite implements Backend { `); const row = stmt.get(this.namespaceId, params.stepAttemptId) as - | StepAttemptRow - | undefined; + StepAttemptRow | undefined; return Promise.resolve(row ? rowToStepAttempt(row) : null); } From e7fe4c7af1c110a05c253c83655c9f3402f55a48 Mon Sep 17 00:00:00 2001 From: vudc Date: Thu, 30 Jul 2026 00:37:07 +0700 Subject: [PATCH 6/9] refactor(openworkflow, dashboard): resume by marking, not deleting Deleting failed step attempts and clearing the run error on resume erased the record of why a run failed and broke parent/child linkage (children are linked through their parent's step attempt). Resume now preserves all history and resets the retry budget with a marker instead. - Add nullable `resumed_at` to workflow_runs (migration 6, both backends) and `WorkflowRun.resumedAt`. - resumeWorkflowRun stamps `resumed_at` and requeues. It no longer deletes step attempts, clears `error`, or resets `attempts`. The deadline gate and conflict resolver are unchanged. - Step replay counts a failed attempt toward the step retry budget only when it finished at/after `resumed_at`, so the failing step retries fresh while every prior attempt stays as history. Wired through StepHistory and createStepExecutionStateFromAttempts. - Update the resume dialog copy and client/server JSDoc to match. Tests: - Step-history unit tests for the marker filter. - Shared backend testsuite asserts resume preserves every attempt plus the error and stamps resumed_at; deadline rejection and child-workflow preservation covered. - End-to-end tests: the failed attempts are kept, and a mid-workflow failure resumes and continues to the downstream steps. - Populate resumedAt in the WorkflowRun test mocks. --- .../src/components/run-resume-action.tsx | 2 +- apps/dashboard/src/lib/api.ts | 4 +- packages/openworkflow/client/client.test.ts | 1 + packages/openworkflow/client/client.ts | 5 +- .../openworkflow/core/workflow-run.test.ts | 1 + packages/openworkflow/core/workflow-run.ts | 1 + packages/openworkflow/postgres/backend.ts | 80 ++++++---------- packages/openworkflow/postgres/postgres.ts | 12 +++ packages/openworkflow/sqlite/backend.ts | 55 +++-------- packages/openworkflow/sqlite/sqlite.ts | 10 ++ .../openworkflow/testing/backend.testsuite.ts | 45 +++++---- .../openworkflow/worker/execution.test.ts | 93 +++++++++++++++++-- packages/openworkflow/worker/execution.ts | 5 +- .../openworkflow/worker/step-history.test.ts | 39 ++++++++ packages/openworkflow/worker/step-history.ts | 17 +++- 15 files changed, 245 insertions(+), 125 deletions(-) diff --git a/apps/dashboard/src/components/run-resume-action.tsx b/apps/dashboard/src/components/run-resume-action.tsx index 5795a691..f3f5f3a0 100644 --- a/apps/dashboard/src/components/run-resume-action.tsx +++ b/apps/dashboard/src/components/run-resume-action.tsx @@ -23,7 +23,7 @@ export function RunResumeAction({ triggerLabel="Resume Run" triggerVariant="default" title="Resume this failed run?" - description="Completed steps stay cached and won't re-run. The failing step will be retried with a fresh retry budget. Previous failed attempts will be discarded." + description="Completed steps stay cached and won't re-run. The failing step is retried with a fresh retry budget, and the run's history is kept." cancelLabel="Cancel" confirmLabel="Resume Run" pendingLabel="Resuming..." diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index 59dba33a..c1e38de8 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -92,8 +92,8 @@ export const cancelWorkflowRunServerFn = createServerFn({ method: "POST" }) /** * Resume a failed workflow run by ID. Flips the run back to `pending` and - * drops failed step attempts so the failing step starts with a fresh retry - * budget; completed steps stay cached. + * gives the failing step a fresh retry budget by only counting failures after + * the resume; completed steps stay cached and history is preserved. */ export const resumeWorkflowRunServerFn = createServerFn({ method: "POST" }) .validator(z.object({ workflowRunId: z.string() })) diff --git a/packages/openworkflow/client/client.test.ts b/packages/openworkflow/client/client.test.ts index 80bc7bb1..5ceb2b21 100644 --- a/packages/openworkflow/client/client.test.ts +++ b/packages/openworkflow/client/client.test.ts @@ -698,6 +698,7 @@ function createMockWorkflowRun( deadlineAt: null, startedAt: null, finishedAt: null, + resumedAt: null, createdAt: currentTime, updatedAt: currentTime, ...overrides, diff --git a/packages/openworkflow/client/client.ts b/packages/openworkflow/client/client.ts index 411f4814..c849c150 100644 --- a/packages/openworkflow/client/client.ts +++ b/packages/openworkflow/client/client.ts @@ -178,8 +178,9 @@ export class OpenWorkflow { /** * Resume a failed workflow run. The run's status flips back to `pending` * so the next worker tick picks it up. Already-completed steps are served - * from history without re-executing; failed step attempts are discarded so - * the failing step starts with a fresh retry budget. + * from history without re-executing. Nothing is deleted: the failing step + * starts with a fresh retry budget because only failures recorded after the + * resume count against it. * @param workflowRunId - The ID of the failed workflow run to resume * @returns The updated workflow run * @throws {Error} If the run does not exist or is not in `failed` status diff --git a/packages/openworkflow/core/workflow-run.test.ts b/packages/openworkflow/core/workflow-run.test.ts index 3fa5da9a..9f980009 100644 --- a/packages/openworkflow/core/workflow-run.test.ts +++ b/packages/openworkflow/core/workflow-run.test.ts @@ -169,6 +169,7 @@ describe("resolveCancelWorkflowRunConflict", () => { deadlineAt: null, startedAt: null, finishedAt: null, + resumedAt: null, createdAt: new Date(0), updatedAt: new Date(0), }; diff --git a/packages/openworkflow/core/workflow-run.ts b/packages/openworkflow/core/workflow-run.ts index 7575ab59..dbbeae5d 100644 --- a/packages/openworkflow/core/workflow-run.ts +++ b/packages/openworkflow/core/workflow-run.ts @@ -114,6 +114,7 @@ export interface WorkflowRun { deadlineAt: Date | null; startedAt: Date | null; finishedAt: Date | null; + resumedAt: Date | null; // Timestamp of the most recent resume createdAt: Date; updatedAt: Date; } diff --git a/packages/openworkflow/postgres/backend.ts b/packages/openworkflow/postgres/backend.ts index f473c5d2..d3ee31a5 100644 --- a/packages/openworkflow/postgres/backend.ts +++ b/packages/openworkflow/postgres/backend.ts @@ -768,61 +768,37 @@ export class BackendPostgres implements Backend { async resumeWorkflowRun( params: ResumeWorkflowRunParams, ): Promise { - return await this.withTransaction(async (tx): Promise => { - const workflowRunsTable = this.workflowRunsTable(tx); - const stepAttemptsTable = this.stepAttemptsTable(tx); - - const [updated] = await tx` - UPDATE ${workflowRunsTable} - SET - "status" = 'pending', - "worker_id" = NULL, - "error" = NULL, - "attempts" = 0, - "started_at" = NULL, - "finished_at" = NULL, - "available_at" = NOW(), - "updated_at" = NOW() - WHERE "namespace_id" = ${this.namespaceId} - AND "id" = ${params.workflowRunId} - AND "status" = 'failed' - AND ("deadline_at" IS NULL OR "deadline_at" > NOW()) - RETURNING * - `; - - if (!updated) { - const [existing] = await tx` - SELECT * - FROM ${workflowRunsTable} - WHERE "namespace_id" = ${this.namespaceId} - AND "id" = ${params.workflowRunId} - LIMIT 1 - `; + const workflowRunsTable = this.workflowRunsTable(); - resolveResumeWorkflowRunConflict( - params.workflowRunId, - existing ?? null, - ); - } + // Stamp the resume marker and requeue. Nothing is deleted and neither + // `error` nor `attempts` is touched: step attempts stay (preserving the + // failure record and parent/child linkage), and the retry budget is reset + // by only counting failures after `resumed_at` during replay. + const [updated] = await this.pg` + UPDATE ${workflowRunsTable} + SET + "status" = 'pending', + "worker_id" = NULL, + "started_at" = NULL, + "finished_at" = NULL, + "available_at" = NOW(), + "resumed_at" = NOW(), + "updated_at" = NOW() + WHERE "namespace_id" = ${this.namespaceId} + AND "id" = ${params.workflowRunId} + AND "status" = 'failed' + AND ("deadline_at" IS NULL OR "deadline_at" > NOW()) + RETURNING * + `; - // Drop failed attempts so the failing step gets a fresh retry budget, - // plus inert running attempts whose kinds hold no external state - // (function, signal-send). Running sleep, signal-wait and workflow - // attempts are preserved: replay resumes them, and deleting them would - // orphan linked child runs and already-delivered signals. Successful - // attempts stay and are replayed from cache. - await tx` - DELETE FROM ${stepAttemptsTable} - WHERE "namespace_id" = ${this.namespaceId} - AND "workflow_run_id" = ${params.workflowRunId} - AND ( - "status" = 'failed' - OR ("status" = 'running' AND "kind" IN ('function', 'signal-send')) - ) - `; + if (!updated) { + const existing = await this.getWorkflowRun({ + workflowRunId: params.workflowRunId, + }); + resolveResumeWorkflowRunConflict(params.workflowRunId, existing); + } - return updated; - }); + return updated; } private async wakeParentWorkflowRun( diff --git a/packages/openworkflow/postgres/postgres.ts b/packages/openworkflow/postgres/postgres.ts index f52f6523..92960758 100644 --- a/packages/openworkflow/postgres/postgres.ts +++ b/packages/openworkflow/postgres/postgres.ts @@ -240,6 +240,18 @@ export function migrations(schema: string): string[] { ON CONFLICT DO NOTHING; COMMIT;`, + + // 6 - resume marker + `BEGIN; + + ALTER TABLE ${quotedSchema}."workflow_runs" + ADD COLUMN IF NOT EXISTS "resumed_at" TIMESTAMPTZ; + + INSERT INTO ${quotedSchema}."openworkflow_migrations"("version") + VALUES (6) + ON CONFLICT DO NOTHING; + + COMMIT;`, ]; } diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index 7162a21a..7e76e523 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -748,28 +748,31 @@ export class BackendSqlite implements Backend { params: ResumeWorkflowRunParams, ): Promise { const currentTime = now(); - let resumed = false; - this.db.exec("BEGIN IMMEDIATE"); - try { - const updateStmt = this.db.prepare(` + // Stamp the resume marker and requeue. Nothing is deleted and neither + // `error` nor `attempts` is touched: step attempts stay (preserving the + // failure record and parent/child linkage), and the retry budget is reset + // by only counting failures after `resumed_at` during replay. + const updateResult = this.db + .prepare( + ` UPDATE "workflow_runs" SET "status" = 'pending', "worker_id" = NULL, - "error" = NULL, - "attempts" = 0, "started_at" = NULL, "finished_at" = NULL, "available_at" = ?, + "resumed_at" = ?, "updated_at" = ? WHERE "namespace_id" = ? AND "id" = ? AND "status" = 'failed' AND ("deadline_at" IS NULL OR "deadline_at" > ?) - `); - - const updateResult = updateStmt.run( + `, + ) + .run( + currentTime, currentTime, currentTime, this.namespaceId, @@ -777,41 +780,11 @@ export class BackendSqlite implements Backend { currentTime, ); - resumed = updateResult.changes > 0; - - if (resumed) { - // Drop failed attempts so the failing step gets a fresh retry budget, - // plus inert running attempts whose kinds hold no external state - // (function, signal-send). Running sleep, signal-wait and workflow - // attempts are preserved: replay resumes them, and deleting them - // would orphan linked child runs and already-delivered signals. - // Successful attempts stay and are replayed from cache. - this.db - .prepare( - ` - DELETE FROM "step_attempts" - WHERE "namespace_id" = ? - AND "workflow_run_id" = ? - AND ( - "status" = 'failed' - OR ("status" = 'running' AND "kind" IN ('function', 'signal-send')) - ) - `, - ) - .run(this.namespaceId, params.workflowRunId); - } - - this.db.exec("COMMIT"); - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; - } - const updated = await this.getWorkflowRun({ workflowRunId: params.workflowRunId, }); - if (!resumed) { + if (updateResult.changes === 0) { resolveResumeWorkflowRunConflict(params.workflowRunId, updated); } @@ -1185,6 +1158,7 @@ interface WorkflowRunRow { deadline_at: string | null; started_at: string | null; finished_at: string | null; + resumed_at: string | null; created_at: string; updated_at: string; } @@ -1259,6 +1233,7 @@ function rowToWorkflowRun(row: WorkflowRunRow): WorkflowRun { deadlineAt: fromISO(row.deadline_at), startedAt: fromISO(row.started_at), finishedAt: fromISO(row.finished_at), + resumedAt: fromISO(row.resumed_at), createdAt, updatedAt, }; diff --git a/packages/openworkflow/sqlite/sqlite.ts b/packages/openworkflow/sqlite/sqlite.ts index 8d11736f..71d96f89 100644 --- a/packages/openworkflow/sqlite/sqlite.ts +++ b/packages/openworkflow/sqlite/sqlite.ts @@ -224,6 +224,16 @@ export function migrations(): string[] { VALUES (5); COMMIT;`, + + // 6 - resume marker + `BEGIN; + + ALTER TABLE "workflow_runs" ADD COLUMN "resumed_at" TEXT; + + INSERT OR IGNORE INTO "openworkflow_migrations" ("version") + VALUES (6); + + COMMIT;`, ]; } diff --git a/packages/openworkflow/testing/backend.testsuite.ts b/packages/openworkflow/testing/backend.testsuite.ts index f8749568..27f6ffc7 100644 --- a/packages/openworkflow/testing/backend.testsuite.ts +++ b/packages/openworkflow/testing/backend.testsuite.ts @@ -66,6 +66,7 @@ export function testBackend(options: TestBackendOptions): void { deadlineAt: newDateInOneYear(), startedAt: null, finishedAt: null, + resumedAt: null, createdAt: new Date(), // - updatedAt: new Date(), // - }; @@ -2574,30 +2575,39 @@ export function testBackend(options: TestBackendOptions): void { }); describe("resumeWorkflowRun()", () => { - test("flips a failed run back to pending and clears failure fields", async () => { + test("requeues a failed run and stamps resumed_at without erasing history", async () => { const backend = await setup(); await createPendingWorkflowRun(backend); const failedId = await claimAndFailNextPendingRun(backend); + const failedRun = await backend.getWorkflowRun({ + workflowRunId: failedId, + }); + expect(failedRun?.error).not.toBeNull(); + const resumed = await backend.resumeWorkflowRun({ workflowRunId: failedId, }); expect(resumed.status).toBe("pending"); - expect(resumed.error).toBeNull(); expect(resumed.workerId).toBeNull(); expect(resumed.startedAt).toBeNull(); expect(resumed.finishedAt).toBeNull(); - // run-level retry budget is reset so the resumed run gets fresh retries - expect(resumed.attempts).toBe(0); expect(resumed.availableAt).not.toBeNull(); expect(deltaSeconds(resumed.availableAt)).toBeLessThan(1); + // resume marker is stamped; the budget is reset by counting failures + // after it, not by mutating the run + expect(resumed.resumedAt).not.toBeNull(); + expect(deltaSeconds(resumed.resumedAt)).toBeLessThan(1); + // error and the claim counter are left as the record of what happened + expect(resumed.error).not.toBeNull(); + expect(resumed.attempts).toBe(failedRun?.attempts); await teardown(backend); }); - test("keeps successful and in-flight attempts, drops failed and inert running ones", async () => { + test("preserves every step attempt and the run error on resume", async () => { const backend = await setup(); const claimed = await createClaimedWorkflowRun(backend); @@ -2633,19 +2643,8 @@ export function testBackend(options: TestBackendOptions): void { error: { message: "boom" }, }); - // Inert running function attempt (e.g. a worker that died mid-step): - // safe to drop because nothing references it. - await backend.createStepAttempt({ - workflowRunId: claimed.id, - workerId, - stepName: "running-fn", - kind: "function", - config: {}, - context: null, - }); - - // In-flight durable wait: must be preserved so replay resumes it - // instead of restarting the timer from zero. + // In-flight durable wait: preserved so replay resumes it rather than + // restarting the timer from zero. await backend.createStepAttempt({ workflowRunId: claimed.id, workerId, @@ -2673,15 +2672,21 @@ export function testBackend(options: TestBackendOptions): void { }); expect(failedRun?.status).toBe("failed"); - await backend.resumeWorkflowRun({ workflowRunId: claimed.id }); + const resumed = await backend.resumeWorkflowRun({ + workflowRunId: claimed.id, + }); + // the failure record survives the resume + expect(resumed.error).not.toBeNull(); + // nothing is deleted: completed, failed and running attempts all remain const attempts = await backend.listStepAttempts({ workflowRunId: claimed.id, limit: 100, }); const survivingNames = attempts.data.map((a) => a.stepName); - expect(survivingNames).toHaveLength(2); + expect(survivingNames).toHaveLength(3); expect(survivingNames).toContain("completed-step"); + expect(survivingNames).toContain("failed-step"); expect(survivingNames).toContain("running-sleep"); await teardown(backend); diff --git a/packages/openworkflow/worker/execution.test.ts b/packages/openworkflow/worker/execution.test.ts index 78048224..acc118a3 100644 --- a/packages/openworkflow/worker/execution.test.ts +++ b/packages/openworkflow/worker/execution.test.ts @@ -3048,7 +3048,7 @@ describe("StepExecutor", () => { const failedBefore = stepsBeforeResume.data.filter( (s) => s.status === "failed", ); - expect(failedBefore.length).toBe(2); + expect(failedBefore).toHaveLength(2); shouldFail = false; await backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }); @@ -3057,21 +3057,23 @@ describe("StepExecutor", () => { workflowRunId: handle.workflowRun.id, }); expect(resumedRun?.status).toBe("pending"); - expect(resumedRun?.error).toBeNull(); expect(resumedRun?.startedAt).toBeNull(); expect(resumedRun?.finishedAt).toBeNull(); expect(resumedRun?.workerId).toBeNull(); + // resume marker is stamped; the failure record is left intact + expect(resumedRun?.resumedAt).not.toBeNull(); + expect(resumedRun?.error).not.toBeNull(); const stepsAfterResume = await backend.listStepAttempts({ workflowRunId: handle.workflowRun.id, limit: 100, }); - // only successful attempts survive; failed and still-running rows are gone + // history is preserved: the two failed "flaky" attempts still exist expect( - stepsAfterResume.data.every( - (s) => s.status === "completed" || s.status === "succeeded", + stepsAfterResume.data.filter( + (s) => s.stepName === "flaky" && s.status === "failed", ), - ).toBe(true); + ).toHaveLength(2); expect( stepsAfterResume.data.some( (s) => s.stepName === "validate" && s.status === "completed", @@ -3096,6 +3098,84 @@ describe("StepExecutor", () => { expect(result).toEqual({ validated: "ok", flaky: "recovered" }); }, 30_000); + test("resumeWorkflowRun continues past the fixed step to downstream steps", async () => { + const backend = await createTestBackend(); + const client = new OpenWorkflow({ backend }); + + const ran: string[] = []; + let gatewayDown = true; + + const workflow = client.defineWorkflow( + { name: `resume-middle-step-${randomUUID()}` }, + async ({ step }) => { + await step.run({ name: "validate" }, () => { + ran.push("validate"); + return "ok"; + }); + + await step.run( + { name: "reserve", retryPolicy: { maximumAttempts: 2 } }, + () => { + ran.push("reserve"); + if (gatewayDown) { + throw new Error("gateway down"); + } + return "auth"; + }, + ); + + await step.run({ name: "confirm" }, () => { + ran.push("confirm"); + return "receipt"; + }); + + await step.run({ name: "send-receipt" }, () => { + ran.push("send-receipt"); + }); + + return "done"; + }, + ); + + const worker = client.newWorker({ concurrency: 1 }); + const handle = await workflow.run(); + + const failedStatus = await tickUntilTerminal( + backend, + worker, + handle.workflowRun.id, + 40, + 25, + { maxWaitMs: 20_000 }, + ); + expect(failedStatus).toBe("failed"); + // validate completed once; reserve exhausted its 2 attempts; the steps + // after the failing one never ran + expect(ran.filter((s) => s === "validate")).toHaveLength(1); + expect(ran.filter((s) => s === "reserve")).toHaveLength(2); + expect(ran).not.toContain("confirm"); + expect(ran).not.toContain("send-receipt"); + + gatewayDown = false; + await backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }); + + const finalStatus = await tickUntilTerminal( + backend, + worker, + handle.workflowRun.id, + 40, + 25, + { maxWaitMs: 20_000 }, + ); + expect(finalStatus).toBe("completed"); + // validate stayed cached (not re-run); reserve retried once more with a + // fresh budget and succeeded; the downstream steps now run + expect(ran.filter((s) => s === "validate")).toHaveLength(1); + expect(ran.filter((s) => s === "reserve")).toHaveLength(3); + expect(ran).toContain("confirm"); + expect(ran).toContain("send-receipt"); + }, 30_000); + test("resumeWorkflowRun throws when the run is not in failed status", async () => { const backend = await createTestBackend(); const client = new OpenWorkflow({ backend }); @@ -4194,6 +4274,7 @@ function createMockWorkflowRun( deadlineAt: null, startedAt: new Date("2026-01-01T00:00:00.000Z"), finishedAt: null, + resumedAt: null, createdAt: new Date("2026-01-01T00:00:00.000Z"), updatedAt: new Date("2026-01-01T00:00:00.000Z"), ...overrides, diff --git a/packages/openworkflow/worker/execution.ts b/packages/openworkflow/worker/execution.ts index b3031b02..35917254 100644 --- a/packages/openworkflow/worker/execution.ts +++ b/packages/openworkflow/worker/execution.ts @@ -1038,7 +1038,10 @@ export async function executeWorkflow( backend, workflowRun.id, ); - const history = new StepHistory({ attempts }); + const history = new StepHistory({ + attempts, + resumedAt: workflowRun.resumedAt, + }); // Complete any elapsed sleep waits first, then park on the earliest // remaining running wait (sleep or runWorkflow timeout). diff --git a/packages/openworkflow/worker/step-history.test.ts b/packages/openworkflow/worker/step-history.test.ts index a218c6f3..7be501e5 100644 --- a/packages/openworkflow/worker/step-history.test.ts +++ b/packages/openworkflow/worker/step-history.test.ts @@ -151,6 +151,45 @@ describe("StepHistory", () => { expect(history.findRunning("a")).toBeUndefined(); }); + test("failures before resumedAt are excluded from the retry budget", () => { + const resumedAt = new Date("2026-01-01T12:00:00.000Z"); + const beforeResume = createMockStepAttempt({ + stepName: "a", + status: "failed", + finishedAt: new Date("2026-01-01T11:59:59.000Z"), + }); + const afterResume = createMockStepAttempt({ + stepName: "a", + status: "failed", + finishedAt: new Date("2026-01-01T12:00:01.000Z"), + }); + + const history = new StepHistory({ + attempts: [beforeResume, afterResume], + resumedAt, + }); + + // only the post-resume failure counts, but both rows remain history + expect(history.failedAttemptCount("a")).toBe(1); + }); + + test("without resumedAt every failure counts", () => { + const first = createMockStepAttempt({ + stepName: "a", + status: "failed", + finishedAt: new Date("2026-01-01T11:00:00.000Z"), + }); + const second = createMockStepAttempt({ + stepName: "a", + status: "failed", + finishedAt: new Date("2026-01-01T12:00:00.000Z"), + }); + + const history = new StepHistory({ attempts: [first, second] }); + + expect(history.failedAttemptCount("a")).toBe(2); + }); + test("replaceRunningAttempt updates the running entry in place", () => { const initial = createMockStepAttempt({ id: "attempt-1", diff --git a/packages/openworkflow/worker/step-history.ts b/packages/openworkflow/worker/step-history.ts index 39532890..24999c99 100644 --- a/packages/openworkflow/worker/step-history.ts +++ b/packages/openworkflow/worker/step-history.ts @@ -41,10 +41,13 @@ export interface StepExecutionState { /** * Build step execution state from loaded attempts in one pass. * @param attempts - Loaded step attempts for the workflow run + * @param resumedAt - Most recent resume timestamp; failures that finished + * before it are kept as history but not counted against the retry budget * @returns Successful cache plus failed-attempt counts by step name */ export function createStepExecutionStateFromAttempts( attempts: readonly StepAttempt[], + resumedAt: Readonly | null = null, ): StepExecutionState { const cache = new Map(); const failedCountsByStepName = new Map(); @@ -58,6 +61,14 @@ export function createStepExecutionStateFromAttempts( } if (attempt.status === "failed") { + // Failures from before the latest resume stay in history (linkage, + // diagnostics) but don't count toward the step's retry budget. + if ( + resumedAt !== null && + (attempt.finishedAt === null || attempt.finishedAt < resumedAt) + ) { + continue; + } const previousCount = failedCountsByStepName.get(attempt.stepName) ?? 0; failedCountsByStepName.set(attempt.stepName, previousCount + 1); failedByStepName.set(attempt.stepName, attempt); @@ -169,6 +180,7 @@ function getEarliestRunningWaitResumeAt( export interface StepHistoryOptions { attempts: readonly StepAttempt[]; stepLimit?: number; + resumedAt?: Readonly | null; } /** @@ -192,7 +204,10 @@ export class StepHistory { this.stepLimit = Math.max(1, options.stepLimit ?? WORKFLOW_STEP_LIMIT); this.stepCount = options.attempts.length; - const state = createStepExecutionStateFromAttempts(options.attempts); + const state = createStepExecutionStateFromAttempts( + options.attempts, + options.resumedAt ?? null, + ); this.cache = state.cache; this.failedCountsByStepName = new Map(state.failedCountsByStepName); this.failedByStepName = new Map(state.failedByStepName); From ccff9f06d37c30879b5e48516609530f9b7fd38c Mon Sep 17 00:00:00 2001 From: James Martinez Date: Sun, 20 Sep 2026 15:03:18 -0500 Subject: [PATCH 7/9] feat(openworkflow): rerun finished workflows from saved steps --- .../src/components/run-action-dialog.tsx | 119 ----- apps/dashboard/src/components/run-action.tsx | 134 +++++ .../src/components/run-cancel-action.tsx | 38 -- .../src/components/run-resume-action.tsx | 37 -- apps/dashboard/src/lib/api.ts | 20 +- apps/dashboard/src/lib/status.ts | 7 - apps/dashboard/src/routes/runs/$runId.tsx | 24 +- apps/docs/docs/retries.mdx | 18 + cspell.config.jsonc | 3 +- oxlint.config.ts | 2 +- packages/openworkflow/client/client.test.ts | 1 - packages/openworkflow/client/client.ts | 30 +- packages/openworkflow/client/rerun.test.ts | 487 ++++++++++++++++++ packages/openworkflow/client/rerun.ts | 37 ++ packages/openworkflow/core/backend.ts | 16 +- packages/openworkflow/core/rerun.ts | 42 ++ .../openworkflow/core/workflow-run.test.ts | 1 - packages/openworkflow/core/workflow-run.ts | 31 -- packages/openworkflow/internal.ts | 2 + packages/openworkflow/postgres/backend.ts | 89 ++-- packages/openworkflow/postgres/postgres.ts | 12 - packages/openworkflow/sqlite/backend.ts | 134 +++-- packages/openworkflow/sqlite/sqlite.ts | 10 - packages/openworkflow/telemetry.test.ts | 70 +++ .../testing/backend-stub.testsuite.ts | 2 +- .../openworkflow/testing/backend.testsuite.ts | 266 ---------- .../openworkflow/worker/execution.test.ts | 208 -------- packages/openworkflow/worker/execution.ts | 5 +- .../openworkflow/worker/step-history.test.ts | 39 -- packages/openworkflow/worker/step-history.ts | 17 +- 30 files changed, 978 insertions(+), 923 deletions(-) delete mode 100644 apps/dashboard/src/components/run-action-dialog.tsx create mode 100644 apps/dashboard/src/components/run-action.tsx delete mode 100644 apps/dashboard/src/components/run-cancel-action.tsx delete mode 100644 apps/dashboard/src/components/run-resume-action.tsx create mode 100644 packages/openworkflow/client/rerun.test.ts create mode 100644 packages/openworkflow/client/rerun.ts create mode 100644 packages/openworkflow/core/rerun.ts diff --git a/apps/dashboard/src/components/run-action-dialog.tsx b/apps/dashboard/src/components/run-action-dialog.tsx deleted file mode 100644 index b3825d65..00000000 --- a/apps/dashboard/src/components/run-action-dialog.tsx +++ /dev/null @@ -1,119 +0,0 @@ -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; -import { Button } from "@/components/ui/button"; -import type { WorkflowRun } from "openworkflow/internal"; -import type { ReactNode } from "react"; -import { useState } from "react"; - -type ButtonVariant = React.ComponentProps["variant"]; - -interface RunActionDialogProps { - triggerLabel: string; - triggerVariant?: ButtonVariant; - title: string; - description: ReactNode; - cancelLabel: string; - confirmLabel: string; - pendingLabel: string; - confirmVariant?: ButtonVariant; - fallbackErrorMessage: string; - action: () => Promise; - onDone?: (() => Promise | void) | undefined; -} - -function getErrorMessage(cause: unknown, fallback: string): string { - if (cause instanceof Error && cause.message) { - return cause.message; - } - - return fallback; -} - -// Confirmation dialog shared by the run action buttons (cancel, resume). -export function RunActionDialog({ - triggerLabel, - triggerVariant = "default", - title, - description, - cancelLabel, - confirmLabel, - pendingLabel, - confirmVariant, - fallbackErrorMessage, - action, - onDone, -}: RunActionDialogProps) { - const [isOpen, setIsOpen] = useState(false); - const [isPending, setIsPending] = useState(false); - const [error, setError] = useState(null); - - async function runAction() { - setIsPending(true); - setError(null); - - try { - await action(); - await onDone?.(); - setIsOpen(false); - } catch (caughtError) { - setError(getErrorMessage(caughtError, fallbackErrorMessage)); - } finally { - setIsPending(false); - } - } - - return ( - { - setIsOpen(nextOpen); - if (!nextOpen) { - setError(null); - } - }} - > - - - - - {title} - {description} - - - {error &&

{error}

} - - - - {cancelLabel} - - { - void runAction(); - }} - disabled={isPending} - > - {isPending ? pendingLabel : confirmLabel} - - - - - ); -} diff --git a/apps/dashboard/src/components/run-action.tsx b/apps/dashboard/src/components/run-action.tsx new file mode 100644 index 00000000..ba2b12b9 --- /dev/null +++ b/apps/dashboard/src/components/run-action.tsx @@ -0,0 +1,134 @@ +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import { cancelWorkflowRunServerFn, rerunWorkflowRunServerFn } from "@/lib/api"; +import { isRunCancelableStatus, TERMINAL_RUN_STATUSES } from "@/lib/status"; +import { useNavigate } from "@tanstack/react-router"; +import type { WorkflowRunStatus } from "openworkflow/internal"; +import { useState } from "react"; + +interface RunActionProps { + action?: "cancel" | "rerun"; + fromStep?: string; + runId: string; + status: WorkflowRunStatus; + onDone?: (() => Promise) | (() => void); +} + +function getErrorMessage(cause: unknown): string { + if (cause instanceof Error && cause.message) { + return cause.message; + } + + return "Unable to update workflow run"; +} + +export function RunAction({ + action = "cancel", + fromStep, + runId, + status, + onDone, +}: RunActionProps) { + const navigate = useNavigate(); + const rerun = action === "rerun"; + const rerunLabel = fromStep === undefined ? "Rerun" : "Rerun from step"; + const actionLabel = rerun ? rerunLabel : "Cancel Run"; + const rerunDescription = + fromStep === undefined + ? "Creates a new run with the original input and version. All steps will execute again." + : `Creates a new run using successful results recorded before "${fromStep}". This step, later work, and earlier failed or unfinished steps will execute again if reached.`; + const [isOpen, setIsOpen] = useState(false); + const [isPending, setIsPending] = useState(false); + const [error, setError] = useState(null); + + if ( + rerun ? !TERMINAL_RUN_STATUSES.has(status) : !isRunCancelableStatus(status) + ) { + return null; + } + + async function performAction() { + setIsPending(true); + setError(null); + + try { + if (rerun) { + const run = await rerunWorkflowRunServerFn({ + data: { workflowRunId: runId, fromStep }, + }); + await navigate({ to: "/runs/$runId", params: { runId: run.id } }); + } else { + await cancelWorkflowRunServerFn({ data: { workflowRunId: runId } }); + } + await onDone?.(); + setIsOpen(false); + } catch (caughtError) { + setError(getErrorMessage(caughtError)); + } finally { + setIsPending(false); + } + } + + return ( + { + setIsOpen(nextOpen); + if (!nextOpen) { + setError(null); + } + }} + > + + + + + + {rerun ? `${rerunLabel}?` : "Cancel this run?"} + + + {rerun + ? rerunDescription + : "This will stop any future progress for this workflow run."} + + + + {error &&

{error}

} + + + + {rerun ? "Cancel" : "Keep Running"} + + { + event.preventDefault(); + void performAction(); + }} + disabled={isPending} + > + {isPending ? "Working..." : actionLabel} + + + + + ); +} diff --git a/apps/dashboard/src/components/run-cancel-action.tsx b/apps/dashboard/src/components/run-cancel-action.tsx deleted file mode 100644 index d8dd2727..00000000 --- a/apps/dashboard/src/components/run-cancel-action.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { RunActionDialog } from "@/components/run-action-dialog"; -import { cancelWorkflowRunServerFn } from "@/lib/api"; -import { isRunCancelableStatus } from "@/lib/status"; -import type { WorkflowRunStatus } from "openworkflow/internal"; - -interface RunCancelActionProps { - runId: string; - status: WorkflowRunStatus; - onCanceled?: (() => Promise) | (() => void); -} - -export function RunCancelAction({ - runId, - status, - onCanceled, -}: RunCancelActionProps) { - if (!isRunCancelableStatus(status)) { - return null; - } - - return ( - - cancelWorkflowRunServerFn({ data: { workflowRunId: runId } }) - } - onDone={onCanceled} - /> - ); -} diff --git a/apps/dashboard/src/components/run-resume-action.tsx b/apps/dashboard/src/components/run-resume-action.tsx deleted file mode 100644 index f3f5f3a0..00000000 --- a/apps/dashboard/src/components/run-resume-action.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { RunActionDialog } from "@/components/run-action-dialog"; -import { resumeWorkflowRunServerFn } from "@/lib/api"; -import { isRunResumableStatus } from "@/lib/status"; -import type { WorkflowRunStatus } from "openworkflow/internal"; - -interface RunResumeActionProps { - runId: string; - status: WorkflowRunStatus; - onResumed?: (() => Promise) | (() => void); -} - -export function RunResumeAction({ - runId, - status, - onResumed, -}: RunResumeActionProps) { - if (!isRunResumableStatus(status)) { - return null; - } - - return ( - - resumeWorkflowRunServerFn({ data: { workflowRunId: runId } }) - } - onDone={onResumed} - /> - ); -} diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index af504272..fa2cb945 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -1,5 +1,6 @@ import { getBackend } from "./backend"; import { createServerFn } from "@tanstack/react-start"; +import { rerunWorkflowRun } from "openworkflow/internal"; import type { PaginatedResponse, PaginationOptions, @@ -90,16 +91,17 @@ export const cancelWorkflowRunServerFn = createServerFn({ method: "POST" }) return backend.cancelWorkflowRun({ workflowRunId: data.workflowRunId }); }); -/** - * Resume a failed workflow run by ID. Flips the run back to `pending` and - * gives the failing step a fresh retry budget by only counting failures after - * the resume; completed steps stay cached and history is preserved. - */ -export const resumeWorkflowRunServerFn = createServerFn({ method: "POST" }) - .validator(z.object({ workflowRunId: z.string() })) +/** Rerun a finished workflow with its original input and version. */ +export const rerunWorkflowRunServerFn = createServerFn({ method: "POST" }) + .validator( + z.object({ workflowRunId: z.string(), fromStep: z.string().optional() }), + ) .handler(async ({ data }): Promise => { - const backend = await getBackend(); - return backend.resumeWorkflowRun({ workflowRunId: data.workflowRunId }); + return await rerunWorkflowRun( + await getBackend(), + data.workflowRunId, + data.fromStep, + ); }); /** diff --git a/apps/dashboard/src/lib/status.ts b/apps/dashboard/src/lib/status.ts index d35fc64e..0a71e50e 100644 --- a/apps/dashboard/src/lib/status.ts +++ b/apps/dashboard/src/lib/status.ts @@ -171,9 +171,6 @@ const CANCELABLE_RUN_STATUSES: ReadonlySet = new Set( ], ); -/** Run statuses that can be resumed from the dashboard. */ -const RESUMABLE_RUN_STATUSES: ReadonlySet = new Set(["failed"]); - const fallbackStatusConfig = STATUS_CONFIG.pending; export function getRunStatusConfig(status: string): StatusConfig { @@ -204,7 +201,3 @@ export function getStatusStatIconClass(status: string): string { export function isRunCancelableStatus(status: string): boolean { return CANCELABLE_RUN_STATUSES.has(status); } - -export function isRunResumableStatus(status: string): boolean { - return RESUMABLE_RUN_STATUSES.has(status); -} diff --git a/apps/dashboard/src/routes/runs/$runId.tsx b/apps/dashboard/src/routes/runs/$runId.tsx index a949c343..18a92a6d 100644 --- a/apps/dashboard/src/routes/runs/$runId.tsx +++ b/apps/dashboard/src/routes/runs/$runId.tsx @@ -1,7 +1,6 @@ import { AppLayout } from "@/components/app-layout"; import { CursorPaginationControls } from "@/components/cursor-pagination-controls"; -import { RunCancelAction } from "@/components/run-cancel-action"; -import { RunResumeAction } from "@/components/run-resume-action"; +import { RunAction } from "@/components/run-action"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; @@ -287,17 +286,11 @@ function RunDetailsPage() { )}
- + { - await router.invalidate(); - }} - /> - { + onDone={async () => { await router.invalidate(); }} /> @@ -484,6 +477,7 @@ function RunDetailsPage() {

Step Inspector

{step.stepName}

+
diff --git a/apps/docs/docs/retries.mdx b/apps/docs/docs/retries.mdx index 1f8b47db..0f71fe9f 100644 --- a/apps/docs/docs/retries.mdx +++ b/apps/docs/docs/retries.mdx @@ -28,6 +28,24 @@ To prevent runaway executions, each workflow run also has a hard cap of 1000 total step attempts. If the run reaches that cap, it fails immediately and is not retried. +## Rerun a Finished Workflow + +Use `rerunWorkflowRun` to create a new run from a failed, completed, or canceled +run. Optionally pass a recorded step name to reuse earlier results: + +```ts +const newRun = await ow.rerunWorkflowRun(originalRunId, { + fromStep: "send-email", // <-- optional +}); +``` + +The new run uses the original input and workflow version. It is ready to run +immediately, with fresh retry counts and no deadline, idempotency key, or parent +run. The source run stays unchanged. + +With `fromStep`, OpenWorkflow copies successful steps from before the selected +step. The new run reuses those results. + ## Step Retries Steps that throw are retried automatically: diff --git a/cspell.config.jsonc b/cspell.config.jsonc index 2faae4e4..51b202a0 100644 --- a/cspell.config.jsonc +++ b/cspell.config.jsonc @@ -30,9 +30,10 @@ "trivago", "tsgolint", - // postgres + // SQL "hashtextextended", "mydb", + "randomblob", "sslmode", "timestamptz", "xact", diff --git a/oxlint.config.ts b/oxlint.config.ts index ac3abf5f..bc02e347 100644 --- a/oxlint.config.ts +++ b/oxlint.config.ts @@ -197,7 +197,7 @@ export default defineConfig({ files: [ "apps/cli/commands.ts", "apps/dashboard/src/components/create-run-form.tsx", - "apps/dashboard/src/components/run-cancel-action.tsx", + "apps/dashboard/src/components/run-action.tsx", "apps/dashboard/src/components/run-list.tsx", "apps/dashboard/src/routes/index.tsx", "apps/dashboard/src/routes/runs/$runId.tsx", diff --git a/packages/openworkflow/client/client.test.ts b/packages/openworkflow/client/client.test.ts index 40d2157d..7fcd476e 100644 --- a/packages/openworkflow/client/client.test.ts +++ b/packages/openworkflow/client/client.test.ts @@ -848,7 +848,6 @@ function createMockWorkflowRun( deadlineAt: null, startedAt: null, finishedAt: null, - resumedAt: null, createdAt: currentTime, updatedAt: currentTime, ...overrides, diff --git a/packages/openworkflow/client/client.ts b/packages/openworkflow/client/client.ts index e09bcb96..bd77473b 100644 --- a/packages/openworkflow/client/client.ts +++ b/packages/openworkflow/client/client.ts @@ -28,6 +28,7 @@ import { traceOperation, } from "../telemetry.js"; import { Worker } from "../worker/worker.js"; +import { rerunWorkflowRun } from "./rerun.js"; const DEFAULT_RESULT_POLL_INTERVAL_MS = 1000; // 1s const DEFAULT_RESULT_TIMEOUT_MS = 5 * 60 * 1000; // 5m @@ -200,21 +201,22 @@ export class OpenWorkflow { } /** - * Resume a failed workflow run. The run's status flips back to `pending` - * so the next worker tick picks it up. Already-completed steps are served - * from history without re-executing. Nothing is deleted: the failing step - * starts with a fresh retry budget because only failures recorded after the - * resume count against it. - * @param workflowRunId - The ID of the failed workflow run to resume - * @returns The updated workflow run - * @throws {Error} If the run does not exist or is not in `failed` status - * @example - * ```ts - * await ow.resumeWorkflowRun("123"); - * ``` + * Rerun a finished workflow with its original input and version. + * @param workflowRunId - ID of a failed, completed, or canceled run + * @param options - Rerun options + * @param options.fromStep - Recorded step name to rerun, reusing earlier + * successful results; omit to run every step again + * @returns A new pending run, leaving the source run unchanged */ - async resumeWorkflowRun(workflowRunId: string): Promise { - return await this.backend.resumeWorkflowRun({ workflowRunId }); + async rerunWorkflowRun( + workflowRunId: string, + options?: { fromStep?: string }, + ): Promise { + return await rerunWorkflowRun( + this.backend, + workflowRunId, + options?.fromStep, + ); } /** diff --git a/packages/openworkflow/client/rerun.test.ts b/packages/openworkflow/client/rerun.test.ts new file mode 100644 index 00000000..26fcbc0f --- /dev/null +++ b/packages/openworkflow/client/rerun.test.ts @@ -0,0 +1,487 @@ +import { isTerminalStatus } from "../core/workflow-run.js"; +import { BackendPostgres } from "../postgres/backend.js"; +import { createTestBackend } from "../postgres/test-backend.testsuite.js"; +import { BackendSqlite } from "../sqlite/backend.js"; +import { OpenWorkflow } from "./client.js"; +import assert from "node:assert/strict"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { + let backend: BackendSqlite | BackendPostgres; + let client: OpenWorkflow; + beforeEach(async () => { + backend = + database === "sqlite" + ? BackendSqlite.connect(":memory:") + : await createTestBackend(); + client = new OpenWorkflow({ backend }); + }); + afterEach(async () => { + vi.restoreAllMocks(); + if (database === "sqlite") await backend.stop(); + }); + + async function finish(workflowRunId: string) { + const worker = client.newWorker(); + for (let tick = 0; tick < 100; tick++) { + await worker.tick(); + const run = await backend.getWorkflowRun({ workflowRunId }); + if (run && isTerminalStatus(run.status)) return run; + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + } + throw new Error("Run did not finish"); + } + + async function history(workflowRunId: string) { + const result = await backend.listStepAttempts({ + workflowRunId, + limit: 1000, + }); + return result.data; + } + + test("copies successful results before execution, preserves the source, and gives fresh retries", async () => { + const calls = { read: 0, write: 0 }; + let broken = true; + const workflow = client.defineWorkflow<{ value: number }, number>( + { name: "recover", version: "v1" }, + async ({ step, input }) => { + const value = await step.run({ name: "read" }, async () => { + calls.read++; + await new Promise((resolve) => { + setTimeout(resolve, 2); + }); + return input.value; + }); + return await step.run( + { + name: "write", + retryPolicy: { maximumAttempts: 2, initialInterval: "1ms" }, + }, + () => { + calls.write++; + if (broken) throw new Error("offline"); + return value; + }, + ); + }, + ); + const original = await workflow.run( + { value: 42 }, + { idempotencyKey: "request" }, + ); + const source = await finish(original.workflowRun.id); + expect(source.status).toBe("failed"); + const sourceHistory = await history(source.id); + const rerun = await client.rerunWorkflowRun(source.id, { + fromStep: "write", + }); + expect(rerun).toMatchObject({ + input: { value: 42 }, + version: "v1", + status: "pending", + attempts: 0, + error: null, + context: null, + idempotencyKey: null, + deadlineAt: null, + parentStepAttemptNamespaceId: null, + parentStepAttemptId: null, + }); + const copied = await history(rerun.id); + expect(copied).toHaveLength(1); + expect(copied[0]).toEqual({ + ...sourceHistory[0], + id: copied[0]?.id, + workflowRunId: rerun.id, + }); + expect(copied[0]?.id).not.toBe(sourceHistory[0]?.id); + await expect(finish(rerun.id)).resolves.toMatchObject({ status: "failed" }); + expect(calls).toEqual({ read: 1, write: 4 }); + + broken = false; + const recovered = await client.rerunWorkflowRun(rerun.id, { + fromStep: "write", + }); + await expect(finish(recovered.id)).resolves.toMatchObject({ output: 42 }); + expect(calls).toEqual({ read: 1, write: 5 }); + const fresh = await client.rerunWorkflowRun(recovered.id); + expect(await history(fresh.id)).toEqual([]); + await expect(finish(fresh.id)).resolves.toMatchObject({ output: 42 }); + expect(calls).toEqual({ read: 2, write: 6 }); + expect(await backend.getWorkflowRun({ workflowRunId: source.id })).toEqual( + source, + ); + expect(await history(source.id)).toEqual(sourceHistory); + }); + + test("copies only successful attempts before the target's first attempt in timestamp and ID order", async () => { + const source = await backend.createWorkflowRun({ + workflowName: "snapshot", + version: null, + input: null, + config: {}, + context: null, + idempotencyKey: null, + parentStepAttemptNamespaceId: null, + parentStepAttemptId: null, + availableAt: null, + deadlineAt: null, + }); + const workerId = "seed"; + await backend.claimWorkflowRun({ workerId, leaseDurationMs: 60_000 }); + async function attempt( + stepName: string, + status: "completed" | "failed" | "running", + ) { + const step = await backend.createStepAttempt({ + workflowRunId: source.id, + workerId, + stepName, + kind: "function", + config: {}, + context: null, + }); + if (status === "completed") + await backend.completeStepAttempt({ + workflowRunId: source.id, + workerId, + stepAttemptId: step.id, + output: stepName, + }); + if (status === "failed") + await backend.failStepAttempt({ + workflowRunId: source.id, + workerId, + stepAttemptId: step.id, + error: { message: "failed" }, + }); + return step; + } + const failed = await attempt("caught", "failed"); + const running = await attempt("unfinished", "running"); + const first = await attempt("first", "completed"); + const second = await attempt("second", "completed"); + const third = await attempt("third", "completed"); + const target = await attempt("target", "failed"); + await attempt("later", "completed"); + await attempt("target", "completed"); + // Fix timestamps to exercise the ID tie-breaker on both databases. + const early = new Date("2026-01-01T00:00:00Z"); + const late = new Date("2026-01-02T00:00:00Z"); + if (backend instanceof BackendSqlite) { + backend["db"] + .prepare( + 'UPDATE "step_attempts" SET "created_at" = ? WHERE "workflow_run_id" = ?', + ) + .run(late.toISOString(), source.id); + for (const id of [ + failed.id, + running.id, + first.id, + second.id, + third.id, + target.id, + ]) { + backend["db"] + .prepare('UPDATE "step_attempts" SET "created_at" = ? WHERE "id" = ?') + .run(early.toISOString(), id); + } + backend["db"] + .prepare( + 'UPDATE "step_attempts" SET "created_at" = ?, "status" = \'succeeded\' WHERE "id" = ?', + ) + .run(new Date(early.getTime() - 1).toISOString(), first.id); + backend["db"] + .prepare('UPDATE "step_attempts" SET "created_at" = ? WHERE "id" = ?') + .run(new Date(early.getTime() + 1).toISOString(), target.id); + } else { + const pg = backend["pg"]; + const table = backend["stepAttemptsTable"](); + await pg`UPDATE ${table} SET "created_at" = ${late} WHERE "workflow_run_id" = ${source.id}`; + await pg`UPDATE ${table} SET "created_at" = ${early} WHERE "id" IN ${pg([failed.id, running.id, first.id, second.id, third.id])}`; + await pg`UPDATE ${table} SET "created_at" = ${new Date(early.getTime() - 1)}, "status" = 'succeeded' WHERE "id" = ${first.id}`; + await pg`UPDATE ${table} SET "created_at" = ${new Date(early.getTime() + 1)} WHERE "id" = ${target.id}`; + } + await backend.cancelWorkflowRun({ workflowRunId: source.id }); + const sourceHistory = await history(source.id); + const saved = sourceHistory.filter( + (step) => + step.id === first.id || step.id === second.id || step.id === third.id, + ); + const rerun = await client.rerunWorkflowRun(source.id, { + fromStep: "target", + }); + const copies = await history(rerun.id); + expect(copies.map((step) => step.stepName).toSorted()).toEqual( + saved.map((step) => step.stepName).toSorted(), + ); + for (const step of copies) { + const original = saved.find((entry) => entry.stepName === step.stepName); + assert.ok(original); + expect(step.id).not.toBe(original.id); + expect(step.id).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + expect(step).toEqual({ + ...original, + id: step.id, + workflowRunId: rerun.id, + }); + } + await client.cancelWorkflowRun(rerun.id); + const selected = copies[2]; + assert.ok(selected); + const chained = await client.rerunWorkflowRun(rerun.id, { + fromStep: selected.stepName, + }); + const chainedHistory = await history(chained.id); + expect(chainedHistory.map((step) => step.stepName).toSorted()).toEqual( + copies + .slice(0, 2) + .map((step) => step.stepName) + .toSorted(), + ); + }); + + test("runs caught failures again before the selected step", async () => { + let failures = 0; + const workflow = client.defineWorkflow( + { name: "caught" }, + async ({ step }) => { + await step + .run({ name: "error", retryPolicy: { maximumAttempts: 1 } }, () => { + failures++; + throw new Error("caught"); + }) + .catch(() => null); + return await step.run({ name: "target" }, () => "ok"); + }, + ); + const source = await workflow.run(); + await finish(source.workflowRun.id); + const rerun = await client.rerunWorkflowRun(source.workflowRun.id, { + fromStep: "target", + }); + expect(await history(rerun.id)).toEqual([]); + await expect(finish(rerun.id)).resolves.toMatchObject({ output: "ok" }); + expect(failures).toBe(2); + }); + + test("replays saved signal results without deliveries or the source run", async () => { + const workflow = client.defineWorkflow( + { name: "signal" }, + async ({ step }) => { + const result = await step.waitForSignal<{ value: number }>({ + signal: "reply", + }); + await new Promise((resolve) => { + setTimeout(resolve, 2); + }); + return await step.run({ name: "target" }, () => result); + }, + ); + const original = await workflow.run(); + await client.newWorker().tick(); + await client.sendSignal({ + signal: "reply", + data: { value: 42 }, + idempotencyKey: "delivery", + }); + await finish(original.workflowRun.id); + const rerun = await client.rerunWorkflowRun(original.workflowRun.id, { + fromStep: "target", + }); + const copied = await history(rerun.id); + expect(copied).toHaveLength(1); + const savedWait = copied[0]; + assert.ok(savedWait); + await expect( + backend.getSignalDelivery({ stepAttemptId: savedWait.id }), + ).resolves.toBeUndefined(); + if (backend instanceof BackendSqlite) { + backend["db"] + .prepare('DELETE FROM "workflow_signals" WHERE "workflow_run_id" = ?') + .run(original.workflowRun.id); + backend["db"] + .prepare('DELETE FROM "workflow_runs" WHERE "id" = ?') + .run(original.workflowRun.id); + } else { + const pg = backend["pg"]; + await pg`DELETE FROM ${backend["workflowSignalsTable"]()} WHERE "workflow_run_id" = ${original.workflowRun.id}`; + await pg`DELETE FROM ${backend["workflowRunsTable"]()} WHERE "id" = ${original.workflowRun.id}`; + } + const deliveries = vi.spyOn(backend, "getSignalDelivery"); + await expect(finish(rerun.id)).resolves.toMatchObject({ + status: "completed", + output: { data: { value: 42 } }, + }); + expect(deliveries).not.toHaveBeenCalled(); + }); + + test("rolls back if rerun creation fails after inserting the new run", async () => { + const workflow = client.defineWorkflow( + { name: "atomic" }, + async ({ step }) => { + await step.run({ name: "prefix" }, async () => { + await new Promise((resolve) => { + setTimeout(resolve, 2); + }); + return "saved"; + }); + await step.run({ name: "target" }, () => null); + }, + ); + const original = await workflow.run(); + await finish(original.workflowRun.id); + const before = await backend.listWorkflowRuns({}); + let restore: () => void; + if (backend instanceof BackendSqlite) { + const sqlite = backend; + const insert = sqlite["insertWorkflowRun"]; + sqlite["insertWorkflowRun"] = (params) => { + insert.call(sqlite, params); + throw new Error("copy failed"); + }; + restore = () => { + sqlite["insertWorkflowRun"] = insert; + }; + } else { + const postgres = backend; + const insert = postgres["insertWorkflowRun"]; + postgres["insertWorkflowRun"] = async (tx, params) => { + await insert.call(postgres, tx, params); + throw new Error("copy failed"); + }; + restore = () => { + postgres["insertWorkflowRun"] = insert; + }; + } + try { + await expect( + client.rerunWorkflowRun(original.workflowRun.id, { + fromStep: "target", + }), + ).rejects.toThrow("copy failed"); + } finally { + restore(); + } + expect(await backend.listWorkflowRuns({})).toEqual(before); + const rerun = await client.rerunWorkflowRun(original.workflowRun.id, { + fromStep: "target", + }); + expect(await history(rerun.id)).toHaveLength(1); + }); + + test("starts fresh children at the restart point and reuses completed children before it", async () => { + let childCalls = 0; + let broken = true; + client.defineWorkflow({ name: "child" }, async ({ step }) => { + await step.run({ name: "earlier-child-work" }, () => ++childCalls); + return await step.run( + { name: "child-failure", retryPolicy: { maximumAttempts: 1 } }, + () => { + if (broken) throw new Error("child failed"); + return "child result"; + }, + ); + }); + const parent = client.defineWorkflow( + { name: "parent" }, + async ({ step }) => { + const result = await step.runWorkflow({ name: "child" }); + return await step.run({ name: "after-child" }, () => result); + }, + ); + const original = await parent.run(); + const failed = await finish(original.workflowRun.id); + expect(failed.status).toBe("failed"); + broken = false; + const rerun = await client.rerunWorkflowRun(failed.id, { + fromStep: "child", + }); + await expect(finish(rerun.id)).resolves.toMatchObject({ + output: "child result", + }); + expect(childCalls).toBe(2); + const priorSteps = await backend.listStepAttempts({ + workflowRunId: rerun.id, + }); + const later = await client.rerunWorkflowRun(rerun.id, { + fromStep: "after-child", + }); + await expect(finish(later.id)).resolves.toMatchObject({ + output: "child result", + }); + expect(childCalls).toBe(2); + const laterSteps = await backend.listStepAttempts({ + workflowRunId: later.id, + }); + expect( + laterSteps.data.find((step) => step.kind === "workflow") + ?.childWorkflowRunId, + ).toBe( + priorSteps.data.find((step) => step.kind === "workflow") + ?.childWorkflowRunId, + ); + expect(await backend.getWorkflowRun({ workflowRunId: failed.id })).toEqual( + failed, + ); + }); + + test("reuses waits and signal sends before the boundary and restarts them at the boundary", async () => { + const send = vi.spyOn(backend, "sendSignal"); + const workflow = client.defineWorkflow( + { name: "waits" }, + async ({ step }) => { + await step.sendSignal({ signal: "notice" }); + await step.sleep("sleep", "1ms"); + const value = await step.waitForSignal({ signal: "reply", timeout: 0 }); + return await step.run({ name: "last" }, () => value); + }, + ); + const original = await workflow.run(); + await expect(finish(original.workflowRun.id)).resolves.toMatchObject({ + status: "completed", + }); + const rerun = await client.rerunWorkflowRun(original.workflowRun.id, { + fromStep: "last", + }); + await expect(finish(rerun.id)).resolves.toMatchObject({ output: null }); + expect(send).toHaveBeenCalledTimes(1); + const again = await client.rerunWorkflowRun(rerun.id, { + fromStep: "notice", + }); + await expect(finish(again.id)).resolves.toMatchObject({ + status: "completed", + }); + expect(send).toHaveBeenCalledTimes(2); + }); + + test("rejects active runs, unknown runs and unknown steps, but accepts canceled runs", async () => { + const workflow = client.defineWorkflow({ name: "canceled" }, ({ step }) => + step.sleep("wait", "1h"), + ); + const original = await workflow.run(); + await expect( + client.rerunWorkflowRun(original.workflowRun.id), + ).rejects.toThrow("Only finished"); + await client.newWorker().tick(); + await expect( + client.rerunWorkflowRun(original.workflowRun.id), + ).rejects.toThrow("Only finished"); + await client.cancelWorkflowRun(original.workflowRun.id); + await expect( + client.rerunWorkflowRun(original.workflowRun.id, { fromStep: "missing" }), + ).rejects.toThrow("does not exist"); + await expect(client.rerunWorkflowRun("missing")).rejects.toThrow( + "does not exist", + ); + const rerun = await client.rerunWorkflowRun(original.workflowRun.id, { + fromStep: "wait", + }); + expect(rerun.status).toBe("pending"); + }); +}); diff --git a/packages/openworkflow/client/rerun.ts b/packages/openworkflow/client/rerun.ts new file mode 100644 index 00000000..f30fb316 --- /dev/null +++ b/packages/openworkflow/client/rerun.ts @@ -0,0 +1,37 @@ +import type { Backend } from "../core/backend.js"; +import type { WorkflowRun } from "../core/workflow-run.js"; +import { + captureTraceContext, + getSpanKind, + setAttributes, + SPAN_NAMES, + traceOperation, + workflowRunAttributes, +} from "../telemetry.js"; + +/** + * Rerun a finished workflow, optionally copying earlier successful steps. + * @param backend - Backend containing the source run + * @param workflowRunId - Source run ID + * @param fromStep - Step to execute again; omit to rerun every step + * @returns A new pending run + */ +export async function rerunWorkflowRun( + backend: Readonly, + workflowRunId: string, + fromStep?: string, +): Promise { + return traceOperation( + SPAN_NAMES.WORKFLOW_RUN_CREATE, + { kind: await getSpanKind("PRODUCER") }, + async (span) => { + const run = await backend.rerunWorkflowRun({ + workflowRunId, + fromStep: fromStep ?? null, + context: captureTraceContext(), + }); + setAttributes(span, workflowRunAttributes(run)); + return run; + }, + ); +} diff --git a/packages/openworkflow/core/backend.ts b/packages/openworkflow/core/backend.ts index 26490ad6..bd2953da 100644 --- a/packages/openworkflow/core/backend.ts +++ b/packages/openworkflow/core/backend.ts @@ -19,6 +19,9 @@ export interface Backend { createWorkflowRun( params: Readonly, ): Promise; + rerunWorkflowRun( + params: Readonly, + ): Promise; getWorkflowRun( params: Readonly, ): Promise; @@ -47,9 +50,6 @@ export interface Backend { cancelWorkflowRun( params: Readonly, ): Promise; - resumeWorkflowRun( - params: Readonly, - ): Promise; // Step Attempts createStepAttempt( @@ -98,6 +98,12 @@ export interface GetWorkflowRunParams { workflowRunId: string; } +export interface RerunWorkflowRunParams { + workflowRunId: string; + fromStep: string | null; + context: JsonValue | null; +} + export interface ListWorkflowRunsParams extends PaginationOptions { status?: WorkflowRunStatus; workflowName?: string; @@ -146,10 +152,6 @@ export interface CancelWorkflowRunParams { workflowRunId: string; } -export interface ResumeWorkflowRunParams { - workflowRunId: string; -} - export interface CreateStepAttemptParams { workflowRunId: string; workerId: string; diff --git a/packages/openworkflow/core/rerun.ts b/packages/openworkflow/core/rerun.ts new file mode 100644 index 00000000..27b5bade --- /dev/null +++ b/packages/openworkflow/core/rerun.ts @@ -0,0 +1,42 @@ +import type { + CreateWorkflowRunParams, + RerunWorkflowRunParams, +} from "./backend.js"; +import { isTerminalStatus, type WorkflowRun } from "./workflow-run.js"; + +/** + * Validate a rerun and reuse the source input, version, and configuration. + * @param source - Source run, read within the copying transaction + * @param request - Rerun request + * @param stepId - First attempt of the requested step, or null + * @returns New run parameters + */ +export function prepareWorkflowRerun( + source: Readonly | null, + request: Readonly, + stepId: string | null, +): CreateWorkflowRunParams { + if (!source) { + throw new Error(`Workflow run ${request.workflowRunId} does not exist`); + } + if (!isTerminalStatus(source.status)) { + throw new Error("Only finished workflow runs can be rerun"); + } + if (request.fromStep !== null && stepId === null) { + throw new Error( + `Step "${request.fromStep}" does not exist in workflow run ${source.id}`, + ); + } + return { + workflowName: source.workflowName, + version: source.version, + input: source.input, + config: source.config, + context: request.context, + idempotencyKey: null, + parentStepAttemptNamespaceId: null, + parentStepAttemptId: null, + availableAt: null, + deadlineAt: null, + }; +} diff --git a/packages/openworkflow/core/workflow-run.test.ts b/packages/openworkflow/core/workflow-run.test.ts index aadf60f0..91d21dbc 100644 --- a/packages/openworkflow/core/workflow-run.test.ts +++ b/packages/openworkflow/core/workflow-run.test.ts @@ -180,7 +180,6 @@ describe("resolveCancelWorkflowRunConflict", () => { deadlineAt: null, startedAt: null, finishedAt: null, - resumedAt: null, createdAt: new Date(0), updatedAt: new Date(0), }; diff --git a/packages/openworkflow/core/workflow-run.ts b/packages/openworkflow/core/workflow-run.ts index 3b58168c..4bb9bb26 100644 --- a/packages/openworkflow/core/workflow-run.ts +++ b/packages/openworkflow/core/workflow-run.ts @@ -55,36 +55,6 @@ export function resolveCancelWorkflowRunConflict( throw new Error("Failed to cancel workflow run"); } -/** - * Resolve the outcome when a resumeWorkflowRun UPDATE affected no rows. The - * UPDATE is gated on `status = 'failed'` AND an unexpired deadline, so a - * zero-row outcome means the run doesn't exist, isn't in a resumable state, - * or has already blown its deadline (in which case resuming it is futile). - * @param workflowRunId - ID of the workflow run (used in error messages) - * @param existing - Current workflow run, or null if not found - * @throws {Error} If the run is missing, not `failed`, or past its deadline - */ -export function resolveResumeWorkflowRunConflict( - workflowRunId: string, - existing: Readonly | null, -): never { - if (!existing) { - throw new Error(`Workflow run ${workflowRunId} does not exist`); - } - - if (existing.status === "failed") { - // The UPDATE also gates on the deadline, so a still-`failed` run that did - // not resume is one whose deadline has already elapsed. - throw new Error( - `Cannot resume workflow run ${workflowRunId}; its deadline has already passed`, - ); - } - - throw new Error( - `Cannot resume workflow run ${workflowRunId} with status ${existing.status}; only failed runs can be resumed`, - ); -} - /** * WorkflowRun represents a single execution instance of a workflow. */ @@ -108,7 +78,6 @@ export interface WorkflowRun { deadlineAt: Date | null; startedAt: Date | null; finishedAt: Date | null; - resumedAt: Date | null; // Timestamp of the most recent resume createdAt: Date; updatedAt: Date; } diff --git a/packages/openworkflow/internal.ts b/packages/openworkflow/internal.ts index 57d1606f..b1f3ddc7 100644 --- a/packages/openworkflow/internal.ts +++ b/packages/openworkflow/internal.ts @@ -12,3 +12,5 @@ export type { StepAttemptStatus, StepKind, } from "./core/step-attempt.js"; + +export { rerunWorkflowRun } from "./client/rerun.js"; diff --git a/packages/openworkflow/postgres/backend.ts b/packages/openworkflow/postgres/backend.ts index 95fb5a2c..db9becba 100644 --- a/packages/openworkflow/postgres/backend.ts +++ b/packages/openworkflow/postgres/backend.ts @@ -5,10 +5,10 @@ import { Backend, WorkflowRunCounts, CancelWorkflowRunParams, - ResumeWorkflowRunParams, ClaimWorkflowRunParams, CreateStepAttemptParams, CreateWorkflowRunParams, + RerunWorkflowRunParams, GetStepAttemptParams, GetWorkflowRunParams, ExtendWorkflowRunLeaseParams, @@ -34,11 +34,11 @@ import { } from "../core/cursor.js"; import { requireRow, wrapError } from "../core/error.js"; import { JsonValue } from "../core/json.js"; +import { prepareWorkflowRerun } from "../core/rerun.js"; import { StepAttempt } from "../core/step-attempt.js"; import { computeFailedWorkflowRunUpdate } from "../core/workflow-definition.js"; import { resolveCancelWorkflowRunConflict, - resolveResumeWorkflowRunConflict, WorkflowRun, } from "../core/workflow-run.js"; import { @@ -220,6 +220,55 @@ export class BackendPostgres implements Backend { }); } + async rerunWorkflowRun( + request: RerunWorkflowRunParams, + ): Promise { + return await this.withTransaction(async (tx) => { + const [source] = await tx` + SELECT * FROM ${this.workflowRunsTable(tx)} + WHERE "namespace_id" = ${this.namespaceId} AND "id" = ${request.workflowRunId} + FOR UPDATE + `; + const stepAttemptsTable = this.stepAttemptsTable(tx); + const [boundary] = + request.fromStep === null + ? [] + : await tx<{ id: string }[]>` + SELECT "id" FROM ${stepAttemptsTable} + WHERE "namespace_id" = ${this.namespaceId} AND "workflow_run_id" = ${request.workflowRunId} + AND "step_name" = ${request.fromStep} + ORDER BY "created_at", "id" + LIMIT 1 + `; + const params = prepareWorkflowRerun( + source ?? null, + request, + boundary?.id ?? null, + ); + const run = await this.insertWorkflowRun(tx, params); + if (boundary) { + await tx` + INSERT INTO ${stepAttemptsTable} ( + "namespace_id", "id", "workflow_run_id", "step_name", "kind", "status", + "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", + "started_at", "finished_at", "created_at", "updated_at" + ) + SELECT "namespace_id", gen_random_uuid(), ${run.id}, "step_name", "kind", "status", + "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", + "started_at", "finished_at", "created_at", "updated_at" + FROM ${stepAttemptsTable} + WHERE "namespace_id" = ${this.namespaceId} AND "workflow_run_id" = ${request.workflowRunId} + AND "status" IN ('completed', 'succeeded') + AND ("created_at", "id") < ( + SELECT "created_at", "id" FROM ${stepAttemptsTable} + WHERE "namespace_id" = ${this.namespaceId} AND "id" = ${boundary.id} + ) + `; + } + return run; + }); + } + private async insertWorkflowRun( pg: Postgres, params: CreateWorkflowRunParams, @@ -768,42 +817,6 @@ export class BackendPostgres implements Backend { return updated; } - async resumeWorkflowRun( - params: ResumeWorkflowRunParams, - ): Promise { - const workflowRunsTable = this.workflowRunsTable(); - - // Stamp the resume marker and requeue. Nothing is deleted and neither - // `error` nor `attempts` is touched: step attempts stay (preserving the - // failure record and parent/child linkage), and the retry budget is reset - // by only counting failures after `resumed_at` during replay. - const [updated] = await this.pg` - UPDATE ${workflowRunsTable} - SET - "status" = 'pending', - "worker_id" = NULL, - "started_at" = NULL, - "finished_at" = NULL, - "available_at" = NOW(), - "resumed_at" = NOW(), - "updated_at" = NOW() - WHERE "namespace_id" = ${this.namespaceId} - AND "id" = ${params.workflowRunId} - AND "status" = 'failed' - AND ("deadline_at" IS NULL OR "deadline_at" > NOW()) - RETURNING * - `; - - if (!updated) { - const existing = await this.getWorkflowRun({ - workflowRunId: params.workflowRunId, - }); - resolveResumeWorkflowRunConflict(params.workflowRunId, existing); - } - - return updated; - } - private async wakeParentWorkflowRun( childWorkflowRun: Readonly, ): Promise { diff --git a/packages/openworkflow/postgres/postgres.ts b/packages/openworkflow/postgres/postgres.ts index 92960758..f52f6523 100644 --- a/packages/openworkflow/postgres/postgres.ts +++ b/packages/openworkflow/postgres/postgres.ts @@ -240,18 +240,6 @@ export function migrations(schema: string): string[] { ON CONFLICT DO NOTHING; COMMIT;`, - - // 6 - resume marker - `BEGIN; - - ALTER TABLE ${quotedSchema}."workflow_runs" - ADD COLUMN IF NOT EXISTS "resumed_at" TIMESTAMPTZ; - - INSERT INTO ${quotedSchema}."openworkflow_migrations"("version") - VALUES (6) - ON CONFLICT DO NOTHING; - - COMMIT;`, ]; } diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index b941b843..74bf25f6 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -4,10 +4,10 @@ import { DEFAULT_RUN_IDEMPOTENCY_PERIOD_MS, Backend, CancelWorkflowRunParams, - ResumeWorkflowRunParams, ClaimWorkflowRunParams, CreateStepAttemptParams, CreateWorkflowRunParams, + RerunWorkflowRunParams, GetStepAttemptParams, GetWorkflowRunParams, ExtendWorkflowRunLeaseParams, @@ -34,11 +34,11 @@ import { } from "../core/cursor.js"; import { requireRow, wrapError } from "../core/error.js"; import { JsonValue } from "../core/json.js"; +import { prepareWorkflowRerun } from "../core/rerun.js"; import { StepAttempt } from "../core/step-attempt.js"; import { computeFailedWorkflowRunUpdate } from "../core/workflow-definition.js"; import { resolveCancelWorkflowRunConflict, - resolveResumeWorkflowRunConflict, WorkflowRun, } from "../core/workflow-run.js"; import { @@ -178,6 +178,86 @@ export class BackendSqlite implements Backend { } } + // oxlint-disable-next-line typescript/require-await -- keep the transaction synchronous + async rerunWorkflowRun( + request: RerunWorkflowRunParams, + ): Promise { + this.db.exec("BEGIN IMMEDIATE"); + try { + // safety: these queries select rows from the corresponding tables. + const source = this.db + .prepare( + ` + SELECT * FROM "workflow_runs" WHERE "namespace_id" = ? AND "id" = ? + `, + ) + .get(this.namespaceId, request.workflowRunId) as + WorkflowRunRow | undefined; + // safety: the query selects the non-null step attempt ID. + const boundary = + request.fromStep === null + ? undefined + : (this.db + .prepare( + ` + SELECT "id" FROM "step_attempts" + WHERE "namespace_id" = ? AND "workflow_run_id" = ? AND "step_name" = ? + ORDER BY "created_at", "id" + LIMIT 1 + `, + ) + .get( + this.namespaceId, + request.workflowRunId, + request.fromStep, + ) as { id: string } | undefined); + const params = prepareWorkflowRerun( + source ? rowToWorkflowRun(source) : null, + request, + boundary?.id ?? null, + ); + const run = this.insertWorkflowRun(params); + if (boundary) { + this.db + .prepare( + ` + INSERT INTO "step_attempts" ( + "namespace_id", "id", "workflow_run_id", "step_name", "kind", "status", + "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", + "started_at", "finished_at", "created_at", "updated_at" + ) + SELECT "namespace_id", + lower(hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-4' || + substr(hex(randomblob(2)), 2) || '-' || substr('89ab', (random() & 3) + 1, 1) || + substr(hex(randomblob(2)), 2) || '-' || hex(randomblob(6))), + ?, "step_name", "kind", "status", + "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", + "started_at", "finished_at", "created_at", "updated_at" + FROM "step_attempts" + WHERE "namespace_id" = ? AND "workflow_run_id" = ? + AND "status" IN ('completed', 'succeeded') + AND ("created_at", "id") < ( + SELECT "created_at", "id" FROM "step_attempts" + WHERE "namespace_id" = ? AND "id" = ? + ) + `, + ) + .run( + run.id, + this.namespaceId, + request.workflowRunId, + this.namespaceId, + boundary.id, + ); + } + this.db.exec("COMMIT"); + return run; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + private insertWorkflowRun(params: CreateWorkflowRunParams): WorkflowRun { const id = generateUUID(); const currentTime = now(); @@ -760,54 +840,6 @@ export class BackendSqlite implements Backend { return updated; } - async resumeWorkflowRun( - params: ResumeWorkflowRunParams, - ): Promise { - const currentTime = now(); - - // Stamp the resume marker and requeue. Nothing is deleted and neither - // `error` nor `attempts` is touched: step attempts stay (preserving the - // failure record and parent/child linkage), and the retry budget is reset - // by only counting failures after `resumed_at` during replay. - const updateResult = this.db - .prepare( - ` - UPDATE "workflow_runs" - SET - "status" = 'pending', - "worker_id" = NULL, - "started_at" = NULL, - "finished_at" = NULL, - "available_at" = ?, - "resumed_at" = ?, - "updated_at" = ? - WHERE "namespace_id" = ? - AND "id" = ? - AND "status" = 'failed' - AND ("deadline_at" IS NULL OR "deadline_at" > ?) - `, - ) - .run( - currentTime, - currentTime, - currentTime, - this.namespaceId, - params.workflowRunId, - currentTime, - ); - - const updated = await this.getWorkflowRun({ - workflowRunId: params.workflowRunId, - }); - - if (updateResult.changes === 0) { - resolveResumeWorkflowRunConflict(params.workflowRunId, updated); - } - - requireRow(updated, "resume workflow run"); - return updated; - } - /** * Return positional placeholders for {@link RUNNING_WORKFLOW_RUN_OWNED_WHERE} * in the order the fragment expects: namespace, run id, worker id. @@ -1182,7 +1214,6 @@ interface WorkflowRunRow extends Record { deadline_at: string | null; started_at: string | null; finished_at: string | null; - resumed_at: string | null; created_at: string; updated_at: string; } @@ -1265,7 +1296,6 @@ function rowToWorkflowRun(row: WorkflowRunRow): WorkflowRun { deadlineAt: fromISO(row.deadline_at), startedAt: fromISO(row.started_at), finishedAt: fromISO(row.finished_at), - resumedAt: fromISO(row.resumed_at), createdAt, updatedAt, }; diff --git a/packages/openworkflow/sqlite/sqlite.ts b/packages/openworkflow/sqlite/sqlite.ts index c1b504a4..daccc91a 100644 --- a/packages/openworkflow/sqlite/sqlite.ts +++ b/packages/openworkflow/sqlite/sqlite.ts @@ -230,16 +230,6 @@ export function migrations(): string[] { VALUES (5); COMMIT;`, - - // 6 - resume marker - `BEGIN; - - ALTER TABLE "workflow_runs" ADD COLUMN "resumed_at" TEXT; - - INSERT OR IGNORE INTO "openworkflow_migrations" ("version") - VALUES (6); - - COMMIT;`, ]; } diff --git a/packages/openworkflow/telemetry.test.ts b/packages/openworkflow/telemetry.test.ts index 22f3c084..ea58f328 100644 --- a/packages/openworkflow/telemetry.test.ts +++ b/packages/openworkflow/telemetry.test.ts @@ -321,6 +321,76 @@ describe("native OpenTelemetry instrumentation", () => { ); }); + test.each([undefined, "target"])( + "links a rerun from %s to its own submission context after reopening storage", + async (fromStep) => { + const ow = new OpenWorkflow({ backend }); + let baggage: string | undefined; + const workflow = ow.defineWorkflow( + { name: "rerun", version: "v1" }, + async ({ step }) => { + baggage = propagation + .getBaggage(context.active()) + ?.getEntry("example")?.value; + await step.run({ name: "prefix" }, () => "saved"); + return await step.run({ name: "target" }, () => 42); + }, + ); + const source = await workflow.run(); + await executeNext(workflow.workflow); + const origin = propagation.setBaggage( + ROOT_CONTEXT, + propagation.createBaggage({ example: { value: "rerun-request" } }), + ); + const rerun = await context.with(origin, () => + tracer.startActiveSpan("rerun-request", async (span) => { + try { + return await ow.rerunWorkflowRun( + source.workflowRun.id, + fromStep === undefined ? undefined : { fromStep }, + ); + } finally { + span.end(); + } + }), + ); + await backend.stop(); + backend = BackendSqlite.connect(databasePath); + await executeNext(workflow.workflow); + + const spans = exporter.getFinishedSpans(); + const request = spans.find((span) => span.name === "rerun-request"); + const submission = spans.find( + (span) => + span.name === SPAN_NAMES.WORKFLOW_RUN_CREATE && + span.attributes[ATTRIBUTE_NAMES.WORKFLOW_RUN_ID] === rerun.id, + ); + const execution = spans.find( + (span) => + span.name === SPAN_NAMES.WORKFLOW_RUN_EXECUTE && + span.attributes[ATTRIBUTE_NAMES.WORKFLOW_RUN_ID] === rerun.id, + ); + assert.ok(request); + assert.ok(submission); + assert.ok(execution); + expect(submission.parentSpanContext?.spanId).toBe( + request.spanContext().spanId, + ); + expect(submission.kind).toBe(SpanKind.PRODUCER); + expect(submission.attributes).toMatchObject({ + [ATTRIBUTE_NAMES.WORKFLOW_NAME]: "rerun", + [ATTRIBUTE_NAMES.WORKFLOW_VERSION]: "v1", + [ATTRIBUTE_NAMES.NAMESPACE_ID]: "default", + }); + expect(execution.parentSpanContext).toBeUndefined(); + expect(execution.links[0]?.context).toMatchObject({ + traceId: submission.spanContext().traceId, + spanId: submission.spanContext().spanId, + }); + expect(baggage).toBe("rerun-request"); + }, + ); + test("traces signal sends and suspended executions without exposing payloads", async () => { const ow = new OpenWorkflow({ backend }); const workflow = ow.defineWorkflow( diff --git a/packages/openworkflow/testing/backend-stub.testsuite.ts b/packages/openworkflow/testing/backend-stub.testsuite.ts index 9329f411..e3b3157b 100644 --- a/packages/openworkflow/testing/backend-stub.testsuite.ts +++ b/packages/openworkflow/testing/backend-stub.testsuite.ts @@ -8,6 +8,7 @@ import type { Backend } from "../core/backend.js"; export function createStubBackend(overrides: Partial): Backend { return { createWorkflowRun: unexpectedBackendCall, + rerunWorkflowRun: unexpectedBackendCall, getWorkflowRun: unexpectedBackendCall, listWorkflowRuns: unexpectedBackendCall, countWorkflowRuns: unexpectedBackendCall, @@ -18,7 +19,6 @@ export function createStubBackend(overrides: Partial): Backend { failWorkflowRun: unexpectedBackendCall, rescheduleWorkflowRunAfterFailedStepAttempt: unexpectedBackendCall, cancelWorkflowRun: unexpectedBackendCall, - resumeWorkflowRun: unexpectedBackendCall, createStepAttempt: unexpectedBackendCall, getStepAttempt: unexpectedBackendCall, listStepAttempts: unexpectedBackendCall, diff --git a/packages/openworkflow/testing/backend.testsuite.ts b/packages/openworkflow/testing/backend.testsuite.ts index 2b6aff68..6cdf20e8 100644 --- a/packages/openworkflow/testing/backend.testsuite.ts +++ b/packages/openworkflow/testing/backend.testsuite.ts @@ -67,7 +67,6 @@ export function testBackend(options: TestBackendOptions): void { deadlineAt: newDateInOneYear(), startedAt: null, finishedAt: null, - resumedAt: null, createdAt: new Date(), // - updatedAt: new Date(), // - }; @@ -2576,271 +2575,6 @@ export function testBackend(options: TestBackendOptions): void { }); }); - describe("resumeWorkflowRun()", () => { - test("requeues a failed run and stamps resumed_at without erasing history", async () => { - const backend = await setup(); - - await createPendingWorkflowRun(backend); - const failedId = await claimAndFailNextPendingRun(backend); - - const failedRun = await backend.getWorkflowRun({ - workflowRunId: failedId, - }); - expect(failedRun?.error).not.toBeNull(); - - const resumed = await backend.resumeWorkflowRun({ - workflowRunId: failedId, - }); - - expect(resumed.status).toBe("pending"); - expect(resumed.workerId).toBeNull(); - expect(resumed.startedAt).toBeNull(); - expect(resumed.finishedAt).toBeNull(); - expect(resumed.availableAt).not.toBeNull(); - expect(deltaSeconds(resumed.availableAt)).toBeLessThan(1); - // resume marker is stamped; the budget is reset by counting failures - // after it, not by mutating the run - expect(resumed.resumedAt).not.toBeNull(); - expect(deltaSeconds(resumed.resumedAt)).toBeLessThan(1); - // error and the claim counter are left as the record of what happened - expect(resumed.error).not.toBeNull(); - expect(resumed.attempts).toBe(failedRun?.attempts); - - await teardown(backend); - }); - - test("preserves every step attempt and the run error on resume", async () => { - const backend = await setup(); - - const claimed = await createClaimedWorkflowRun(backend); - const workerId = claimed.workerId!; // eslint-disable-line @typescript-eslint/no-non-null-assertion - - const completed = await backend.createStepAttempt({ - workflowRunId: claimed.id, - workerId, - stepName: "completed-step", - kind: "function", - config: {}, - context: null, - }); - await backend.completeStepAttempt({ - workflowRunId: claimed.id, - stepAttemptId: completed.id, - workerId, - output: { ok: true }, - }); - - const failed = await backend.createStepAttempt({ - workflowRunId: claimed.id, - workerId, - stepName: "failed-step", - kind: "function", - config: {}, - context: null, - }); - await backend.failStepAttempt({ - workflowRunId: claimed.id, - stepAttemptId: failed.id, - workerId, - error: { message: "boom" }, - }); - - // In-flight durable wait: preserved so replay resumes it rather than - // restarting the timer from zero. - await backend.createStepAttempt({ - workflowRunId: claimed.id, - workerId, - stepName: "running-sleep", - kind: "sleep", - config: {}, - context: { - kind: "sleep", - resumeAt: new Date(Date.now() + 60_000).toISOString(), - }, - }); - - await backend.failWorkflowRun({ - workflowRunId: claimed.id, - workerId, - error: { message: "run failed" }, - retryPolicy: { - ...DEFAULT_WORKFLOW_RETRY_POLICY, - maximumAttempts: 1, - }, - }); - - const failedRun = await backend.getWorkflowRun({ - workflowRunId: claimed.id, - }); - expect(failedRun?.status).toBe("failed"); - - const resumed = await backend.resumeWorkflowRun({ - workflowRunId: claimed.id, - }); - // the failure record survives the resume - expect(resumed.error).not.toBeNull(); - - // nothing is deleted: completed, failed and running attempts all remain - const attempts = await backend.listStepAttempts({ - workflowRunId: claimed.id, - limit: 100, - }); - const survivingNames = attempts.data.map((a) => a.stepName); - expect(survivingNames).toHaveLength(3); - expect(survivingNames).toContain("completed-step"); - expect(survivingNames).toContain("failed-step"); - expect(survivingNames).toContain("running-sleep"); - - await teardown(backend); - }); - - test("preserves an in-flight child-workflow attempt so the child is not orphaned", async () => { - const backend = await setup(); - - const parent = await createClaimedWorkflowRun(backend); - const workerId = parent.workerId!; // eslint-disable-line @typescript-eslint/no-non-null-assertion - - // Running kind='workflow' attempt with a child run linked back to it. - const workflowAttempt = await backend.createStepAttempt({ - workflowRunId: parent.id, - workerId, - stepName: "invoke-child", - kind: "workflow", - config: {}, - context: { kind: "workflow", timeoutAt: null }, - }); - - const child = await backend.createWorkflowRun({ - workflowName: randomUUID(), - version: null, - idempotencyKey: null, - input: null, - config: {}, - context: null, - parentStepAttemptNamespaceId: workflowAttempt.namespaceId, - parentStepAttemptId: workflowAttempt.id, - availableAt: null, - deadlineAt: null, - }); - expect(child.parentStepAttemptId).toBe(workflowAttempt.id); - - await backend.failWorkflowRun({ - workflowRunId: parent.id, - workerId, - error: { message: "sibling failed" }, - retryPolicy: { - ...DEFAULT_WORKFLOW_RETRY_POLICY, - maximumAttempts: 1, - }, - }); - - await backend.resumeWorkflowRun({ workflowRunId: parent.id }); - - // The running workflow attempt must survive the resume... - const attempts = await backend.listStepAttempts({ - workflowRunId: parent.id, - limit: 100, - }); - expect(attempts.data.some((a) => a.id === workflowAttempt.id)).toBe( - true, - ); - - // ...so the child's parent pointer is not nulled by ON DELETE SET NULL. - const childAfter = await backend.getWorkflowRun({ - workflowRunId: child.id, - }); - expect(childAfter?.parentStepAttemptId).toBe(workflowAttempt.id); - - await teardown(backend); - }); - - test("throws and preserves history when the deadline has passed", async () => { - const backend = await setup(); - - const created = await backend.createWorkflowRun({ - workflowName: randomUUID(), - version: null, - idempotencyKey: null, - input: null, - config: {}, - context: null, - parentStepAttemptNamespaceId: null, - parentStepAttemptId: null, - availableAt: null, - deadlineAt: new Date(Date.now() - 1000), - }); - - // Claiming triggers the deadline sweep, which flips the run to failed; - // the run itself is then excluded from the claim, so this returns null. - await backend.claimWorkflowRun({ - workerId: randomUUID(), - leaseDurationMs: 100, - }); - - const failedRun = await backend.getWorkflowRun({ - workflowRunId: created.id, - }); - expect(failedRun?.status).toBe("failed"); - expect(failedRun?.error).not.toBeNull(); - - await expect( - backend.resumeWorkflowRun({ workflowRunId: created.id }), - ).rejects.toThrow(/deadline has already passed/); - - // Resume must not have destroyed the run's failure diagnostics. - const afterResume = await backend.getWorkflowRun({ - workflowRunId: created.id, - }); - expect(afterResume?.status).toBe("failed"); - expect(afterResume?.error).not.toBeNull(); - - await teardown(backend); - }); - - test("throws when resuming a run that is not failed", async () => { - const backend = await setup(); - - const created = await createPendingWorkflowRun(backend); - - await expect( - backend.resumeWorkflowRun({ workflowRunId: created.id }), - ).rejects.toThrow(/Cannot resume workflow run .* with status pending/); - - await teardown(backend); - }); - - test("throws when resuming a non-existent workflow run", async () => { - const backend = await setup(); - - const nonExistentId = randomUUID(); - - await expect( - backend.resumeWorkflowRun({ workflowRunId: nonExistentId }), - ).rejects.toThrow(`Workflow run ${nonExistentId} does not exist`); - - await teardown(backend); - }); - - test("a resumed run is claimable by workers again", async () => { - const backend = await setup(); - - await createPendingWorkflowRun(backend); - const failedId = await claimAndFailNextPendingRun(backend); - - await backend.resumeWorkflowRun({ workflowRunId: failedId }); - - const claimed = await backend.claimWorkflowRun({ - workerId: randomUUID(), - leaseDurationMs: 100, - }); - - expect(claimed?.id).toBe(failedId); - expect(claimed?.status).toBe("running"); - - await teardown(backend); - }); - }); - describe("sendSignal()", () => { test("returns empty when no active waiters", async () => { const result = await backend.sendSignal({ diff --git a/packages/openworkflow/worker/execution.test.ts b/packages/openworkflow/worker/execution.test.ts index 824a98ae..1e79c92e 100644 --- a/packages/openworkflow/worker/execution.test.ts +++ b/packages/openworkflow/worker/execution.test.ts @@ -3095,213 +3095,6 @@ describe("StepExecutor", () => { expect(status).toBe("failed"); sendSignalSpy.mockRestore(); }); - - test("resumeWorkflowRun re-runs the failed step without re-executing completed steps", async () => { - const backend = await createTestBackend(); - const client = new OpenWorkflow({ backend }); - - let validateRuns = 0; - let flakyRuns = 0; - let shouldFail = true; - - const workflow = client.defineWorkflow( - { name: `resume-from-failure-${randomUUID()}` }, - async ({ step }) => { - const validated = await step.run({ name: "validate" }, () => { - validateRuns++; - return "ok"; - }); - - const flaky = await step.run( - { name: "flaky", retryPolicy: { maximumAttempts: 2 } }, - () => { - flakyRuns++; - if (shouldFail) { - throw new Error("simulated upstream failure"); - } - return "recovered"; - }, - ); - - return { validated, flaky }; - }, - ); - - const worker = client.newWorker({ concurrency: 1 }); - const handle = await workflow.run(); - - const failedStatus = await tickUntilTerminal( - backend, - worker, - handle.workflowRun.id, - 40, - 25, - { maxWaitMs: 20_000 }, - ); - expect(failedStatus).toBe("failed"); - expect(validateRuns).toBe(1); - expect(flakyRuns).toBe(2); - - const stepsBeforeResume = await backend.listStepAttempts({ - workflowRunId: handle.workflowRun.id, - limit: 100, - }); - const failedBefore = stepsBeforeResume.data.filter( - (s) => s.status === "failed", - ); - expect(failedBefore).toHaveLength(2); - - shouldFail = false; - await backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }); - - const resumedRun = await backend.getWorkflowRun({ - workflowRunId: handle.workflowRun.id, - }); - expect(resumedRun?.status).toBe("pending"); - expect(resumedRun?.startedAt).toBeNull(); - expect(resumedRun?.finishedAt).toBeNull(); - expect(resumedRun?.workerId).toBeNull(); - // resume marker is stamped; the failure record is left intact - expect(resumedRun?.resumedAt).not.toBeNull(); - expect(resumedRun?.error).not.toBeNull(); - - const stepsAfterResume = await backend.listStepAttempts({ - workflowRunId: handle.workflowRun.id, - limit: 100, - }); - // history is preserved: the two failed "flaky" attempts still exist - expect( - stepsAfterResume.data.filter( - (s) => s.stepName === "flaky" && s.status === "failed", - ), - ).toHaveLength(2); - expect( - stepsAfterResume.data.some( - (s) => s.stepName === "validate" && s.status === "completed", - ), - ).toBe(true); - - const finalStatus = await tickUntilTerminal( - backend, - worker, - handle.workflowRun.id, - 40, - 25, - { maxWaitMs: 20_000 }, - ); - expect(finalStatus).toBe("completed"); - // "validate" was cached from the original run, not re-executed - expect(validateRuns).toBe(1); - // "flaky" ran 2 times on the original run + 1 on resume - expect(flakyRuns).toBe(3); - - const result = await handle.result(); - expect(result).toEqual({ validated: "ok", flaky: "recovered" }); - }, 30_000); - - test("resumeWorkflowRun continues past the fixed step to downstream steps", async () => { - const backend = await createTestBackend(); - const client = new OpenWorkflow({ backend }); - - const ran: string[] = []; - let gatewayDown = true; - - const workflow = client.defineWorkflow( - { name: `resume-middle-step-${randomUUID()}` }, - async ({ step }) => { - await step.run({ name: "validate" }, () => { - ran.push("validate"); - return "ok"; - }); - - await step.run( - { name: "reserve", retryPolicy: { maximumAttempts: 2 } }, - () => { - ran.push("reserve"); - if (gatewayDown) { - throw new Error("gateway down"); - } - return "auth"; - }, - ); - - await step.run({ name: "confirm" }, () => { - ran.push("confirm"); - return "receipt"; - }); - - await step.run({ name: "send-receipt" }, () => { - ran.push("send-receipt"); - }); - - return "done"; - }, - ); - - const worker = client.newWorker({ concurrency: 1 }); - const handle = await workflow.run(); - - const failedStatus = await tickUntilTerminal( - backend, - worker, - handle.workflowRun.id, - 40, - 25, - { maxWaitMs: 20_000 }, - ); - expect(failedStatus).toBe("failed"); - // validate completed once; reserve exhausted its 2 attempts; the steps - // after the failing one never ran - expect(ran.filter((s) => s === "validate")).toHaveLength(1); - expect(ran.filter((s) => s === "reserve")).toHaveLength(2); - expect(ran).not.toContain("confirm"); - expect(ran).not.toContain("send-receipt"); - - gatewayDown = false; - await backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }); - - const finalStatus = await tickUntilTerminal( - backend, - worker, - handle.workflowRun.id, - 40, - 25, - { maxWaitMs: 20_000 }, - ); - expect(finalStatus).toBe("completed"); - // validate stayed cached (not re-run); reserve retried once more with a - // fresh budget and succeeded; the downstream steps now run - expect(ran.filter((s) => s === "validate")).toHaveLength(1); - expect(ran.filter((s) => s === "reserve")).toHaveLength(3); - expect(ran).toContain("confirm"); - expect(ran).toContain("send-receipt"); - }, 30_000); - - test("resumeWorkflowRun throws when the run is not in failed status", async () => { - const backend = await createTestBackend(); - const client = new OpenWorkflow({ backend }); - - const workflow = client.defineWorkflow( - { name: `resume-invalid-${randomUUID()}` }, - async ({ step }) => { - return await step.run({ name: "noop" }, () => "ok"); - }, - ); - - const handle = await workflow.run(); - - await expect( - backend.resumeWorkflowRun({ workflowRunId: handle.workflowRun.id }), - ).rejects.toThrow(/Cannot resume workflow run.*pending/); - }); - - test("resumeWorkflowRun throws when the run does not exist", async () => { - const backend = await createTestBackend(); - - await expect( - backend.resumeWorkflowRun({ workflowRunId: randomUUID() }), - ).rejects.toThrow(/does not exist/); - }); }); describe("executeWorkflow", () => { @@ -4514,7 +4307,6 @@ function createMockWorkflowRun( deadlineAt: null, startedAt: new Date("2026-01-01T00:00:00.000Z"), finishedAt: null, - resumedAt: null, createdAt: new Date("2026-01-01T00:00:00.000Z"), updatedAt: new Date("2026-01-01T00:00:00.000Z"), ...overrides, diff --git a/packages/openworkflow/worker/execution.ts b/packages/openworkflow/worker/execution.ts index aa492dd1..c4b8bd9e 100644 --- a/packages/openworkflow/worker/execution.ts +++ b/packages/openworkflow/worker/execution.ts @@ -1166,10 +1166,7 @@ async function executeWorkflowAttempt( backend, workflowRun.id, ); - const history = new StepHistory({ - attempts, - resumedAt: workflowRun.resumedAt, - }); + const history = new StepHistory({ attempts }); // Complete any elapsed sleep waits first, then park on the earliest // remaining running wait (sleep, signal, or child workflow). diff --git a/packages/openworkflow/worker/step-history.test.ts b/packages/openworkflow/worker/step-history.test.ts index c7f4c013..374a6bb3 100644 --- a/packages/openworkflow/worker/step-history.test.ts +++ b/packages/openworkflow/worker/step-history.test.ts @@ -151,45 +151,6 @@ describe("StepHistory", () => { expect(history.findRunning("a")).toBeUndefined(); }); - test("failures before resumedAt are excluded from the retry budget", () => { - const resumedAt = new Date("2026-01-01T12:00:00.000Z"); - const beforeResume = createMockStepAttempt({ - stepName: "a", - status: "failed", - finishedAt: new Date("2026-01-01T11:59:59.000Z"), - }); - const afterResume = createMockStepAttempt({ - stepName: "a", - status: "failed", - finishedAt: new Date("2026-01-01T12:00:01.000Z"), - }); - - const history = new StepHistory({ - attempts: [beforeResume, afterResume], - resumedAt, - }); - - // only the post-resume failure counts, but both rows remain history - expect(history.failedAttemptCount("a")).toBe(1); - }); - - test("without resumedAt every failure counts", () => { - const first = createMockStepAttempt({ - stepName: "a", - status: "failed", - finishedAt: new Date("2026-01-01T11:00:00.000Z"), - }); - const second = createMockStepAttempt({ - stepName: "a", - status: "failed", - finishedAt: new Date("2026-01-01T12:00:00.000Z"), - }); - - const history = new StepHistory({ attempts: [first, second] }); - - expect(history.failedAttemptCount("a")).toBe(2); - }); - test("replaceRunningAttempt updates the running entry in place", () => { const initial = createMockStepAttempt({ id: "attempt-1", diff --git a/packages/openworkflow/worker/step-history.ts b/packages/openworkflow/worker/step-history.ts index d34d734c..9e28c5ad 100644 --- a/packages/openworkflow/worker/step-history.ts +++ b/packages/openworkflow/worker/step-history.ts @@ -37,13 +37,10 @@ export interface StepExecutionState { /** * Build step execution state from loaded attempts in one pass. * @param attempts - Loaded step attempts for the workflow run - * @param resumedAt - Most recent resume timestamp; failures that finished - * before it are kept as history but not counted against the retry budget * @returns Successful cache plus failed-attempt counts by step name */ export function createStepExecutionStateFromAttempts( attempts: readonly StepAttempt[], - resumedAt: Readonly | null = null, ): StepExecutionState { const cache = new Map(); const failedCountsByStepName = new Map(); @@ -57,14 +54,6 @@ export function createStepExecutionStateFromAttempts( } if (attempt.status === "failed") { - // Failures from before the latest resume stay in history (linkage, - // diagnostics) but don't count toward the step's retry budget. - if ( - resumedAt !== null && - (attempt.finishedAt === null || attempt.finishedAt < resumedAt) - ) { - continue; - } const previousCount = failedCountsByStepName.get(attempt.stepName) ?? 0; failedCountsByStepName.set(attempt.stepName, previousCount + 1); failedByStepName.set(attempt.stepName, attempt); @@ -181,7 +170,6 @@ function getEarliestRunningWait( export interface StepHistoryOptions { attempts: readonly StepAttempt[]; stepLimit?: number; - resumedAt?: Readonly | null; } /** @@ -205,10 +193,7 @@ export class StepHistory { this.stepLimit = Math.max(1, options.stepLimit ?? WORKFLOW_STEP_LIMIT); this.stepCount = options.attempts.length; - const state = createStepExecutionStateFromAttempts( - options.attempts, - options.resumedAt ?? null, - ); + const state = createStepExecutionStateFromAttempts(options.attempts); this.cache = new Map(state.cache); this.failedCountsByStepName = new Map(state.failedCountsByStepName); this.failedByStepName = new Map(state.failedByStepName); From b3753acb9a3af44eae0042938e941e3279a73ae4 Mon Sep 17 00:00:00 2001 From: James Martinez Date: Sun, 20 Sep 2026 17:19:55 -0500 Subject: [PATCH 8/9] fix(openworkflow): preserve step order across workflow reruns --- packages/openworkflow/client/rerun.test.ts | 96 ++++++++++--------- packages/openworkflow/core/rerun.ts | 99 +++++++++++++++++--- packages/openworkflow/postgres/backend.ts | 24 ++--- packages/openworkflow/sqlite/backend.ts | 49 +++++----- packages/openworkflow/worker/execution.ts | 6 +- packages/openworkflow/worker/step-history.ts | 8 +- 6 files changed, 175 insertions(+), 107 deletions(-) diff --git a/packages/openworkflow/client/rerun.test.ts b/packages/openworkflow/client/rerun.test.ts index 26fcbc0f..2bda8390 100644 --- a/packages/openworkflow/client/rerun.test.ts +++ b/packages/openworkflow/client/rerun.test.ts @@ -48,11 +48,8 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { const workflow = client.defineWorkflow<{ value: number }, number>( { name: "recover", version: "v1" }, async ({ step, input }) => { - const value = await step.run({ name: "read" }, async () => { + const value = await step.run({ name: "read" }, () => { calls.read++; - await new Promise((resolve) => { - setTimeout(resolve, 2); - }); return input.value; }); return await step.run( @@ -75,6 +72,8 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { const source = await finish(original.workflowRun.id); expect(source.status).toBe("failed"); const sourceHistory = await history(source.id); + const read = sourceHistory.find((step) => step.stepName === "read"); + assert.ok(read); const rerun = await client.rerunWorkflowRun(source.id, { fromStep: "write", }); @@ -93,11 +92,11 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { const copied = await history(rerun.id); expect(copied).toHaveLength(1); expect(copied[0]).toEqual({ - ...sourceHistory[0], + ...read, id: copied[0]?.id, workflowRunId: rerun.id, }); - expect(copied[0]?.id).not.toBe(sourceHistory[0]?.id); + expect(copied[0]?.id).not.toBe(read.id); await expect(finish(rerun.id)).resolves.toMatchObject({ status: "failed" }); expect(calls).toEqual({ read: 1, write: 4 }); @@ -109,6 +108,7 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { expect(calls).toEqual({ read: 1, write: 5 }); const fresh = await client.rerunWorkflowRun(recovered.id); expect(await history(fresh.id)).toEqual([]); + expect(fresh.context).toBeNull(); await expect(finish(fresh.id)).resolves.toMatchObject({ output: 42 }); expect(calls).toEqual({ read: 2, write: 6 }); expect(await backend.getWorkflowRun({ workflowRunId: source.id })).toEqual( @@ -117,7 +117,7 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { expect(await history(source.id)).toEqual(sourceHistory); }); - test("copies only successful attempts before the target's first attempt in timestamp and ID order", async () => { + test("copies successful steps by index", async () => { const source = await backend.createWorkflowRun({ workflowName: "snapshot", version: null, @@ -134,12 +134,14 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { await backend.claimWorkflowRun({ workerId, leaseDurationMs: 60_000 }); async function attempt( stepName: string, + stepIndex: number, status: "completed" | "failed" | "running", ) { const step = await backend.createStepAttempt({ workflowRunId: source.id, workerId, stepName, + stepIndex, kind: "function", config: {}, context: null, @@ -160,15 +162,16 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { }); return step; } - const failed = await attempt("caught", "failed"); - const running = await attempt("unfinished", "running"); - const first = await attempt("first", "completed"); - const second = await attempt("second", "completed"); - const third = await attempt("third", "completed"); - const target = await attempt("target", "failed"); - await attempt("later", "completed"); - await attempt("target", "completed"); - // Fix timestamps to exercise the ID tie-breaker on both databases. + await attempt("caught", 0, "failed"); + await attempt("unfinished", 1, "running"); + await attempt("first", 2, "failed"); + const first = await attempt("first", 2, "completed"); + const second = await attempt("second", 3, "completed"); + const third = await attempt("third", 4, "completed"); + await attempt("target", 5, "failed"); + await attempt("later", 6, "completed"); + await attempt("target", 5, "completed"); + // Earlier writes, tied timestamps, and replacement UUIDs cannot change the prefix. const early = new Date("2026-01-01T00:00:00Z"); const late = new Date("2026-01-02T00:00:00Z"); if (backend instanceof BackendSqlite) { @@ -177,33 +180,22 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { 'UPDATE "step_attempts" SET "created_at" = ? WHERE "workflow_run_id" = ?', ) .run(late.toISOString(), source.id); - for (const id of [ - failed.id, - running.id, - first.id, - second.id, - third.id, - target.id, - ]) { - backend["db"] - .prepare('UPDATE "step_attempts" SET "created_at" = ? WHERE "id" = ?') - .run(early.toISOString(), id); - } backend["db"] .prepare( - 'UPDATE "step_attempts" SET "created_at" = ?, "status" = \'succeeded\' WHERE "id" = ?', + 'UPDATE "step_attempts" SET "status" = \'succeeded\' WHERE "id" = ?', ) - .run(new Date(early.getTime() - 1).toISOString(), first.id); + .run(first.id); backend["db"] - .prepare('UPDATE "step_attempts" SET "created_at" = ? WHERE "id" = ?') - .run(new Date(early.getTime() + 1).toISOString(), target.id); + .prepare( + 'UPDATE "step_attempts" SET "created_at" = ? WHERE "workflow_run_id" = ? AND "step_name" = \'target\'', + ) + .run(early.toISOString(), source.id); } else { const pg = backend["pg"]; const table = backend["stepAttemptsTable"](); await pg`UPDATE ${table} SET "created_at" = ${late} WHERE "workflow_run_id" = ${source.id}`; - await pg`UPDATE ${table} SET "created_at" = ${early} WHERE "id" IN ${pg([failed.id, running.id, first.id, second.id, third.id])}`; - await pg`UPDATE ${table} SET "created_at" = ${new Date(early.getTime() - 1)}, "status" = 'succeeded' WHERE "id" = ${first.id}`; - await pg`UPDATE ${table} SET "created_at" = ${new Date(early.getTime() + 1)} WHERE "id" = ${target.id}`; + await pg`UPDATE ${table} SET "status" = 'succeeded' WHERE "id" = ${first.id}`; + await pg`UPDATE ${table} SET "created_at" = ${early} WHERE "workflow_run_id" = ${source.id} AND "step_name" = 'target'`; } await backend.cancelWorkflowRun({ workflowRunId: source.id }); const sourceHistory = await history(source.id); @@ -214,6 +206,9 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { const rerun = await client.rerunWorkflowRun(source.id, { fromStep: "target", }); + expect(rerun.context).toEqual({ + rerunStepIndices: { caught: 0, unfinished: 1 }, + }); const copies = await history(rerun.id); expect(copies.map((step) => step.stepName).toSorted()).toEqual( saved.map((step) => step.stepName).toSorted(), @@ -232,22 +227,20 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { }); } await client.cancelWorkflowRun(rerun.id); - const selected = copies[2]; - assert.ok(selected); const chained = await client.rerunWorkflowRun(rerun.id, { - fromStep: selected.stepName, + fromStep: "third", }); + expect(chained.context).toEqual(rerun.context); const chainedHistory = await history(chained.id); - expect(chainedHistory.map((step) => step.stepName).toSorted()).toEqual( - copies - .slice(0, 2) - .map((step) => step.stepName) - .toSorted(), - ); + expect(chainedHistory.map((step) => step.stepName).toSorted()).toEqual([ + "first", + "second", + ]); }); - test("runs caught failures again before the selected step", async () => { + test("preserves caught-failure order across chained reruns", async () => { let failures = 0; + let savedCalls = 0; const workflow = client.defineWorkflow( { name: "caught" }, async ({ step }) => { @@ -257,6 +250,7 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { throw new Error("caught"); }) .catch(() => null); + await step.run({ name: "saved" }, () => ++savedCalls); return await step.run({ name: "target" }, () => "ok"); }, ); @@ -265,9 +259,19 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { const rerun = await client.rerunWorkflowRun(source.workflowRun.id, { fromStep: "target", }); - expect(await history(rerun.id)).toEqual([]); + expect(rerun.context).toEqual({ rerunStepIndices: { error: 0 } }); await expect(finish(rerun.id)).resolves.toMatchObject({ output: "ok" }); expect(failures).toBe(2); + expect(savedCalls).toBe(1); + + const chained = await client.rerunWorkflowRun(rerun.id, { + fromStep: "error", + }); + expect(await history(chained.id)).toEqual([]); + expect(chained.context).toBeNull(); + await expect(finish(chained.id)).resolves.toMatchObject({ output: "ok" }); + expect(failures).toBe(3); + expect(savedCalls).toBe(2); }); test("replays saved signal results without deliveries or the source run", async () => { diff --git a/packages/openworkflow/core/rerun.ts b/packages/openworkflow/core/rerun.ts index 27b5bade..233cb010 100644 --- a/packages/openworkflow/core/rerun.ts +++ b/packages/openworkflow/core/rerun.ts @@ -2,41 +2,110 @@ import type { CreateWorkflowRunParams, RerunWorkflowRunParams, } from "./backend.js"; +import type { JsonValue } from "./json.js"; +import type { StepAttempt } from "./step-attempt.js"; import { isTerminalStatus, type WorkflowRun } from "./workflow-run.js"; +interface PreparedWorkflowRerun { + params: CreateWorkflowRunParams; + stepIndex: number | null; +} + /** * Validate a rerun and reuse the source input, version, and configuration. * @param source - Source run, read within the copying transaction * @param request - Rerun request - * @param stepId - First attempt of the requested step, or null - * @returns New run parameters + * @param steps - Source attempt metadata, read in the same transaction + * @returns New run parameters and the exclusive step-index boundary */ export function prepareWorkflowRerun( source: Readonly | null, request: Readonly, - stepId: string | null, -): CreateWorkflowRunParams { + steps: readonly Pick[], +): PreparedWorkflowRerun { if (!source) { throw new Error(`Workflow run ${request.workflowRunId} does not exist`); } if (!isTerminalStatus(source.status)) { throw new Error("Only finished workflow runs can be rerun"); } - if (request.fromStep !== null && stepId === null) { + const boundary = steps.find((step) => step.stepName === request.fromStep); + if (request.fromStep !== null && !boundary) { throw new Error( `Step "${request.fromStep}" does not exist in workflow run ${source.id}`, ); } + if (boundary && steps.some((step) => step.stepIndex === null)) { + throw new Error( + "Cannot rerun from a step without recorded step order; rerun the entire workflow instead", + ); + } + const stepIndex = boundary?.stepIndex ?? null; + let context = request.context; + if (stepIndex !== null) { + const stepIndices = getRerunStepIndices(source.context); + const completed = new Set(); + for (const step of steps) { + if (step.stepIndex !== null) { + stepIndices.set(step.stepName, step.stepIndex); + } + if (step.status === "completed" || step.status === "succeeded") { + completed.add(step.stepName); + } + } + for (const [name, index] of stepIndices) { + if (index >= stepIndex || completed.has(name)) stepIndices.delete(name); + } + if (stepIndices.size > 0) { + if (!isJsonObject(context)) context = {}; + context = { + ...context, + rerunStepIndices: Object.fromEntries(stepIndices), + }; + } + } return { - workflowName: source.workflowName, - version: source.version, - input: source.input, - config: source.config, - context: request.context, - idempotencyKey: null, - parentStepAttemptNamespaceId: null, - parentStepAttemptId: null, - availableAt: null, - deadlineAt: null, + stepIndex, + params: { + workflowName: source.workflowName, + version: source.version, + input: source.input, + config: source.config, + context, + idempotencyKey: null, + parentStepAttemptNamespaceId: null, + parentStepAttemptId: null, + availableAt: null, + deadlineAt: null, + }, }; } + +/** + * Read step order retained independently of copied successful attempts. + * @param context - Persisted workflow execution metadata + * @returns Recorded indices, including steps omitted from a rerun's history + */ +export function getRerunStepIndices(context: JsonValue): Map { + if (!isJsonObject(context)) { + return new Map(); + } + const indices = context["rerunStepIndices"]; + if (!isJsonObject(indices)) { + return new Map(); + } + return new Map( + Object.entries(indices).filter( + (entry): entry is [string, number] => + typeof entry[1] === "number" && + Number.isSafeInteger(entry[1]) && + entry[1] >= 0, + ), + ); +} + +function isJsonObject( + value: JsonValue | undefined, +): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} diff --git a/packages/openworkflow/postgres/backend.ts b/packages/openworkflow/postgres/backend.ts index bd314c29..66942b07 100644 --- a/packages/openworkflow/postgres/backend.ts +++ b/packages/openworkflow/postgres/backend.ts @@ -230,39 +230,33 @@ export class BackendPostgres implements Backend { FOR UPDATE `; const stepAttemptsTable = this.stepAttemptsTable(tx); - const [boundary] = + const steps = request.fromStep === null ? [] - : await tx<{ id: string }[]>` - SELECT "id" FROM ${stepAttemptsTable} + : await tx[]>` + SELECT "step_name", "step_index", "status" FROM ${stepAttemptsTable} WHERE "namespace_id" = ${this.namespaceId} AND "workflow_run_id" = ${request.workflowRunId} - AND "step_name" = ${request.fromStep} - ORDER BY "created_at", "id" - LIMIT 1 `; - const params = prepareWorkflowRerun( + const { params, stepIndex } = prepareWorkflowRerun( source ?? null, request, - boundary?.id ?? null, + steps, ); const run = await this.insertWorkflowRun(tx, params); - if (boundary) { + if (stepIndex !== null) { await tx` INSERT INTO ${stepAttemptsTable} ( - "namespace_id", "id", "workflow_run_id", "step_name", "kind", "status", + "namespace_id", "id", "workflow_run_id", "step_name", "step_index", "kind", "status", "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", "started_at", "finished_at", "created_at", "updated_at" ) - SELECT "namespace_id", gen_random_uuid(), ${run.id}, "step_name", "kind", "status", + SELECT "namespace_id", gen_random_uuid(), ${run.id}, "step_name", "step_index", "kind", "status", "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", "started_at", "finished_at", "created_at", "updated_at" FROM ${stepAttemptsTable} WHERE "namespace_id" = ${this.namespaceId} AND "workflow_run_id" = ${request.workflowRunId} AND "status" IN ('completed', 'succeeded') - AND ("created_at", "id") < ( - SELECT "created_at", "id" FROM ${stepAttemptsTable} - WHERE "namespace_id" = ${this.namespaceId} AND "id" = ${boundary.id} - ) + AND "step_index" < ${stepIndex} `; } return run; diff --git a/packages/openworkflow/sqlite/backend.ts b/packages/openworkflow/sqlite/backend.ts index 2a8c2df8..33732ac6 100644 --- a/packages/openworkflow/sqlite/backend.ts +++ b/packages/openworkflow/sqlite/backend.ts @@ -193,36 +193,38 @@ export class BackendSqlite implements Backend { ) .get(this.namespaceId, request.workflowRunId) as WorkflowRunRow | undefined; - // safety: the query selects the non-null step attempt ID. - const boundary = + // safety: the query selects step names, nullable indices, and statuses. + const steps = request.fromStep === null - ? undefined + ? [] : (this.db .prepare( ` - SELECT "id" FROM "step_attempts" - WHERE "namespace_id" = ? AND "workflow_run_id" = ? AND "step_name" = ? - ORDER BY "created_at", "id" - LIMIT 1 + SELECT "step_name", "step_index", "status" FROM "step_attempts" + WHERE "namespace_id" = ? AND "workflow_run_id" = ? `, ) - .get( - this.namespaceId, - request.workflowRunId, - request.fromStep, - ) as { id: string } | undefined); - const params = prepareWorkflowRerun( + .all(this.namespaceId, request.workflowRunId) as Pick< + StepAttemptRow, + "step_name" | "step_index" | "status" + >[]); + const { params, stepIndex } = prepareWorkflowRerun( source ? rowToWorkflowRun(source) : null, request, - boundary?.id ?? null, + steps.map((step) => ({ + stepName: step.step_name, + stepIndex: step.step_index, + // safety: backend transitions write domain status values. + status: step.status as StepAttempt["status"], + })), ); const run = this.insertWorkflowRun(params); - if (boundary) { + if (stepIndex !== null) { this.db .prepare( ` INSERT INTO "step_attempts" ( - "namespace_id", "id", "workflow_run_id", "step_name", "kind", "status", + "namespace_id", "id", "workflow_run_id", "step_name", "step_index", "kind", "status", "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", "started_at", "finished_at", "created_at", "updated_at" ) @@ -230,25 +232,16 @@ export class BackendSqlite implements Backend { lower(hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-4' || substr(hex(randomblob(2)), 2) || '-' || substr('89ab', (random() & 3) + 1, 1) || substr(hex(randomblob(2)), 2) || '-' || hex(randomblob(6))), - ?, "step_name", "kind", "status", + ?, "step_name", "step_index", "kind", "status", "config", "context", "output", "child_workflow_run_namespace_id", "child_workflow_run_id", "started_at", "finished_at", "created_at", "updated_at" FROM "step_attempts" WHERE "namespace_id" = ? AND "workflow_run_id" = ? AND "status" IN ('completed', 'succeeded') - AND ("created_at", "id") < ( - SELECT "created_at", "id" FROM "step_attempts" - WHERE "namespace_id" = ? AND "id" = ? - ) + AND "step_index" < ? `, ) - .run( - run.id, - this.namespaceId, - request.workflowRunId, - this.namespaceId, - boundary.id, - ); + .run(run.id, this.namespaceId, request.workflowRunId, stepIndex); } this.db.exec("COMMIT"); return run; diff --git a/packages/openworkflow/worker/execution.ts b/packages/openworkflow/worker/execution.ts index a290f2c3..6c40c536 100644 --- a/packages/openworkflow/worker/execution.ts +++ b/packages/openworkflow/worker/execution.ts @@ -6,6 +6,7 @@ import { type SerializedError, } from "../core/error.js"; import type { JsonValue } from "../core/json.js"; +import { getRerunStepIndices } from "../core/rerun.js"; import type { StandardSchemaV1 } from "../core/standard-schema.js"; import type { StepAttempt, @@ -1167,7 +1168,10 @@ async function executeWorkflowAttempt( backend, workflowRun.id, ); - const history = new StepHistory({ attempts }); + const history = new StepHistory({ + attempts, + stepIndices: getRerunStepIndices(workflowRun.context), + }); // Complete any elapsed sleep waits first, then park on the earliest // remaining running wait (sleep, signal, or child workflow). diff --git a/packages/openworkflow/worker/step-history.ts b/packages/openworkflow/worker/step-history.ts index 19304435..fc55f270 100644 --- a/packages/openworkflow/worker/step-history.ts +++ b/packages/openworkflow/worker/step-history.ts @@ -169,6 +169,7 @@ function getEarliestRunningWait( */ export interface StepHistoryOptions { attempts: readonly StepAttempt[]; + stepIndices?: ReadonlyMap; stepLimit?: number; } @@ -184,7 +185,7 @@ export class StepHistory { private readonly failedCountsByStepName: Map; private readonly failedByStepName: Map; private readonly runningByStepName: Map; - private readonly persistedStepIndices = new Map(); + private readonly persistedStepIndices: Map; private readonly resolvedStepNames = new Map(); private readonly expectedNextStepIndexByName = new Map(); private nextStepIndex = 0; @@ -194,11 +195,14 @@ export class StepHistory { constructor(options: Readonly) { this.stepLimit = Math.max(1, options.stepLimit ?? WORKFLOW_STEP_LIMIT); this.stepCount = options.attempts.length; + this.persistedStepIndices = new Map(options.stepIndices); for (const attempt of options.attempts) { if (attempt.stepIndex === null) continue; this.persistedStepIndices.set(attempt.stepName, attempt.stepIndex); - this.nextStepIndex = Math.max(this.nextStepIndex, attempt.stepIndex + 1); + } + for (const index of this.persistedStepIndices.values()) { + this.nextStepIndex = Math.max(this.nextStepIndex, index + 1); } const state = createStepExecutionStateFromAttempts(options.attempts); From f25767683f2b0078fc9ee82f72faf327c4976a99 Mon Sep 17 00:00:00 2001 From: James Martinez Date: Sun, 20 Sep 2026 17:33:04 -0500 Subject: [PATCH 9/9] test(openworkflow): coverage --- packages/openworkflow/client/rerun.test.ts | 60 +++++++++++++++++++++- packages/openworkflow/core/rerun.test.ts | 46 +++++++++++++++++ packages/openworkflow/core/rerun.ts | 24 ++++----- 3 files changed, 116 insertions(+), 14 deletions(-) create mode 100644 packages/openworkflow/core/rerun.test.ts diff --git a/packages/openworkflow/client/rerun.test.ts b/packages/openworkflow/client/rerun.test.ts index 2bda8390..fdf6e293 100644 --- a/packages/openworkflow/client/rerun.test.ts +++ b/packages/openworkflow/client/rerun.test.ts @@ -203,12 +203,17 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { (step) => step.id === first.id || step.id === second.id || step.id === third.id, ); - const rerun = await client.rerunWorkflowRun(source.id, { + const context = { traceContext: { traceparent: "new-trace" } }; + const rerun = await backend.rerunWorkflowRun({ + workflowRunId: source.id, fromStep: "target", + context, }); expect(rerun.context).toEqual({ + ...context, rerunStepIndices: { caught: 0, unfinished: 1 }, }); + expect(context).toEqual({ traceContext: { traceparent: "new-trace" } }); const copies = await history(rerun.id); expect(copies.map((step) => step.stepName).toSorted()).toEqual( saved.map((step) => step.stepName).toSorted(), @@ -230,7 +235,9 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { const chained = await client.rerunWorkflowRun(rerun.id, { fromStep: "third", }); - expect(chained.context).toEqual(rerun.context); + expect(chained.context).toEqual({ + rerunStepIndices: { caught: 0, unfinished: 1 }, + }); const chainedHistory = await history(chained.id); expect(chainedHistory.map((step) => step.stepName).toSorted()).toEqual([ "first", @@ -488,4 +495,53 @@ describe.each(["sqlite", "postgres"])("reruns (%s)", (database) => { }); expect(rerun.status).toBe("pending"); }); + + test.each(["prefix", "target"])( + "rejects partial reruns when %s has no recorded order, but permits a full rerun", + async (legacyStep) => { + let calls = 0; + const workflow = client.defineWorkflow( + { name: "legacy-step-order" }, + async ({ step }) => { + await step.run({ name: "prefix" }, () => ++calls); + return await step.run({ name: "target" }, () => ++calls); + }, + ); + const original = await workflow.run(); + const source = await finish(original.workflowRun.id); + expect(source.status).toBe("completed"); + if (backend instanceof BackendSqlite) { + backend["db"] + .prepare( + 'UPDATE "step_attempts" SET "step_index" = NULL WHERE "workflow_run_id" = ? AND "step_name" = ?', + ) + .run(source.id, legacyStep); + } else { + const pg = backend["pg"]; + await pg`UPDATE ${backend["stepAttemptsTable"]()} SET "step_index" = NULL + WHERE "workflow_run_id" = ${source.id} AND "step_name" = ${legacyStep}`; + } + const sourceHistory = await history(source.id); + const before = await backend.listWorkflowRuns({}); + + await expect( + client.rerunWorkflowRun(source.id, { fromStep: "target" }), + ).rejects.toThrow( + "Cannot rerun from a step without recorded step order; rerun the entire workflow instead", + ); + expect(await backend.listWorkflowRuns({})).toEqual(before); + + const rerun = await client.rerunWorkflowRun(source.id); + expect(await history(rerun.id)).toEqual([]); + await expect(finish(rerun.id)).resolves.toMatchObject({ + status: "completed", + output: 4, + }); + expect(calls).toBe(4); + expect( + await backend.getWorkflowRun({ workflowRunId: source.id }), + ).toEqual(source); + expect(await history(source.id)).toEqual(sourceHistory); + }, + ); }); diff --git a/packages/openworkflow/core/rerun.test.ts b/packages/openworkflow/core/rerun.test.ts new file mode 100644 index 00000000..2e38206a --- /dev/null +++ b/packages/openworkflow/core/rerun.test.ts @@ -0,0 +1,46 @@ +import type { JsonValue } from "./json.js"; +import { getRerunStepIndices } from "./rerun.js"; +import { describe, expect, test } from "vitest"; + +describe("getRerunStepIndices", () => { + test.each<{ context: JsonValue }>([ + { context: null }, + { context: "legacy context" }, + { context: [] }, + { context: {} }, + { context: { rerunStepIndices: null } }, + { context: { rerunStepIndices: [0, 1] } }, + { context: { rerunStepIndices: "invalid" } }, + ])( + "ignores absent or malformed step-order metadata: $context", + ({ context }) => { + expect(getRerunStepIndices(context)).toEqual(new Map()); + }, + ); + + test("retains only nonnegative safe integer indices without changing the context", () => { + const indices = { + first: 0, + later: 5, + largest: Number.MAX_SAFE_INTEGER, + negative: -1, + fractional: 1.5, + unsafe: Number.MAX_SAFE_INTEGER + 1, + numericString: "2", + missing: null, + nested: { index: 3 }, + }; + const context = { rerunStepIndices: { ...indices } }; + const result = getRerunStepIndices(context); + + expect(result).toEqual( + new Map([ + ["first", 0], + ["later", 5], + ["largest", Number.MAX_SAFE_INTEGER], + ]), + ); + result.set("first", 10); + expect(context.rerunStepIndices).toEqual(indices); + }); +}); diff --git a/packages/openworkflow/core/rerun.ts b/packages/openworkflow/core/rerun.ts index 233cb010..2c310493 100644 --- a/packages/openworkflow/core/rerun.ts +++ b/packages/openworkflow/core/rerun.ts @@ -35,24 +35,24 @@ export function prepareWorkflowRerun( `Step "${request.fromStep}" does not exist in workflow run ${source.id}`, ); } - if (boundary && steps.some((step) => step.stepIndex === null)) { - throw new Error( - "Cannot rerun from a step without recorded step order; rerun the entire workflow instead", - ); - } - const stepIndex = boundary?.stepIndex ?? null; - let context = request.context; - if (stepIndex !== null) { - const stepIndices = getRerunStepIndices(source.context); - const completed = new Set(); + const stepIndices = getRerunStepIndices(source.context); + const completed = new Set(); + if (boundary) { for (const step of steps) { - if (step.stepIndex !== null) { - stepIndices.set(step.stepName, step.stepIndex); + if (step.stepIndex === null) { + throw new Error( + "Cannot rerun from a step without recorded step order; rerun the entire workflow instead", + ); } + stepIndices.set(step.stepName, step.stepIndex); if (step.status === "completed" || step.status === "succeeded") { completed.add(step.stepName); } } + } + const stepIndex = boundary?.stepIndex ?? null; + let context = request.context; + if (stepIndex !== null) { for (const [name, index] of stepIndices) { if (index >= stepIndex || completed.has(name)) stepIndices.delete(name); }