fix(solana): a settle whose outcome the gateway does not know is waited out, never "no charge"

src/tools/video.ts

@@ -1199,7 +1199,9 @@ Returns a permanent blockrun-hosted video URL (the gateway mirrors the asset to
         //    the Solana audio route (settled optimistically at POST) via the
         //    shared helper — and the failure came after: deadline, poll
         //    error, terminal failure, or a submit that never answered
-        //    ("unknown"). The ledger must carry it (finally releases the
+        //    ("unknown"). Also the Solana wallet rail's payment for a finished
+        //    clip whose settlement outcome the gateway reported as unknown
+        //    ("unknown"): booked as a maybe, never "no charge". The ledger must carry it (finally releases the
         //    reservation, so without this booking the cap silently rises by
         //    the clip price), and the one thing the caller must not do is
         //    "try again": that submits and bills a second job.
@@ -1209,7 +1211,10 @@ Returns a permanent blockrun-hosted video URL (the gateway mirrors the asset to
           const billedTo = account ? "the BlockRun account" : "the Solana wallet";
           const what = err.billing === "billed"
             ? `Video generation did not return a clip, but the job was billed to ${billedTo} when the gateway accepted it${err.jobId ? ` (job ${err.jobId})` : ""}.`
-            : `Video generation got no answer to its submit, so the job MAY have been accepted and billed to ${billedTo}.`;
+            // "unknown" is also the Solana rail's settling poll whose outcome
+            // the gateway itself reported as unknown (utils/solana-402.ts
+            // isSettleHeld), not only a submit that never answered.
+            : `Video generation did not return a clip, and the job MAY have been billed to ${billedTo}: a request carrying the payment got no definitive answer.`;
           const where = account ? "https://user.blockrun.ai/dashboard/activity" : `blockrun_wallet action:"report" or the wallet's recent transactions`;
           return {
             content: [{ type: "text", text: `${what} Check ${where} before doing anything else — a new blockrun_video call starts and bills a second job.\nError: ${errMsg}` }],

src/utils/api-key-call.ts

@@ -137,7 +137,8 @@ function connectionNeverOpened(err: unknown): boolean {
   return typeof cause?.code === "string" && NEVER_CONNECTED.has(cause.code);
 }
 
-function retryAfterMs(response: Response): number {
+/** Retry-After in ms (delta-seconds only), or 0 when absent/unreadable. */
+export function retryAfterMs(response: Response): number {
   const raw = response.headers.get("retry-after");
   const seconds = raw === null ? NaN : Number(raw.trim());
   return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : 0;

src/utils/solana-402.ts

@@ -56,7 +56,7 @@ import { fetchWithTimeout } from "./http.js";
 import { JobFailedError, pollTimeoutFor } from "./poll.js";
 import { resolveSolanaKey } from "./wallet.js";
 import { amountToUsd } from "./budget.js";
-import { BilledJobError } from "./api-key-call.js";
+import { BilledJobError, retryAfterMs } from "./api-key-call.js";
 import { createPollSigner, pollRouteFor, type PollSigner } from "./poll-ownership.js";
 
 const QUOTE_TIMEOUT_MS = 15_000;
@@ -192,6 +192,23 @@ function settleFailureReason(response: Response): string | undefined {
   }
 }
 
+/**
+ * A paid poll's 402 that leaves a payment's outcome OPEN: the gateway's settle
+ * timed out ("Payment settlement outcome unknown", Retry-After: 90) and it is
+ * holding the job's claim with that authorization recorded until it can check
+ * on chain, or another poll of this job holds the claim. The held payment may
+ * still land; the gateway's next answer after the hold is either the delivered
+ * clip (it landed) or a fresh settle (it did not). Not a refusal, so it must
+ * neither spend the re-sign budget nor ever be reported as "no charge".
+ */
+const SETTLE_HELD_REASONS = new Set([
+  "settlement in progress on a concurrent poll",
+  "prior settlement outcome not yet resolvable",
+]);
+function isSettleHeld(reason: string | undefined, body: Record<string, unknown>): boolean {
+  return body.error === "Payment settlement outcome unknown" || (reason !== undefined && SETTLE_HELD_REASONS.has(reason));
+}
+
 export interface SolanaPaidPostResult {
   data: Record<string, unknown>;
   /** Actual USD charged, from the 402 quote. Null when unparseable — callers fall back to their estimate. */
@@ -562,10 +579,18 @@ export async function solanaPaidAsyncPost(
   // drift apart: on a settled-at-submit route the money is gone whatever the
   // failure was, and the tool has to book it and name the job — the same
   // BilledJobError the account rail throws, for the same reason.
+  // Set once a payment for the finished job got an answer that leaves its
+  // outcome open (isSettleHeld). Sticky: this client never observes the hold
+  // resolve as "did not land", only as a delivery, so after it no give-up may
+  // say "no charge" — it is booked as a maybe, the safe direction.
+  let settleOutcomeUnknown = false;
+  const maybeChargedNote = `A payment for the finished job was sent and the gateway reported its settlement outcome as unknown, so it MAY have been charged — check the wallet's recent transactions before retrying.`;
   const giveUp = (core: string, uncharged: string): Error =>
     settledAtSubmit
       ? new BilledJobError(`${core} ${settledNote} ${reclaimNote}`, { paidUsd: original.paidUsd, jobId, billing: "billed" })
-      : new Error(`${core} ${uncharged} ${reclaimNote}`);
+      : settleOutcomeUnknown
+        ? new BilledJobError(`${core} ${maybeChargedNote} ${reclaimNote}`, { paidUsd: original.paidUsd, jobId, billing: "unknown" })
+        : new Error(`${core} ${uncharged} ${reclaimNote}`);
 
   let resignsLeft = maxReactiveResigns;
   let lastStatus = typeof submitData.status === "string" ? submitData.status : "queued";
@@ -597,6 +622,9 @@ export async function solanaPaidAsyncPost(
   // the first payment-signed poll after a fallback answer a response that
   // just arrived, not a job that needs more time.
   let pollNow = false;
+  // The gateway's Retry-After for the next poll, when it gave one that matters
+  // (a held settle); otherwise the poll interval.
+  let nextWaitMs: number | undefined;
   const fallBackToPaidPolling = (why: string) => {
     ownershipPolling = false;
     pollNow = true;
@@ -616,18 +644,27 @@ export async function solanaPaidAsyncPost(
   // model no throw after the submit can un-settle the transfer, so any throw
   // that is not already a BilledJobError becomes one; payment-on-completion
   // routes keep their plain refusals untouched.
+  //
+  // The same conversion once a payment for the finished job has an OPEN
+  // outcome (settleOutcomeUnknown): a 500 poll answer, a challenge that
+  // cannot be read, a funding refusal of the fresh settle — none of them can
+  // prove the held transfer did not land, so every exit is booked as a maybe.
   try {
     return await pollUntilDone();
   } catch (err) {
-    if (!settledAtSubmit || err instanceof BilledJobError) throw err;
+    if (!(settledAtSubmit || settleOutcomeUnknown) || err instanceof BilledJobError) throw err;
     const msg = (err instanceof Error ? err.message : String(err)).replace(/\s*No charge was made\.?/i, "");
     throw giveUp(`${what} could not be collected: ${msg}`, "");
   }
 
   async function pollUntilDone(): Promise<SolanaPaidPostResult> {
   while (Date.now() < deadline) {
-    if (!pollNow) await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+    if (!pollNow) {
+      const waitMs = Math.min(nextWaitMs ?? pollIntervalMs, Math.max(0, deadline - Date.now()));
+      await new Promise((resolve) => setTimeout(resolve, waitMs));
+    }
     pollNow = false;
+    nextWaitMs = undefined;
 
     if (ownershipPolling && pollSigner && pollRoute) {
       const pollTimeout = pollTimeoutFor(deadline, Date.now(), pollTimeoutMs);
@@ -651,7 +688,15 @@ export async function solanaPaidAsyncPost(
           // ONE payment that settles it. Pinned to the submit's quote like
           // every re-sign — a finished job is no reason to pay more, or
           // someone else.
-          const fresh = parseSolanaChallenge(await readPaymentRequired(ownedResp, ownedData));
+          let fresh: SolanaPaymentContext;
+          try {
+            fresh = parseSolanaChallenge(await readPaymentRequired(ownedResp, ownedData));
+          } catch (err) {
+            // Nothing signed. The payment-signed path fetches and validates
+            // its own challenge, so the finished clip is still collected.
+            fallBackToPaidPolling(`the completion challenge could not be read (${err instanceof Error ? err.message : String(err)})`);
+            continue;
+          }
           const originalFeePayer = (original.details.extra as { feePayer?: string } | undefined)?.feePayer;
           const freshFeePayer = (fresh.details.extra as { feePayer?: string } | undefined)?.feePayer;
           if (
@@ -736,8 +781,26 @@ export async function solanaPaidAsyncPost(
       // The gateway's settle-failure 402 carries PAYMENT-RESPONSE (the reason),
       // not a fresh challenge; the challenge comes from a separate unpaid GET.
       // Its body is informational only — consume it to release the socket.
-      lastSettleReason = settleFailureReason(pollResp) ?? lastSettleReason;
-      await pollResp.json().catch(() => ({}));
+      const thisReason = settleFailureReason(pollResp);
+      lastSettleReason = thisReason ?? lastSettleReason;
+      const refusalBody = await pollResp.json().catch(() => ({})) as Record<string, unknown>;
+      // Classified on THIS answer's reason and body, never the remembered
+      // reason: the route's "Payment verification failed" 402 carries no
+      // PAYMENT-RESPONSE, and reading it as "still held" would re-sign a
+      // transaction every poll interval for the whole budget instead of
+      // taking the bounded re-sign path. Checked first: the body's error is
+      // the authoritative class of the unknown-outcome answer.
+      if (!settledAtSubmit && isSettleHeld(thisReason, refusalBody)) {
+        // Wait as told (the hold lasts up to ~90s), then poll again with a
+        // fresh blockhash. Safe from a double charge: the gateway lets one
+        // poll settle per job and checks on chain whether a held
+        // authorization landed before it settles anything new (sol #398).
+        // Bounded by the deadline, not by the re-sign budget.
+        settleOutcomeUnknown = true;
+        nextWaitMs = retryAfterMs(pollResp) || undefined;
+        nextResignAt = 0;
+        continue;
+      }
       if (isFundingFailure(lastSettleReason)) {
         // On a settled-at-submit route the poll never settles, so a funding
         // reason there is the delivery signature being refused — the charge
@@ -799,7 +862,7 @@ export async function solanaPaidAsyncPost(
   // needs a signed poll to observe "completed". An ownership-proof poll
   // carries no payment, so it is never the one in flight.
   const deadlineCore = `${what} did not complete within ${Math.round(pollBudgetMs / 1000)}s (last status: ${lastStatus}).`;
-  if (settledAtSubmit) throw giveUp(deadlineCore, "");
+  if (settledAtSubmit || settleOutcomeUnknown) throw giveUp(deadlineCore, "");
   throw new Error(
     paidPollInFlight
       ? `${deadlineCore} No settlement receipt was observed by this client, but a poll carrying the payment signature was still in flight at the deadline and can settle server-side, so check the wallet's recent transactions before retrying. ${reclaimNote}`

test/solana-poll-ownership.test.ts

@@ -325,3 +325,94 @@ test("pollOwnership:false keeps payment-signed polling from the first poll", asy
   assert.equal(hasProof(gets()[0]), false);
   assert.equal(gets()[0].headers["PAYMENT-SIGNATURE"], "signed-svm-payment-1");
 });
+
+// ---- the settling poll: an outcome the gateway itself does not know ----
+//
+// The gateway answers a settle that timed out with 402 "Payment settlement
+// outcome unknown" + Retry-After: 90 and HOLDS the job's claim with that
+// authorization recorded; until it resolves on chain, every other poll gets
+// "settlement in progress on a concurrent poll". That payment may still land.
+// Every finished video now reaches this path on its first paid poll, so the
+// helper must wait as told instead of burning its re-sign budget in seconds,
+// and must never report "no charge" while that outcome is open.
+const reasonHeader = (errorReason: string) => Buffer.from(JSON.stringify({ success: false, errorReason })).toString("base64");
+const outcomeUnknown = (retryAfter = "0.03") => ({
+  status: 402, ok: false,
+  headers: headers({ "payment-response": reasonHeader("facilitator_timeout"), "retry-after": retryAfter }),
+  json: async () => ({ error: "Payment settlement outcome unknown", status: "completed" }),
+});
+const heldByConcurrentPoll = () => ({
+  status: 402, ok: false,
+  headers: headers({ "payment-response": reasonHeader("settlement in progress on a concurrent poll") }),
+  json: async () => ({ error: "Payment Required" }),
+});
+const alreadySettled = () => ({ status: 200, ok: true, headers: headers(), json: async () => ({ status: "completed", data: [{ url: "u" }], payment: { status: "already_settled" } }) });
+
+test("video: an unknown settle outcome is waited out (Retry-After), not given up on after the re-sign budget", async () => {
+  const at: number[] = [];
+  const stamp = (f: () => unknown) => () => { at.push(Date.now()); return f(); };
+  script = [quote, videoSubmit, completion402, stamp(() => outcomeUnknown()), stamp(heldByConcurrentPoll), heldByConcurrentPoll, heldByConcurrentPoll, heldByConcurrentPoll, alreadySettled];
+  const r = await solanaPaidAsyncPost("/v1/videos/generations", { prompt: "t" }, { ...fast, maxReactiveResigns: 3 });
+  assert.equal((r.data.data as Array<{ url: string }>)[0].url, "u", "the held payment landed: delivered, not abandoned");
+  assert.ok(at[1] - at[0] >= 25, `waited the gateway's Retry-After before re-polling (${at[1] - at[0]}ms)`);
+  // Every request after the completion challenge carried a payment; no unpaid challenge fetches while held.
+  assert.ok(gets().slice(1).every((g) => typeof g.headers["PAYMENT-SIGNATURE"] === "string"));
+});
+
+test("video: a deadline while the settle outcome is unknown is booked as MAY have been charged, never 'no charge'", async () => {
+  const { BilledJobError } = await import("../src/utils/api-key-call.js");
+  script = [quote, videoSubmit, completion402, () => outcomeUnknown("0.02"), ...Array.from({ length: 400 }, () => heldByConcurrentPoll)];
+  await assert.rejects(solanaPaidAsyncPost("/v1/videos/generations", { prompt: "t" }, { ...fast, pollBudgetMs: 300, pollIntervalMs: 5 }), (err: Error) => {
+    assert.ok(err instanceof BilledJobError, `typed so the tool books it (got ${err.name}: ${err.message})`);
+    assert.equal((err as InstanceType<typeof BilledJobError>).billing, "unknown");
+    assert.equal((err as InstanceType<typeof BilledJobError>).paidUsd, 0.5);
+    assert.doesNotMatch(err.message, /no charge was made/i);
+    assert.match(err.message, /may/i);
+    return true;
+  });
+  script = [];
+});
+
+test("video: a completion 402 whose challenge cannot be read falls back to payment-signed polling, not an abort", async () => {
+  // No payment-required header and no accepts in the body: the paid path
+  // fetches and validates its own challenge, so the finished clip is still collected.
+  const unreadable = () => ({ status: 402, ok: false, headers: headers(), json: async () => ({ error: "Payment Required", status: "completed" }) });
+  script = [quote, videoSubmit, unreadable, settled];
+  const r = await solanaPaidAsyncPost("/v1/videos/generations", { prompt: "t" }, fast);
+  assert.equal(r.txHash, "solana-tx");
+  assert.deepEqual(gets().map(hasProof), [true, false]);
+  assert.equal(paymentsSigned, 2, "submit + one fresh signature for the paid poll");
+});
+
+test("video: after a hold, a 402 with no PAYMENT-RESPONSE is a refusal on the bounded re-sign path, not another hold", async () => {
+  // The route's "Payment verification failed" 402 carries no PAYMENT-RESPONSE.
+  // A sticky "last reason" from the earlier hold must not re-classify it as
+  // held: that would re-sign an SPL transaction every poll interval for the
+  // whole budget and never reach the re-sign ceiling.
+  const verificationFailed = () => ({ status: 402, ok: false, headers: headers(), json: async () => ({ error: "Payment verification failed" }) });
+  const freshChallenge = () => ({ status: 402, ok: false, headers: headers({ "payment-required": "fresh" }), json: async () => ({}) });
+  script = [
+    quote, videoSubmit, completion402, () => outcomeUnknown("0.01"), heldByConcurrentPoll,
+    verificationFailed, freshChallenge, verificationFailed, freshChallenge, verificationFailed,
+  ];
+  await assert.rejects(solanaPaidAsyncPost("/v1/videos/generations", { prompt: "t" }, { ...fast, maxReactiveResigns: 2 }), (err: Error) => {
+    assert.match(err.message, /did not go through after 2 re-signs/);
+    assert.doesNotMatch(err.message, /no charge was made/i, "a held payment may have landed");
+    return true;
+  });
+  // Two unpaid challenge fetches = the two reactive re-signs, then the ceiling.
+  assert.equal(gets().filter((g) => !hasProof(g) && g.headers["PAYMENT-SIGNATURE"] === undefined).length, 2);
+});
+
+test("video: every give-up after a hold is booked as MAY have been charged — a 500 poll answer included", async () => {
+  const { BilledJobError } = await import("../src/utils/api-key-call.js");
+  const serverError = () => ({ status: 500, ok: false, headers: headers(), json: async () => ({ error: "Internal server error" }) });
+  script = [quote, videoSubmit, completion402, () => outcomeUnknown("0.01"), serverError];
+  await assert.rejects(solanaPaidAsyncPost("/v1/videos/generations", { prompt: "t" }, fast), (err: Error) => {
+    assert.ok(err instanceof BilledJobError, `got ${err.name}: ${err.message}`);
+    assert.equal((err as InstanceType<typeof BilledJobError>).billing, "unknown");
+    assert.match(err.message, /poll error 500/);
+    assert.match(err.message, /MAY have been charged/);
+    return true;
+  });
+});