From cf6b2849bc444cdae3ca40cb5fac38e1697c5583 Mon Sep 17 00:00:00 2001 From: Stephen DeLorme Date: Sun, 6 Sep 2026 16:28:34 -0400 Subject: [PATCH] Recover dropped WebRTC routes on the existing player connection Signed-off-by: Stephen DeLorme --- README.md | 12 +++ scripts/mutate.mjs | 38 +++++++- scripts/recovery-check.ts | 137 ++++++++++++++++++++++++++++ scripts/simcheck.ts | 2 + src/net/browser.ts | 4 +- src/net/recovery.ts | 187 ++++++++++++++++++++++++++++++++++++++ src/net/signal.ts | 14 +++ src/net/webrtc.ts | 28 +++++- 8 files changed, 418 insertions(+), 4 deletions(-) create mode 100644 scripts/recovery-check.ts create mode 100644 src/net/recovery.ts diff --git a/README.md b/README.md index ccf28e8..fec1d75 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,18 @@ and offers RETRY — a fresh join on the same code — or PLAY SOLO. A host with policy, headless; `simcheck` walks it through a drop inside the grace, a recovery, a second outage that runs out, ICE failing, and the channel closing. +**A dropped route gets one automatic recovery window.** Signalling stays connected for the +life of the peer. Either browser can notice an outage, but only the joiner creates the ICE +restart offer, avoiding competing offers. The existing peer connection, data channel, and +seat stay in place. Restart messages are addressed to the established peer and carry an +attempt number; early candidates wait for their description, and lost offers/answers are +retried. After eight seconds without recovery the existing CONNECTION LOST / RETRY flow +takes over. Closing or leaving also closes recovery listeners and the joiner's signalling. +This can repair a lost route when another direct path is available; it does not provide a +TURN relay or keep a match alive after the host leaves. `net/recovery.ts` owns negotiation +and its deadline; `scripts/recovery-check.ts` exercises loss, ordering, peer binding and +cleanup as part of the full simulation suite. + **Choose the wing before it flies.** `net/lobby.ts` reserves a seat only after a valid protocol-v2 HELLO with a hull choice. Every waiting browser gets a versioned LOBBY roster: seat number, hull, human/AI pilot, and which seat is yours. Host hull changes and departures diff --git a/scripts/mutate.mjs b/scripts/mutate.mjs index 5053b72..6e39cee 100644 --- a/scripts/mutate.mjs +++ b/scripts/mutate.mjs @@ -29,6 +29,42 @@ import { readFileSync, writeFileSync } from 'node:fs' /** @type {{ name: string, file: string, from: string, to: string }[]} */ const MUTATIONS = [ + { + name: 'recovery offers reuse the old ICE credentials', + file: 'src/net/recovery.ts', + from: 'pc.createOffer({ iceRestart: true })', + to: 'pc.createOffer({ iceRestart: false })', + }, + { + name: 'a stranger can answer a recovery offer', + file: 'src/net/recovery.ts', + from: 'closed || message.from !== peer || message.to !== signal.pubkey', + to: 'closed || message.to !== signal.pubkey', + }, + { + name: 'lost recovery offers are never resent', + file: 'src/net/recovery.ts', + from: 'if (now() - sentAt >= 1000) resend()', + to: '/* no retry */', + }, + { + name: 'recovery waits forever instead of offering manual retry', + file: 'src/net/recovery.ts', + from: 'if (now() >= deadline) { fail(); return }', + to: '/* no deadline */', + }, + { + name: 'early restart candidates are discarded at remote description', + file: 'src/net/recovery.ts', + from: 'for (const candidate of early.get(generation) ?? [])', + to: 'for (const candidate of [])', + }, + { + name: 'recovery signalling listener survives channel cleanup', + file: 'src/net/recovery.ts', + from: ' stop()', + to: ' /* leave the subscription alive */', + }, /* ---- Wing reservations and launch --------------------------------------- */ { name: 'lobby ignores the requested hull', @@ -995,7 +1031,7 @@ function runSuite() { * If you added or removed checks on purpose, bump this in the same commit. If you did not, * something stopped running. */ -const EXPECTED_ASSERTIONS = 718 +const EXPECTED_ASSERTIONS = 737 const PASS_SUMMARY = 'All checks passed.' const SUMMARY = /check\(s\) failed\.$|All checks passed\.$/ diff --git a/scripts/recovery-check.ts b/scripts/recovery-check.ts new file mode 100644 index 0000000..9b97a42 --- /dev/null +++ b/scripts/recovery-check.ts @@ -0,0 +1,137 @@ +import { createIceRecovery } from '../src/net/recovery' +import type { Signal, SignalMessage } from '../src/net/signal' + +/** Exercise negotiation ordering and loss separately from the browser's ICE engine. */ +export async function testIceRecovery(check: (label: string, condition: boolean, detail?: string) => void) { + const flush = async () => { for (let i = 0; i < 30; i++) await Promise.resolve() } + function rig(initiator = true) { + let time = 0 + let listener: (message: SignalMessage) => void = () => {} + let removed = 0 + let failures = 0 + let offers = 0 + let answers = 0 + const states: string[] = [] + const sent: SignalMessage[] = [] + const applied: RTCSessionDescriptionInit[] = [] + const candidates: RTCIceCandidateInit[] = [] + let onCandidate: (event: RTCPeerConnectionIceEvent) => void = () => {} + const signal: Signal = { + pubkey: 'self', + async send(message) { sent.push({ ...message, from: 'self' } as SignalMessage) }, + listen(handler) { listener = handler; return () => { removed++; listener = () => {} } }, + close() {}, + } + const fake = { + iceConnectionState: 'disconnected', signalingState: 'stable', + localDescription: null as RTCSessionDescriptionInit | null, + async createOffer(options?: RTCOfferOptions) { + offers++ + return { type: 'offer', sdp: options?.iceRestart ? 'restart credentials' : 'old credentials' } as RTCSessionDescriptionInit + }, + async createAnswer() { answers++; return { type: 'answer', sdp: 'new answer' } as RTCSessionDescriptionInit }, + async setLocalDescription(description: RTCSessionDescriptionInit) { + fake.localDescription = description + fake.signalingState = description.type === 'offer' ? 'have-local-offer' : 'stable' + }, + async setRemoteDescription(description: RTCSessionDescriptionInit) { + applied.push(description) + fake.signalingState = description.type === 'offer' ? 'have-remote-offer' : 'stable' + }, + async addIceCandidate(candidate: RTCIceCandidateInit) { candidates.push(candidate) }, + addEventListener(_type: string, handler: typeof onCandidate) { onCandidate = handler }, + removeEventListener() { onCandidate = () => {} }, + } + const recovery = createIceRecovery({ + pc: fake as unknown as RTCPeerConnection, signal, peer: 'peer', initiator, + now: () => time, onIce: state => states.push(state), onFailure: () => { failures++ }, + }) + return { + recovery, sent, applied, candidates, fake, states, + receive: (message: SignalMessage) => listener(message), + candidate: (candidate: RTCIceCandidateInit) => onCandidate({ candidate: { toJSON: () => candidate } } as RTCPeerConnectionIceEvent), + tick: (value: number) => { time = value; recovery.poll() }, + counts: () => ({ removed, failures, offers, answers }), + } + } + console.log('\nAn ICE restart preserves negotiation identity and has a bounded fallback') + const client = rig() + client.recovery.ice('failed') + await flush() + check('failed ICE starts a restart offer before declaring the link lost', client.states[0] === 'disconnected' && client.counts().failures === 0 && client.sent.some(m => m.type === 'restart-offer' && m.sdp === 'restart credentials')) + client.recovery.ice('disconnected') + await flush() + check('repeated outage events keep one negotiation', client.counts().offers === 1) + client.receive({ type: 'restart-answer', from: 'stranger', to: 'self', generation: 1, sdp: 'forged' }) + client.receive({ type: 'restart-answer', from: 'peer', to: 'elsewhere', generation: 1, sdp: 'misaddressed' }) + client.receive({ type: 'restart-answer', from: 'peer', to: 'self', generation: 2, sdp: 'future' }) + await flush() + check('only the established peer and current addressed attempt can answer', client.applied.length === 0) + client.receive({ type: 'restart-ice', from: 'peer', to: 'self', generation: 1, candidate: { candidate: 'early' } }) + await flush() + check('early candidates wait for the matching remote description', client.candidates.length === 0) + const beforeRetry = client.sent.length + client.candidate({ candidate: 'local' }) + client.tick(1000) + check('lost offers and their candidates are retransmitted', client.sent.length >= beforeRetry + 3) + client.receive({ type: 'restart-answer', from: 'peer', to: 'self', generation: 1, sdp: 'answer' }) + await flush() + check('the answer installs once and drains early candidates', client.applied.length === 1 && client.candidates[0]?.candidate === 'early') + client.receive({ type: 'restart-answer', from: 'peer', to: 'self', generation: 1, sdp: 'duplicate' }) + await flush() + check('duplicate answers cannot overwrite a settled remote description', client.applied.length === 1) + client.fake.iceConnectionState = 'connected' + client.recovery.ice('connected') + client.tick(9000) + check('connected ICE cancels the old outage deadline', client.states.at(-1) === 'connected' && client.counts().failures === 0) + client.fake.iceConnectionState = 'failed' + client.recovery.ice('failed') + await flush() + check('a later outage uses a fresh attempt on the same connection', client.counts().offers === 2 && client.sent.some(m => m.type === 'restart-offer' && m.generation === 2)) + const previousCandidates = client.candidates.length + client.receive({ type: 'restart-ice', from: 'peer', to: 'self', generation: 1, candidate: { candidate: 'stale' } }) + await flush() + check('old-attempt candidates cannot enter the new negotiation', client.candidates.length === previousCandidates) + client.tick(16999) + check('the recovery gets the full eight-second grace', client.counts().failures === 0) + client.recovery.ice('disconnected') + client.tick(17000) + check('repeated disconnection cannot extend recovery indefinitely', client.counts().failures === 1 && client.states.at(-1) === 'failed') + const endedAt = client.sent.length + client.recovery.ice('disconnected') + client.candidate({ candidate: 'after close' }) + client.tick(30000) + await flush() + check('terminal failure removes signalling and stops retries', client.sent.length === endedAt && client.counts().removed === 1) + + const host = rig(false) + host.recovery.ice('disconnected') + await flush() + check('the host asks the joiner to offer instead of competing with it', host.counts().offers === 0 && host.sent[0]?.type === 'restart-request') + host.receive({ type: 'restart-ice', from: 'peer', to: 'self', generation: 1, candidate: { candidate: 'before offer' } }) + const offer: SignalMessage = { type: 'restart-offer', from: 'peer', to: 'self', generation: 1, sdp: 'offer' } + host.receive(offer) + host.receive(offer) + await flush() + check('duplicate offers install one description and one answer', host.applied.length === 1 && host.counts().answers === 1) + check('host candidates arriving before the offer are preserved', host.candidates[0]?.candidate === 'before offer') + const beforeDuplicate = host.sent.length + host.fake.iceConnectionState = 'connected' + host.recovery.ice('connected') + host.receive(offer) + await flush() + check('a lost answer can be resent even after the host reconnects', host.sent.length > beforeDuplicate && host.sent.at(-1)?.type === 'restart-answer') + host.recovery.close() + + const cancelled = rig() + cancelled.recovery.ice('disconnected') + cancelled.recovery.close() + await flush() + check('leaving during offer creation cannot publish a late negotiation', cancelled.sent.length === 0 && cancelled.counts().removed === 1) + + const requested = rig() + requested.receive({ type: 'restart-request', from: 'peer', to: 'self', generation: 1 }) + await flush() + check('a host-reported outage starts recovery on the joiner', requested.counts().offers === 1) + requested.recovery.close() +} diff --git a/scripts/simcheck.ts b/scripts/simcheck.ts index 08c99d2..710b0ee 100644 --- a/scripts/simcheck.ts +++ b/scripts/simcheck.ts @@ -12,6 +12,7 @@ */ import * as THREE from 'three' +import { testIceRecovery } from './recovery-check' import type { Audio } from '../src/core/audio' import type { Input, InputState } from '../src/core/input' import type { MatchResult, RunResult, SeatLine } from '../src/core/scores' @@ -7993,6 +7994,7 @@ testALinkThatDropsIsNoticed() testTheWingLaunchesItsReservations() testPredictedWeaponsAreOnlyPresentation() testClientWeaponsUnderLatency() +await testIceRecovery(check) console.log(failures === 0 ? '\nAll checks passed.' : `\n${failures} check(s) failed.`) process.exit(failures === 0 ? 0 : 1) diff --git a/src/net/browser.ts b/src/net/browser.ts index a985474..bf0b4d1 100644 --- a/src/net/browser.ts +++ b/src/net/browser.ts @@ -149,9 +149,11 @@ export async function joinMatch(game: Game, code: string, ship: ShipId, handlers } try { channel = await connectAsClient(signal, handlers.status, hooks) - } finally { + } catch (error) { signal.close() + throw error } + channel.onClose(() => signal.close()) if (!handlers.active()) { channel.close(); throw new Error('join cancelled') } const open = channel open.onClose(() => monitor.closed()) diff --git a/src/net/recovery.ts b/src/net/recovery.ts new file mode 100644 index 0000000..c2839f5 --- /dev/null +++ b/src/net/recovery.ts @@ -0,0 +1,187 @@ +import { LINK_GRACE_MS } from './link' +import type { RecoveryMessage, Signal } from './signal' + +type Outgoing = RecoveryMessage extends infer M ? M extends RecoveryMessage ? Omit : never : never + +/** Only the joiner offers, so simultaneous outages cannot produce offer glare. */ +export function createIceRecovery(options: { + pc: RTCPeerConnection + signal: Signal + peer: string + initiator: boolean + now: () => number + onIce: (state: RTCIceConnectionState) => void + onFailure: () => void +}) { + const { pc, signal, peer, initiator, now, onIce, onFailure } = options + let generation = 0 + let recovering = false + let closed = false + let remoteReady = false + let busy = false + let deadline = 0 + let sentAt = -Infinity + let outbound: Outgoing | null = null + let answer: Outgoing | null = null + let localCandidates: RTCIceCandidateInit[] = [] + const early = new Map() + + function send(message: Outgoing) { + if (!closed) void signal.send(message).catch(() => {}) // poll retries until the existing grace expires + } + + function resend(message = outbound) { + if (!message) return + sentAt = now() + send(message) + for (const candidate of localCandidates) { + send({ type: 'restart-ice', to: peer, generation, candidate }) + } + } + + function begin() { + if (recovering) return + recovering = true + remoteReady = false + deadline = now() + LINK_GRACE_MS + onIce('disconnected') + } + + function fail() { + if (closed) return + close() + onIce('failed') + onFailure() + } + + function settled() { + if (closed || !recovering || !remoteReady || busy || pc.signalingState !== 'stable') return + if (pc.iceConnectionState !== 'connected' && pc.iceConnectionState !== 'completed') return + recovering = false + outbound = null + onIce(pc.iceConnectionState) + } + + function prepare(next: number) { + generation = next + remoteReady = false + answer = null + localCandidates = [] + for (const key of early.keys()) if (key !== generation) early.delete(key) + } + + async function applyRemote(description: RTCSessionDescriptionInit) { + await pc.setRemoteDescription(description) + if (closed) return + remoteReady = true + for (const candidate of early.get(generation) ?? []) { + await pc.addIceCandidate(candidate).catch(() => {}) + } + early.delete(generation) + } + + async function offer() { + if (closed || busy || recovering) return + begin() + prepare(generation + 1) + busy = true + try { + const description = await pc.createOffer({ iceRestart: true }) + if (closed) return + await pc.setLocalDescription(description) + if (closed) return + outbound = { type: 'restart-offer', to: peer, generation, sdp: pc.localDescription!.sdp } + resend() + } catch { fail() } finally { busy = false } + } + + function request() { + if (recovering || closed) return + if (initiator) { void offer(); return } + begin() + outbound = { type: 'restart-request', to: peer, generation: generation + 1 } + resend() + } + + async function receive(message: RecoveryMessage) { + if (closed || message.from !== peer || message.to !== signal.pubkey) return + if (message.type === 'restart-request') { + if (!initiator) return + if (message.generation === generation + 1) request() + else if (recovering && message.generation === generation) resend() + return + } + if (message.type === 'restart-ice') { + if (message.generation < generation || message.generation > generation + (initiator ? 0 : 1)) return + if (message.generation === generation && remoteReady) { + await pc.addIceCandidate(message.candidate).catch(() => {}) + } else { + const queue = early.get(message.generation) ?? [] + if (queue.length < 64) queue.push(message.candidate) + early.set(message.generation, queue) + } + return + } + if (message.type === 'restart-offer') { + if (initiator) return + if (message.generation === generation) { if (answer) resend(answer); return } + if (message.generation !== generation + 1 || busy) return + begin() + prepare(message.generation) + busy = true + try { + await applyRemote({ type: 'offer', sdp: message.sdp }) + if (closed) return + const description = await pc.createAnswer() + if (closed) return + await pc.setLocalDescription(description) + if (closed) return + answer = outbound = { type: 'restart-answer', to: peer, generation, sdp: pc.localDescription!.sdp } + resend() + } catch { fail() } finally { busy = false } + settled() + return + } + if (!initiator || message.generation !== generation || remoteReady || busy || !recovering) return + busy = true + try { + await applyRemote({ type: 'answer', sdp: message.sdp }) + } catch { fail() } finally { busy = false } + settled() + } + + const stop = signal.listen(message => { + if (message.type.startsWith('restart-')) void receive(message as RecoveryMessage) + }) + const candidate = (event: RTCPeerConnectionIceEvent) => { + if (closed || generation === 0 || !event.candidate) return + const value = event.candidate.toJSON() + if (localCandidates.length < 64) localCandidates.push(value) + send({ type: 'restart-ice', to: peer, generation, candidate: value }) + } + pc.addEventListener('icecandidate', candidate) + + function close() { + if (closed) return + closed = true + stop() + pc.removeEventListener('icecandidate', candidate) + early.clear() + localCandidates = [] + } + + return { + ice(state: RTCIceConnectionState) { + if (closed) return + if (state === 'disconnected' || state === 'failed') { request(); return } + if (recovering && (state === 'connected' || state === 'completed')) { settled(); return } + onIce(state) + }, + poll() { + if (closed || !recovering) return + if (now() >= deadline) { fail(); return } + if (now() - sentAt >= 1000) resend() + }, + close, + } +} diff --git a/src/net/signal.ts b/src/net/signal.ts index ccc709e..3a6c456 100644 --- a/src/net/signal.ts +++ b/src/net/signal.ts @@ -28,6 +28,13 @@ export type SignalMessage = | { type: 'offer'; from: string; sdp: string } | { type: 'answer'; from: string; to: string; sdp: string } | { type: 'ice'; from: string; to: string; candidate: RTCIceCandidateInit } + | RecoveryMessage + +export type RecoveryMessage = { from: string; to: string; generation: number } & ( + | { type: 'restart-request' } + | { type: 'restart-offer' | 'restart-answer'; sdp: string } + | { type: 'restart-ice'; candidate: RTCIceCandidateInit } +) export interface Signal { readonly pubkey: string @@ -53,6 +60,13 @@ function wellFormed(m: unknown): m is SignalMessage { if (typeof m !== 'object' || m === null) return false const x = m as Record if (typeof x.from !== 'string') return false + if (typeof x.type === 'string' && x.type.startsWith('restart-')) { + if (typeof x.to !== 'string' || !Number.isSafeInteger(x.generation) || (x.generation as number) < 1) return false + if (x.type === 'restart-request') return true + if (x.type === 'restart-offer' || x.type === 'restart-answer') return typeof x.sdp === 'string' + if (x.type === 'restart-ice') return typeof x.candidate === 'object' && x.candidate !== null + return false + } if (x.type === 'offer') return typeof x.sdp === 'string' if (x.type === 'answer') return typeof x.sdp === 'string' && typeof x.to === 'string' if (x.type === 'ice') return typeof x.to === 'string' && typeof x.candidate === 'object' && x.candidate !== null diff --git a/src/net/webrtc.ts b/src/net/webrtc.ts index b101856..4be3e34 100644 --- a/src/net/webrtc.ts +++ b/src/net/webrtc.ts @@ -37,6 +37,7 @@ import type { Channel } from './channel' import type { Signal, SignalMessage } from './signal' +import { createIceRecovery } from './recovery' export function iceServers(): RTCIceServer[] { const servers: RTCIceServer[] = [{ urls: 'stun:stun.l.google.com:19302' }] @@ -109,6 +110,21 @@ export interface LinkHooks { onRoute?: (route: string) => void } +function recoverable( + pc: RTCPeerConnection, channel: Channel, signal: Signal, peer: string, + initiator: boolean, hooks: LinkHooks, +) { + pc.onicecandidate = null // recovery candidates carry an attempt number + const recovery = createIceRecovery({ + pc, signal, peer, initiator, now: () => performance.now(), + onIce: state => hooks.onIce?.(state), + onFailure: () => channel.close(), + }) + const timer = setInterval(() => recovery.poll(), 250) + channel.onClose(() => { clearInterval(timer); recovery.close() }) + return recovery +} + /** * The candidate pair the connection is using, as one line: * `host/udp 192.168.1.20:51234 ↔ srflx 203.0.113.9:3478`. Empty when the @@ -184,6 +200,8 @@ export function connectAsClient(signal: Signal, status: Status = () => {}, hooks const pc = new RTCPeerConnection({ iceServers: iceServers() }) const dc = pc.createDataChannel(DATA_CHANNEL_LABEL, DATA_CHANNEL_OPTIONS) let host: string | null = null + let recovery: ReturnType | undefined + const linkHooks: LinkHooks = { ...hooks, onIce: state => recovery ? recovery.ice(state) : hooks.onIce?.(state) } const queued: RTCIceCandidateInit[] = [] /** * The host's candidates may reach the relays before its answer does — relays @@ -237,7 +255,9 @@ export function connectAsClient(signal: Signal, status: Status = () => {}, hooks await signal.send({ type: 'offer', sdp: pc.localDescription!.sdp }) await answered status('connecting') - return watch(pc, Promise.resolve(dc), status, hooks) + const channel = await watch(pc, Promise.resolve(dc), status, linkHooks) + recovery = recoverable(pc, channel, signal, host!, true, hooks) + return channel })() return connected @@ -257,6 +277,8 @@ export function acceptAsHost( ): Promise { const pc = new RTCPeerConnection({ iceServers: iceServers() }) const peer = offer.from + let recovery: ReturnType | undefined + const linkHooks: LinkHooks = { ...hooks, onIce: state => recovery ? recovery.ice(state) : hooks.onIce?.(state) } pc.onicecandidate = (ev) => { if (ev.candidate) void signal.send({ type: 'ice', to: peer, candidate: ev.candidate.toJSON() }) @@ -283,7 +305,9 @@ export function acceptAsHost( await pc.setLocalDescription(await pc.createAnswer()) status('answer sent') await signal.send({ type: 'answer', to: peer, sdp: pc.localDescription!.sdp }) - return watch(pc, dc, status, hooks) + const channel = await watch(pc, dc, status, linkHooks) + recovery = recoverable(pc, channel, signal, peer, false, hooks) + return channel })() return connected