From 6f2176b47584e8916d69ba0a0d8999802f1e4f79 Mon Sep 17 00:00:00 2001 From: Evan Brown <153393779+etgbrown@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:27:05 -0600 Subject: [PATCH 1/2] fix(streamer): run FEC gap waits side by side, not one after another The FEC reorder queue started a block's 52 ms retransmission wait, and sent its first NACK, only when the block reached the head of the queue. A loss that damaged N blocks therefore held the queue for N x 52 ms, and with it the reference-invalidation request (queued when the next frame assembles) and the recovery frame the server sends in reply. The backlog was then released at once, which overflowed the decoder hand-off and cost another IDR round trip. A block's wait now starts when a packet from a later block arrives, and a hole's when the block behind it arrives. All waits run in parallel: - every gap is requested as soon as it is evident, with up to eight NACK reports per 4 ms tick instead of one; - a stalled block asks for every missing data run it needs to reconstruct, because its wait is no longer extended for a second round; - giving up on a gap forgets only that gap's NACK requests instead of all; - the queue reports one discontinuity per pass, so a complete frame between two abandoned gaps is delivered instead of discarded. Evidence, five 2-minute 3840x2160@120 AV1 sessions on a link that loses packets every 15 s (48 loss episodes): 21 gave up on two or more gaps in series, 52 ms apart, before the first invalidation left. The six longest (0.65-1.5 s) gave up on 15 to 66 gaps each, and in five of them a recovery keyframe was released 4-52 ms before another keyframe that the server had sent 0.3-0.5 s after it: both had been waiting in the queue. The plain reorder queue (streams without an FEC layout) is unchanged. Co-Authored-By: Claude Fable 5.2 Co-Authored-By: Claude Opus 5.5 --- .../streamer-comparison/transport-recovery.md | 20 ++ .../opennow-streamer-transport/src/nvst.rs | 273 ++++++++++++----- .../src/nvst_recovery_tests.rs | 290 +++++++++++++++++- 3 files changed, 501 insertions(+), 82 deletions(-) diff --git a/docs/streamer-comparison/transport-recovery.md b/docs/streamer-comparison/transport-recovery.md index 23ac18f57..712aed6dd 100644 --- a/docs/streamer-comparison/transport-recovery.md +++ b/docs/streamer-comparison/transport-recovery.md @@ -43,6 +43,26 @@ buffered successors to proceed through the existing discontinuity/RFI path. TR13 cites **D** macOS timed source-queue operations and **L** Windows `NACK waiting queue limit time: 52 ms`. +FEC blocks wait for retransmission side by side. A block's 52 ms wait starts +when a packet from a later block arrives, and a hole's when the block behind it +arrives, not when the gap reaches the head of the queue. Each gap is requested +as soon as it is evident, with up to eight NACK reports per 4 ms tick, and a +stalled block asks for every missing data run it needs to reconstruct, because +its wait is not extended for a second round. Giving up on one gap forgets only +that gap's requests. The queue reports one discontinuity at a time, so a +complete frame between two abandoned gaps is still delivered. + +Before this, a damaged block started a fresh 52 ms wait, and its first NACK, +only after the block ahead of it had been resolved. A loss that damaged N +blocks held the queue for N × 52 ms, together with the invalidation request +(queued when the next frame assembles) and the recovery frame the server sends +in reply. In five 2-minute 3840×2160 @ 120 AV1 sessions over a link that drops +packets every 15 s, 21 of 48 loss episodes gave up on two or more gaps in +series before the first invalidation left, and the six longest episodes +(0.65–1.5 s from the first gap to a usable frame) gave up on 15 to 66 gaps +each. The plain reorder queue, used only when a stream carries no FEC layout, +still waits per gap in sequence. + Unified-socket Mjolnir calls the same produced-frame watchdog and publishes the same typed stall and recovery events as the dedicated receiver. Authenticated partial video can no longer keep that route frozen indefinitely while resetting diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs index aac36b959..fc0488a7d 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs @@ -95,6 +95,9 @@ const RTCP_SENDER_SSRC: u32 = 0x4f4e_4f57; // "ONOW" const SRTCP_RR_INTERVAL: Duration = Duration::from_secs(1); const RTCP_RECOVERY_INTERVAL: Duration = Duration::from_millis(4); const NACK_RETRY_INTERVAL: Duration = Duration::from_millis(4); +// Gaps in separate FEC blocks are requested side by side but are not contiguous. One report +// per recovery tick would stagger them by 4 ms each inside a 52 ms wait. +const MAX_NACK_REPORTS_PER_TICK: usize = 8; const DEFAULT_NACK_RTT: Duration = Duration::from_millis(30); const NACK_TRACKING_TIMEOUT: Duration = Duration::from_millis(52); const MAX_NACK_ATTEMPTS: u8 = 3; @@ -1375,6 +1378,16 @@ impl NvstFeedbackState { .unwrap_or_else(std::sync::PoisonError::into_inner) .clear(); } + + /// Drops requests the reorder queue has given up on. Later gaps keep their own deadline + /// and attempt count instead of being requested again from scratch. + fn forget_nacks_through(&self, last_abandoned_index: u64) { + let mut pending = self + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *pending = pending.split_off(&last_abandoned_index.saturating_add(1)); + } } /// Shared handle to the NVST feedback plane (cheap to clone, shared across threads). @@ -3069,6 +3082,7 @@ fn build_rtcp_nack( packet } +/// Returns whether a report was sent, so the caller can send the next pending run. fn send_pending_nack( feedback: &NvstFeedbackState, now: Instant, @@ -3077,20 +3091,20 @@ fn send_pending_nack( mjolnir: bool, sender_ssrc: u32, media_ssrc: u32, -) { +) -> bool { let Some(channel) = (if mjolnir { Some(channels.control_partial) } else { channels.rtcp }) else { - return; + return false; }; if rtc.channel(channel).is_none() { - return; + return false; } let retry_rtt = mjolnir.then(|| feedback.ping(now).unwrap_or(DEFAULT_NACK_RTT)); let Some((first, last)) = feedback.take_nack(now, retry_rtt) else { - return; + return false; }; let admitted = if mjolnir { let missing: Vec = (first..=last).map(|index| index as u16).collect(); @@ -3112,6 +3126,7 @@ fn send_pending_nack( } else { feedback.mark_nack_send_failed(first, last); } + admitted } fn send_pending_frame_invalidation( @@ -3654,6 +3669,8 @@ struct RtpReorderBuffer { struct ReorderResult { ready: Vec, nack: Option<(u64, u64)>, + /// Gaps behind the head of the FEC queue, requested as soon as they are evident. + later_nacks: Vec<(u64, u64)>, recovery: Option, dropped: Option, fec_repaired: usize, @@ -3682,6 +3699,7 @@ impl RtpReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: Some(NvstDropReason::StaleRtpPacket { index }), fec_repaired: 0, @@ -3691,6 +3709,7 @@ impl RtpReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: Some(NvstDropReason::DuplicateRtpPacket { index }), fec_repaired: 0, @@ -3767,6 +3786,7 @@ impl RtpReorderBuffer { ReorderResult { ready, nack, + later_nacks: Vec::new(), recovery, dropped: None, fec_repaired: 0, @@ -3817,6 +3837,12 @@ struct FecBlock { base_index: u64, shards: Vec>, repair_failed: bool, + /// When the block's first packet arrived, or for a block that fills a hole behind buffered + /// successors, when the first of those arrived. A hole in front of it was evident by then. + arrived_at: Instant, + /// When a later block first arrived, or reconstruction failed. From then on, whatever + /// this block still lacks was lost rather than delayed, and its NACK wait is running. + stalled_since: Option, } struct Gf256Tables { @@ -4036,16 +4062,22 @@ fn reconstruct_nvst_cauchy_data( } impl FecBlock { - fn new(layout: FecPacketLayout, packet_index: u64) -> Self { + fn new(layout: FecPacketLayout, packet_index: u64, now: Instant) -> Self { let total = layout.data_shards + layout.parity_shards; Self { layout, base_index: packet_index.saturating_sub(layout.shard_index as u64), shards: (0..total).map(|_| None).collect(), repair_failed: false, + arrived_at: now, + stalled_since: None, } } + fn end_index(&self) -> u64 { + self.base_index + (self.layout.data_shards + self.layout.parity_shards) as u64 + } + fn matches(&self, layout: FecPacketLayout, packet_index: u64) -> bool { self.layout.frame_index == layout.frame_index && self.layout.block_index == layout.block_index @@ -4083,6 +4115,33 @@ impl FecBlock { )) } + /// The missing data runs that retransmission has to fill before the block can be + /// reconstructed. Parity already received covers the rest, so a later run is included + /// only when the earlier ones leave the block short. The block's wait is not extended + /// for a second round, so everything it needs is requested at once. + fn missing_runs_to_complete(&self) -> Vec<(u64, u64)> { + let present = self.shards.iter().flatten().count(); + let mut shortfall = self.layout.data_shards.saturating_sub(present); + let mut runs = Vec::new(); + let mut shard = 0; + while shortfall > 0 && shard < self.layout.data_shards { + if self.shards[shard].is_some() { + shard += 1; + continue; + } + let first = shard; + while shard < self.layout.data_shards && self.shards[shard].is_none() { + shard += 1; + } + runs.push(( + self.base_index + first as u64, + self.base_index + shard as u64 - 1, + )); + shortfall = shortfall.saturating_sub(shard - first); + } + runs + } + fn insert(&mut self, layout: FecPacketLayout, packet: RtpPacket) -> Option { let slot = &mut self.shards[layout.shard_index]; if slot.is_some() { @@ -4230,7 +4289,6 @@ impl FecBlock { struct FecReorderBuffer { blocks: BTreeMap, completed_through: Option, - gap_wait: Option<(u64, Instant)>, shard_len: usize, } @@ -4239,7 +4297,6 @@ impl FecReorderBuffer { Self { blocks: BTreeMap::new(), completed_through: None, - gap_wait: None, shard_len: video_packet_size + NVST_FEC_RTP_HEADER_ALLOWANCE, } } @@ -4247,7 +4304,6 @@ impl FecReorderBuffer { fn reset(&mut self) { self.blocks.clear(); self.completed_through = None; - self.gap_wait = None; } fn push(&mut self, packet: RtpPacket, layout: FecPacketLayout, now: Instant) -> ReorderResult { @@ -4259,19 +4315,22 @@ impl FecReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: None, fec_repaired: 0, }; } + let created = !self.blocks.contains_key(&packet_base); let block = self .blocks .entry(packet_base) - .or_insert_with(|| FecBlock::new(layout, packet.index)); + .or_insert_with(|| FecBlock::new(layout, packet.index, now)); if !block.matches(layout, packet.index) { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: Some(NvstDropReason::Unsupported( NvstUnsupportedFeature::FecRepair, @@ -4284,54 +4343,88 @@ impl FecReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped, fec_repaired: 0, }; } - self.poll_deadline(now) + let later_nacks = if created { + self.start_gap_waits(packet_base, now) + } else { + Vec::new() + }; + let mut result = self.poll_deadline(now); + result.later_nacks = later_nacks; + result } + /// A packet from a new block proves that whatever the earlier blocks still lack was + /// skipped, not delayed. Their NACK waits start now and run side by side, so a burst that + /// damages several blocks holds the queue for one NACK window instead of one per block. + /// Returns the gaps to request at once. + fn start_gap_waits(&mut self, new_base: u64, now: Instant) -> Vec<(u64, u64)> { + let mut nacks = Vec::new(); + let mut expected = self.completed_through; + for block in self.blocks.range_mut(..new_base).map(|(_, block)| block) { + if block.stalled_since.is_none() { + block.stalled_since = Some(now); + nacks.append(&mut block.missing_runs_to_complete()); + } + expected = Some(block.end_index()); + } + if let Some(expected) = expected + && new_base > expected + && new_base - expected < MAX_REORDER_WINDOW as u64 + { + nacks.push((expected, new_base - 1)); + } + // A block that fills a hole behind buffered successors was overtaken when the first + // of them arrived. Neither it nor a hole still in front of it gets a fresh wait. + let overtaken_at = self + .blocks + .range(new_base + 1..) + .map(|(_, block)| block.arrived_at) + .min(); + if let (Some(overtaken_at), Some(block)) = (overtaken_at, self.blocks.get_mut(&new_base)) { + block.arrived_at = overtaken_at; + block.stalled_since = Some(overtaken_at); + } + nacks + } + + /// Reports at most one discontinuity per call, followed by the packets behind it. Waits + /// overlap, so several blocks can run out of time together; the caller polls again while + /// a call makes progress. Packets completed ahead of a discontinuity are delivered first. fn poll_deadline(&mut self, now: Instant) -> ReorderResult { let mut result = ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: None, fec_repaired: 0, }; - loop { - let Some((&base, head)) = self.blocks.first_key_value() else { - self.gap_wait = None; - break; - }; - + while let Some((&base, head)) = self.blocks.first_key_value() { if let Some(expected) = self.completed_through && base > expected { - result.nack = Some((expected, base - 1)); - let started = match self.gap_wait { - Some((waiting_for, started)) if waiting_for == expected => started, - _ => { - self.gap_wait = Some((expected, now)); - now - } - }; - if now.saturating_duration_since(started) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + if now.saturating_duration_since(head.arrived_at) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT && base.saturating_sub(expected) < MAX_REORDER_WINDOW as u64 && self.blocks.len() < MAX_PENDING_FEC_BLOCKS { + result.nack = Some((expected, base - 1)); + break; + } + if result.recovery.is_some() || !result.ready.is_empty() { break; } result.recovery = Some(NvstRecovery::PacketGap { first_missing_index: expected, last_missing_index: base - 1, }); - result.ready.clear(); - result.nack = None; self.completed_through = Some(base); - self.gap_wait = None; continue; } @@ -4340,8 +4433,7 @@ impl FecReorderBuffer { .blocks .remove(&base) .expect("head FEC block is present"); - let completed_through = - base + (completed.layout.data_shards + completed.layout.parity_shards) as u64; + let completed_through = completed.end_index(); // Only reconstruction can fail. Healthy blocks move their RTP // buffers straight to assembly without copying the video payload. let repair_backup = completed.missing_data_range().map(|_| completed.clone()); @@ -4350,12 +4442,12 @@ impl FecReorderBuffer { result.ready.append(&mut ready); result.fec_repaired += repaired; self.completed_through = Some(completed_through); - self.gap_wait = None; continue; } Err(reason) => { let mut pending = repair_backup.expect("failed repair has missing data"); pending.repair_failed = true; + pending.stalled_since = Some(now); let (first_missing, last_missing) = pending .first_missing_data_run() .expect("failed FEC reconstruction has missing data"); @@ -4371,37 +4463,29 @@ impl FecReorderBuffer { ); self.blocks.insert(base, pending); result.nack = Some((first_missing, last_missing)); - self.gap_wait = Some((first_missing, now)); break; } } } - // A later block proves that the head block has stopped arriving normally. Ask for - // only the first contiguous missing run and retain all subsequent blocks while the - // retransmission is in flight. - if self.blocks.len() == 1 && self.gap_wait.is_none() { + // Until a later block arrives, the head block is still arriving normally. After + // that, ask for the first contiguous missing run and retain all subsequent blocks + // while the retransmission is in flight. + let Some(stalled_since) = head.stalled_since else { break; - } - let (first_missing, last_missing) = head - .first_missing_data_run() - .expect("an incomplete FEC block has missing data"); - result.nack = Some((first_missing, last_missing)); - let started = match self.gap_wait { - Some((waiting_for, started)) if waiting_for == first_missing => started, - _ => { - self.gap_wait = Some((first_missing, now)); - now - } }; let block_span = self .blocks .last_key_value() .map_or(0, |(&last_base, _)| last_base.saturating_sub(base)); - if now.saturating_duration_since(started) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + if now.saturating_duration_since(stalled_since) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT && block_span < MAX_REORDER_WINDOW as u64 && self.blocks.len() < MAX_PENDING_FEC_BLOCKS { + result.nack = head.first_missing_data_run(); + break; + } + if result.recovery.is_some() || !result.ready.is_empty() { break; } @@ -4416,11 +4500,7 @@ impl FecReorderBuffer { first_missing_index, last_missing_index, }); - result.ready.clear(); - result.nack = None; - self.completed_through = - Some(base + (failed.layout.data_shards + failed.layout.parity_shards) as u64); - self.gap_wait = None; + self.completed_through = Some(failed.end_index()); // Emit any complete successors after the assembler has been marked discontinuous. } result @@ -4818,21 +4898,20 @@ impl NvstVideoReceiver { video, ); - let result = if let Some(layout) = FecPacketLayout::from_packet(&packet) { - if layout.shard_index >= layout.data_shards { - self.fec_packets += 1; - } - let result = self.fec_reorder.push(packet, layout, now); - if layout.shard_index < layout.data_shards - && matches!(result.dropped, Some(NvstDropReason::Unsupported(_))) - { - self.invalidate_picture(now); - } - result - } else { - self.reorder.push(packet, now) + let Some(layout) = FecPacketLayout::from_packet(&packet) else { + let result = self.reorder.push(packet, now); + return self.consume_reorder_result(result, now); }; - self.consume_reorder_result(result, now) + if layout.shard_index >= layout.data_shards { + self.fec_packets += 1; + } + let result = self.fec_reorder.push(packet, layout, now); + if layout.shard_index < layout.data_shards + && matches!(result.dropped, Some(NvstDropReason::Unsupported(_))) + { + self.invalidate_picture(now); + } + self.consume_fec_result(result, now) } pub fn poll_recovery(&mut self, now: Instant) -> Vec { @@ -4844,10 +4923,28 @@ impl NvstVideoReceiver { let reordered = self.reorder.poll_deadline(now); let mut events = self.consume_reorder_result(reordered, now); let fec = self.fec_reorder.poll_deadline(now); - events.extend(self.consume_reorder_result(fec, now)); + events.extend(self.consume_fec_result(fec, now)); events } + /// The FEC queue reports one discontinuity per pass, so that packets completed ahead of + /// it are delivered first. Keep polling while a pass makes progress. + fn consume_fec_result( + &mut self, + mut result: ReorderResult, + now: Instant, + ) -> Vec { + let mut events = Vec::new(); + loop { + let progressed = result.recovery.is_some() || !result.ready.is_empty(); + events.extend(self.consume_reorder_result(result, now)); + if !progressed { + return events; + } + result = self.fec_reorder.poll_deadline(now); + } + } + fn consume_reorder_result( &mut self, result: ReorderResult, @@ -4855,7 +4952,9 @@ impl NvstVideoReceiver { ) -> Vec { self.fec_repaired_packets += result.fec_repaired as u64; let mut events = Vec::new(); - if let Some((first_missing_index, last_missing_index)) = result.nack { + for (first_missing_index, last_missing_index) in + result.nack.into_iter().chain(result.later_nacks) + { self.config .feedback .request_nack(first_missing_index, last_missing_index, now); @@ -4869,7 +4968,15 @@ impl NvstVideoReceiver { if let Some(recovery) = result.recovery { self.packet_gap_recoveries += 1; self.invalidate_picture(now); - self.config.feedback.clear_nacks(); + match recovery { + NvstRecovery::PacketGap { + last_missing_index, .. + } => self + .config + .feedback + .forget_nacks_through(last_missing_index), + _ => self.config.feedback.clear_nacks(), + } events.push(NvstReceiveEvent::RecoveryNeeded(recovery)); } for packet in result.ready { @@ -8171,15 +8278,19 @@ fn run_nvst_webrtc_bundle( && let Some((media_ssrc, _)) = feedback.stream_snapshot() { if let Some(channels) = input_channels { - send_pending_nack( - &feedback, - now, - &mut rtc, - channels, - mjolnir, - rtcp_sender_ssrc, - media_ssrc, - ); + for _ in 0..MAX_NACK_REPORTS_PER_TICK { + if !send_pending_nack( + &feedback, + now, + &mut rtc, + channels, + mjolnir, + rtcp_sender_ssrc, + media_ssrc, + ) { + break; + } + } } last_recovery_send = now; } @@ -13584,7 +13695,7 @@ mod tests { plaintext, origin: RtpPacketOrigin::Authenticated, }; - let mut block = FecBlock::new(layout(0), base_index); + let mut block = FecBlock::new(layout(0), base_index, Instant::now()); assert_eq!(block.insert(layout(0), packet(0, data[0].clone())), None); assert_eq!(block.insert(layout(2), packet(2, data[2].clone())), None); assert_eq!(block.insert(layout(3), packet(3, data[3].clone())), None); @@ -13678,7 +13789,7 @@ mod tests { plaintext, origin: RtpPacketOrigin::Authenticated, }; - let mut block = FecBlock::new(layout(0), base_index); + let mut block = FecBlock::new(layout(0), base_index, Instant::now()); assert_eq!(block.insert(layout(1), packet(1, parity[0].clone())), None); assert!(block.has_enough_shards()); diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs index 7cac69fe7..83ef80047 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs @@ -57,9 +57,12 @@ fn multiple_reorder_gaps_do_not_revalidate_post_gap_references() { _ => None, }) .collect(); + // The FEC queue reports each gap separately, so the complete keyframe between the + // two gaps is delivered instead of being discarded with the second one. + let between_gaps: &[(u32, bool, bool)] = if fec { &[(2, true, false)] } else { &[] }; assert_eq!( frames, - [(100, false, false), (101, false, true)], + [between_gaps, &[(100, false, false), (101, false, true)]].concat(), "fec={fec}" ); assert!(receiver.config.feedback.keyframe_request_pending()); @@ -958,3 +961,288 @@ fn rfi_retries_preserve_the_episode_deadline_and_stop_on_recovery() { assert_eq!(stream.feedback.frame_invalidation.take(), None); assert!(!stream.feedback.keyframe_request_pending()); } + +/// One frame as a single FEC block of two data packets; its parity packet (index `3 * frame +/// + 2`) is never sent, so a frame that loses a data packet cannot be reconstructed. +fn typed_fec_frame(frame: u32, frame_type: u8, nal: u8) -> [Vec; 2] { + let base = (frame * 3) as u16; + let with_fec_word = |mut packet: Vec, shard: u32| { + packet[28..32].copy_from_slice(&((50_u32 << 4) | (shard << 12) | (2 << 22)).to_le_bytes()); + packet + }; + [ + with_fec_word( + build_plaintext_rtp( + base, + FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + frame, + &[0x01, 0, 0, frame_type, 0, 0, 0, 0, 0, 0, 0, 1, nal, 0x88], + ), + 0, + ), + with_fec_word( + build_plaintext_rtp(base + 1, FLAG_EOF | FLAG_CONTAINS_PIC_DATA, frame, &[0x99]), + 1, + ), + ] +} + +impl TypedStream { + fn fec_frame(&mut self, frame: u32, frame_type: u8, after: Duration) -> Vec<(u32, bool, bool)> { + let nal = if frame_type == 2 { 0x65 } else { 0x41 }; + let [first, last] = typed_fec_frame(frame, frame_type, nal); + let mut frames = self.frames(first, after); + frames.extend(self.frames(last, after)); + frames + } + + /// Delivers only the first packet of a P-frame. + fn damaged_fec_frame(&mut self, frame: u32, after: Duration) { + let [first, _] = typed_fec_frame(frame, 1, 0x41); + assert!(self.push(first, after).is_empty()); + } + + fn poll(&mut self, after: Duration) -> Vec { + self.receiver.poll_recovery(self.origin + after) + } +} + +fn gaps(events: &[NvstReceiveEvent]) -> Vec<(u64, u64)> { + events + .iter() + .filter_map(|event| match event { + NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { + first_missing_index, + last_missing_index, + }) => Some((*first_missing_index, *last_missing_index)), + _ => None, + }) + .collect() +} + +#[test] +fn consecutive_damaged_frames_share_one_nack_window() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + assert_eq!(stream.fec_frame(2, 1, ms(0)), [(2, false, true)]); + // Frames 3, 4 and 5 each lose a packet. Each loss is evident when the next frame starts. + stream.damaged_fec_frame(3, ms(0)); + stream.damaged_fec_frame(4, ms(1)); + stream.damaged_fec_frame(5, ms(2)); + assert!(stream.fec_frame(6, 1, ms(3)).is_empty()); + // All three retransmissions are requested at once, not one per expired wait. + let now = stream.origin + ms(3); + assert_eq!(stream.feedback.take_nack(now, None), Some((10, 10))); + assert_eq!(stream.feedback.take_nack(now, None), Some((13, 13))); + assert_eq!(stream.feedback.take_nack(now, None), Some((16, 16))); + + assert!(stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT).is_empty()); + assert_eq!( + gaps(&stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(1))), + [(10, 10)] + ); + // Serial waits would hold frame 6, and with it the invalidation request, until 158 ms. + let events = stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(3)); + assert_eq!(gaps(&events), [(13, 13), (16, 16)]); + assert!(delivered(events).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((3, 5))); + assert_eq!(stream.fec_frame(7, 2, ms(60)), [(7, true, true)]); + assert_eq!(stream.receiver.packet_gap_recoveries, 3); + assert_eq!( + stream.stats(), + "rfiSent=0 rfiDropped=1 rfiRecoveredI=1 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0 rfiIdrBackstop=0" + ); +} + +#[test] +fn a_recovery_frame_between_two_expired_gaps_is_delivered() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 2, ms(1)).is_empty()); + stream.damaged_fec_frame(4, ms(2)); + assert!(stream.fec_frame(5, 1, ms(3)).is_empty()); + // Both waits are over by the time the queue is polled. The I-frame sits between the + // two discontinuities and must reach the decoder. + let events = stream.poll(ms(200)); + assert_eq!(gaps(&events), [(7, 7), (13, 13)]); + assert_eq!(delivered(events), [(3, true, true)]); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((4, 4))); +} + +#[test] +fn giving_up_on_a_gap_keeps_the_requests_for_later_gaps() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + stream.damaged_fec_frame(4, ms(30)); + assert!(stream.fec_frame(5, 1, ms(31)).is_empty()); + let sent_at = stream.origin + ms(31); + assert_eq!(stream.feedback.take_nack(sent_at, None), Some((7, 7))); + assert_eq!(stream.feedback.take_nack(sent_at, None), Some((13, 13))); + + assert_eq!( + gaps(&stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(1))), + [(7, 7)] + ); + let pending = stream.feedback.pending_nacks.lock().unwrap(); + assert_eq!(pending.keys().copied().collect::>(), [13]); + assert_eq!(pending[&13].attempts, 1, "the request was not started over"); + assert_eq!(pending[&13].last_sent_at, Some(sent_at)); +} + +#[test] +fn fec_holes_wait_from_the_packet_that_revealed_them() { + let now = Instant::now(); + let ms = Duration::from_millis; + let mut reorder = FecReorderBuffer::new(48); + // One data and one parity packet per block: block `n` owns indices `2n` and `2n + 1`. + let data = |block: u64| { + let plaintext = build_plaintext_rtp( + (block * 2) as u16, + FLAG_SOF | FLAG_EOF, + block as u32, + &[0, 0, 0, 1, 0x65], + ); + let packet = RtpPacket { + index: block * 2, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let layout = FecPacketLayout { + frame_index: block as u32, + block_index: 0, + last_block_index: 0, + shard_index: 0, + data_shards: 1, + parity_shards: 1, + }; + (packet, layout) + }; + let indices = |result: &ReorderResult| -> Vec { + result.ready.iter().map(|packet| packet.index).collect() + }; + let (packet, layout) = data(0); + assert_eq!(indices(&reorder.push(packet, layout, now)), [0]); + // Blocks 1 and 3 are lost whole. + let (packet, layout) = data(2); + let first_hole = reorder.push(packet, layout, now); + assert_eq!(first_hole.nack, Some((2, 3))); + let (packet, layout) = data(4); + let second_hole = reorder.push(packet, layout, now + ms(10)); + assert_eq!(second_hole.later_nacks, [(6, 7)]); + assert!(second_hole.ready.is_empty()); + + let at = now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT; + let first = reorder.poll_deadline(at); + assert_eq!( + first.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 2, + last_missing_index: 3 + }) + ); + assert_eq!(indices(&first), [4]); + let waiting = reorder.poll_deadline(at + ms(10) - Duration::from_nanos(1)); + assert_eq!(waiting.recovery, None); + assert_eq!(waiting.nack, Some((6, 7))); + // The second hole was revealed 10 ms after the first and ends 10 ms after it, not 52. + let second = reorder.poll_deadline(at + ms(10)); + assert_eq!( + second.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 6, + last_missing_index: 7 + }) + ); + assert_eq!(indices(&second), [8]); + assert!(reorder.blocks.is_empty()); +} + +#[test] +fn a_block_that_fills_a_hole_does_not_restart_the_wait() { + let now = Instant::now(); + let ms = Duration::from_millis; + let mut reorder = FecReorderBuffer::new(48); + // Two data packets and one parity packet per block: block `n` starts at index `3n`. + let packet = |index: u64| { + let plaintext = build_plaintext_rtp(index as u16, FLAG_SOF, (index / 3) as u32, &[0x55]); + let packet = RtpPacket { + index, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let layout = FecPacketLayout { + frame_index: (index / 3) as u32, + block_index: 0, + last_block_index: 0, + shard_index: (index % 3) as usize, + data_shards: 2, + parity_shards: 1, + }; + (packet, layout) + }; + let mut push = |index: u64, at: Instant| { + let (packet, layout) = packet(index); + reorder.push(packet, layout, at) + }; + push(0, now); + assert_eq!(push(1, now).ready.len(), 2); + // Block 1 is lost whole, then one of its packets is retransmitted 40 ms later. + push(6, now); + assert_eq!(push(7, now).nack, Some((3, 5))); + let partial = push(3, now + ms(40)); + assert_eq!(partial.nack, Some((4, 4))); + assert!(partial.recovery.is_none()); + let just_before = now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT - Duration::from_nanos(1); + assert!(reorder.poll_deadline(just_before).recovery.is_none()); + let expired = reorder.poll_deadline(now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT); + assert_eq!( + expired.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 4, + last_missing_index: 4 + }) + ); + assert_eq!(expired.ready.len(), 2); +} + +#[test] +fn a_stalled_block_requests_every_run_it_needs_at_once() { + let now = Instant::now(); + let mut reorder = FecReorderBuffer::new(48); + // Six data packets and two parity packets per block: block `n` starts at index `8n`. + let mut push = |index: u64| { + let plaintext = build_plaintext_rtp(index as u16, FLAG_SOF, (index / 8) as u32, &[0x55]); + let packet = RtpPacket { + index, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let layout = FecPacketLayout { + frame_index: (index / 8) as u32, + block_index: 0, + last_block_index: 0, + shard_index: (index % 8) as usize, + data_shards: 6, + parity_shards: 2, + }; + reorder.push(packet, layout, now) + }; + // Block 0 keeps data 0 and 3: it is four packets short, in two separate runs. + push(0); + push(3); + // Block 1 keeps data 8, 11, 12, 13 and one parity packet: one more packet completes it. + assert_eq!(push(8).later_nacks, [(1, 2), (4, 5)]); + for index in [11, 12, 13, 14] { + assert!(push(index).later_nacks.is_empty()); + } + assert_eq!(push(16).later_nacks, [(9, 10)]); +} From c5a3cebb805d57ebd4f952f8c052b5e05f10ff04 Mon Sep 17 00:00:00 2001 From: Evan Brown <153393779+etgbrown@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:52:08 -0600 Subject: [PATCH 2/2] fix(nvst): use the server's unprotected retransmissions and hold gaps until they arrive The Mjolnir server answers most NACKs with copies sent as payload type 106 on the video SSRC without SRTP protection: no authentication tag, and the body is the plaintext GameStream payload. In a live dump all 3 rejected first shards began with the GameStream frame header, and two copies of one packet differed only in the re-stamped RTP timestamp. The receiver ran them through SRTP unprotect and dropped them as authentication failures; 46 of 46 sampled failures answered a NACK sent 100-218 ms earlier. NACK repair never filled a gap. - A datagram that fails authentication is accepted only as the answer to one of this receiver's NACKs from the last 500 ms, on the bound SSRC, once, through the replay window. - The answers complete frames 150-200 ms after the gap opens, well after the 52 ms give-up. While frame types are known, a gap is now held for 220 ms. The recovery frame is still requested at 52 ms (an early invalidation request), and a complete recovery frame queued behind the gap ends the hold at once, so holding does not delay recovery. Co-Authored-By: Claude Opus 5.5 --- .../opennow-streamer-transport/src/nvst.rs | 192 +++++++++++++++++- .../src/nvst_invalidation.rs | 42 +++- .../src/nvst_nack_tests.rs | 17 ++ .../src/nvst_recovery_tests.rs | 143 +++++++++++-- 4 files changed, 370 insertions(+), 24 deletions(-) diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs index fc0488a7d..ce21c11a5 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs @@ -228,6 +228,13 @@ const MAX_REORDER_WINDOW: usize = 2_048; // Keep the wait exceptional and bounded, but long enough for a retransmission to beat an IDR // reset. This adds latency only while a sequence gap is open; the normal path still drains at once. const MJOLNIR_REORDER_DEQUEUE_TIMEOUT: Duration = Duration::from_millis(52); +/// Mjolnir answers a NACK 150-200 ms after the gap opens (measured over Starlink at a 27 ms +/// round trip). While a recovery frame can still be requested at the usual time, a gap is +/// held this long so the retransmission can fill it. +const RETRANSMISSION_WAIT: Duration = Duration::from_millis(220); +/// How long after its NACK a packet may still arrive as an unprotected retransmission. +const RETRANSMISSION_ACCEPT_WINDOW: Duration = Duration::from_millis(500); +const MAX_RETRANSMISSION_REQUESTS: usize = 4_096; // The replay filter must be at least as deep as the reorder/NACK window. The old 64-packet bitmap // rejected valid retransmissions after roughly 10 ms on a high-bitrate stream, guaranteeing that // every NACK eventually degraded into a keyframe reset. @@ -651,6 +658,9 @@ pub struct NvstFeedbackState { pub(crate) frame_invalidation: FrameInvalidationFeedback, pending_nacks: Mutex>, nack_overflows: AtomicU64, + /// Packets named in NACKs and when they were first requested. See + /// `take_retransmission_request`. + retransmission_requests: Mutex>, last_sender_frame: AtomicU32, completed_frame_bytes: AtomicU64, pending_frame_acks: Mutex>, @@ -685,6 +695,7 @@ impl Default for NvstFeedbackState { frame_invalidation: FrameInvalidationFeedback::default(), pending_nacks: Mutex::new(BTreeMap::new()), nack_overflows: AtomicU64::new(0), + retransmission_requests: Mutex::new(BTreeMap::new()), last_sender_frame: AtomicU32::new(0), completed_frame_bytes: AtomicU64::new(0), pending_frame_acks: Mutex::new(VecDeque::new()), @@ -1014,6 +1025,19 @@ impl NvstFeedbackState { } } + /// Whether `index` was asked for in a NACK within `RETRANSMISSION_ACCEPT_WINDOW`; consumes + /// the request. The Mjolnir server sends most NACK answers without SRTP protection + /// (payload type 106, no authentication tag), so a packet that fails authentication is + /// accepted only as the answer to a request, once. + fn take_retransmission_request(&self, index: u64, now: Instant) -> bool { + let mut requests = self + .retransmission_requests + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + requests.retain(|_, at| now.saturating_duration_since(*at) < RETRANSMISSION_ACCEPT_WINDOW); + requests.remove(&index).is_some() + } + fn resolve_nack(&self, received_index: u64) -> bool { let mut pending = self .pending_nacks @@ -1358,6 +1382,16 @@ impl NvstFeedbackState { packet.attempts = packet.attempts.saturating_add(1); last = index; } + let mut requests = self + .retransmission_requests + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for index in first..=last { + requests.entry(index).or_insert(now); + } + while requests.len() > MAX_RETRANSMISSION_REQUESTS { + requests.pop_first(); + } Some((first, last)) } @@ -2608,6 +2642,25 @@ enum SrtpCipher { } impl SrtpReceiver { + /// Takes an unprotected retransmission at `index`. The replay window still rejects a + /// second copy, and the index is committed like an authenticated packet's. + fn accept_unprotected( + &mut self, + datagram: &[u8], + header: RtpHeader, + index: u64, + ) -> Result { + self.replay.check(index)?; + header.payload(datagram).map_err(NvstDropReason::MalformedRtp)?; + self.replay.commit(index); + Ok(RtpPacket { + index, + header, + plaintext: datagram.to_vec(), + origin: RtpPacketOrigin::Authenticated, + }) + } + fn from_material(material: &NvstSrtpMaterial) -> Self { let cipher = match material { NvstSrtpMaterial::AeadAes128Gcm { @@ -4160,6 +4213,19 @@ impl FecBlock { self.shards.iter().filter(|shard| shard.is_some()).count() >= self.layout.data_shards } + /// Whether this block opens a frame that ends a loss episode, read from the frame header + /// in the frame's first packet. + fn opens_recovery_frame(&self) -> bool { + self.layout.block_index == 0 + && self.shards[0] + .as_ref() + .and_then(|packet| packet.header.payload(&packet.plaintext).ok()) + .is_some_and(|payload| { + matches!(payload, [version, _, _, kind, ..] if version & 0x7f == 1 + && NvstFrameType::from_wire(kind & 0x0f).clears_invalidation()) + }) + } + fn finish(mut self, shard_len: usize) -> Result<(Vec, usize), NvstDropReason> { let missing_data = self.shards[..self.layout.data_shards] .iter() @@ -4289,6 +4355,11 @@ impl FecBlock { struct FecReorderBuffer { blocks: BTreeMap, completed_through: Option, + /// How much longer than `MJOLNIR_REORDER_DEQUEUE_TIMEOUT` a gap is held for its + /// retransmission, set by the receiver. See `overdue`. + grace: Duration, + /// A loss episode is open: frames are withheld until a recovery frame. Set by the receiver. + recovering: bool, shard_len: usize, } @@ -4297,6 +4368,8 @@ impl FecReorderBuffer { Self { blocks: BTreeMap::new(), completed_through: None, + grace: Duration::ZERO, + recovering: false, shard_len: video_packet_size + NVST_FEC_RTP_HEADER_ALLOWANCE, } } @@ -4360,6 +4433,50 @@ impl FecReorderBuffer { result } + /// The frame after the head gap, once the gap has waited the usual 52 ms but is still held + /// for its retransmission. On a long path a retransmission can miss 52 ms by a few + /// milliseconds; the recovery frame requested at 52 ms cannot arrive before 52 ms plus a + /// round trip, so holding the gap that long costs nothing when the retransmission is lost. + fn overdue(&self, now: Instant) -> Option { + let (&base, head) = self.blocks.first_key_value()?; + let between_blocks = self.completed_through.is_some_and(|expected| base > expected); + let since = if between_blocks { + head.arrived_at + } else { + head.stalled_since? + }; + let lost_head = !between_blocks || head.layout.block_index > 0; + (now.saturating_duration_since(since) >= MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + .then_some(head.layout.frame_index + u32::from(lost_head)) + } + + /// Whether the head gap should stop waiting because a complete frame that ends the loss + /// episode is buffered behind it. Nothing in front of that frame can be shown once it is + /// decoded. Only once a recovery frame has been asked for, so an I-frame that merely + /// follows a fresh gap does not cut its retransmission short. + fn recovery_frame_ready(&self, waited: Duration) -> bool { + if !self.recovering && waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT { + return false; + } + let mut awaited = None; + for (&base, block) in &self.blocks { + let layout = block.layout; + let continues = awaited == Some((layout.frame_index, layout.block_index, base)); + awaited = None; + if !(continues || block.opens_recovery_frame()) + || !block.has_enough_shards() + || block.repair_failed + { + continue; + } + if layout.block_index == layout.last_block_index { + return true; + } + awaited = Some((layout.frame_index, layout.block_index + 1, block.end_index())); + } + false + } + /// A packet from a new block proves that whatever the earlier blocks still lack was /// skipped, not delayed. Their NACK waits start now and run side by side, so a burst that /// damages several blocks holds the queue for one NACK window instead of one per block. @@ -4410,11 +4527,16 @@ impl FecReorderBuffer { if let Some(expected) = self.completed_through && base > expected { - if now.saturating_duration_since(head.arrived_at) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + let waited = now.saturating_duration_since(head.arrived_at); + if waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + self.grace && base.saturating_sub(expected) < MAX_REORDER_WINDOW as u64 && self.blocks.len() < MAX_PENDING_FEC_BLOCKS + && !self.recovery_frame_ready(waited) { - result.nack = Some((expected, base - 1)); + // In the extra wait the request has expired. A new one could not be + // answered in time and would only have the packets sent twice. + result.nack = (waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + .then_some((expected, base - 1)); break; } if result.recovery.is_some() || !result.ready.is_empty() { @@ -4478,11 +4600,15 @@ impl FecReorderBuffer { .blocks .last_key_value() .map_or(0, |(&last_base, _)| last_base.saturating_sub(base)); - if now.saturating_duration_since(stalled_since) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + let waited = now.saturating_duration_since(stalled_since); + if waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + self.grace && block_span < MAX_REORDER_WINDOW as u64 && self.blocks.len() < MAX_PENDING_FEC_BLOCKS + && !self.recovery_frame_ready(waited) { - result.nack = head.first_missing_data_run(); + if waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT { + result.nack = head.first_missing_data_run(); + } break; } if result.recovery.is_some() || !result.ready.is_empty() { @@ -4618,6 +4744,7 @@ pub struct NvstVideoReceiver { fec_repaired_packets: u64, frames_emitted: u64, replay_rejections: u64, + unprotected_retransmissions: u64, stale_packets: u64, recovered_retransmissions: u64, packet_gap_recoveries: u64, @@ -4654,6 +4781,7 @@ impl NvstVideoReceiver { fec_repaired_packets: 0, frames_emitted: 0, replay_rejections: 0, + unprotected_retransmissions: 0, stale_packets: 0, recovered_retransmissions: 0, packet_gap_recoveries: 0, @@ -4799,6 +4927,21 @@ impl NvstVideoReceiver { } } + /// An unprotected answer to one of this receiver's NACKs, on the bound SSRC. + fn unprotected_retransmission(&mut self, datagram: &[u8], now: Instant) -> Option { + let header = RtpHeader::parse(datagram).ok()?; + if self.bound_ssrc.is_some_and(|ssrc| ssrc != header.ssrc) { + return None; + } + let index = self.srtp.replay.guess_packet_index(header.sequence_number).ok()?; + if !self.config.feedback.take_retransmission_request(index, now) { + return None; + } + let packet = self.srtp.accept_unprotected(datagram, header, index).ok()?; + self.unprotected_retransmissions += 1; + Some(packet) + } + pub fn process_datagram( &mut self, source: SocketAddr, @@ -4827,12 +4970,18 @@ impl NvstVideoReceiver { } let packet = match self.srtp.unprotect(datagram) { Ok(packet) => packet, - Err(reason) => { - if reason == NvstDropReason::ReplayRejected { - self.replay_rejections += 1; + Err(reason) => match (reason == NvstDropReason::AuthenticationFailed) + .then(|| self.unprotected_retransmission(datagram, now)) + .flatten() + { + Some(packet) => packet, + None => { + if reason == NvstDropReason::ReplayRejected { + self.replay_rejections += 1; + } + return vec![NvstReceiveEvent::Dropped(reason)]; } - return vec![NvstReceiveEvent::Dropped(reason)]; - } + }, }; if let Some(expected) = self.config.expected_payload_type && packet.header.payload_type != expected @@ -4905,6 +5054,7 @@ impl NvstVideoReceiver { if layout.shard_index >= layout.data_shards { self.fec_packets += 1; } + self.tune_fec_reorder(); let result = self.fec_reorder.push(packet, layout, now); if layout.shard_index < layout.data_shards && matches!(result.dropped, Some(NvstDropReason::Unsupported(_))) @@ -4922,11 +5072,28 @@ impl NvstVideoReceiver { .poll_backstop(now, RFI_BACKSTOP_TIMEOUT, &self.config.feedback); let reordered = self.reorder.poll_deadline(now); let mut events = self.consume_reorder_result(reordered, now); + self.tune_fec_reorder(); let fec = self.fec_reorder.poll_deadline(now); events.extend(self.consume_fec_result(fec, now)); events } + fn tune_fec_reorder(&mut self) { + self.fec_reorder.recovering = self.frame_invalidation.active(); + self.fec_reorder.grace = self.retransmission_grace(); + } + + /// Holds a gap until its retransmission can arrive, while frame types are known. The + /// recovery frame is still requested at the usual time (`anticipate`, or the open episode's + /// request), and a complete recovery frame queued behind the gap ends the wait, so holding + /// longer does not delay recovery. + fn retransmission_grace(&self) -> Duration { + if !self.frame_invalidation.frame_types_known() { + return Duration::ZERO; + } + RETRANSMISSION_WAIT.saturating_sub(MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + } + /// The FEC queue reports one discontinuity per pass, so that packets completed ahead of /// it are delivered first. Keep polling while a pass makes progress. fn consume_fec_result( @@ -4939,6 +5106,10 @@ impl NvstVideoReceiver { let progressed = result.recovery.is_some() || !result.ready.is_empty(); events.extend(self.consume_reorder_result(result, now)); if !progressed { + if let Some(frame) = self.fec_reorder.overdue(now) { + self.frame_invalidation + .anticipate(frame, &self.config.feedback); + } return events; } result = self.fec_reorder.poll_deadline(now); @@ -5105,7 +5276,7 @@ impl NvstVideoReceiver { /// depend on when buffered log lines happen to flush. pub fn stats_line(&self, origin: Instant) -> String { format!( - "elapsed={:.1}s auth={} fec={} fecRecovered={} frames={} replay={} stale={} nackRecovered={} nackPending={} nackOverflow={} gaps={} {} ssrc={:?}", + "elapsed={:.1}s auth={} fec={} fecRecovered={} frames={} replay={} stale={} nackRecovered={} nackPending={} nackOverflow={} unprotectedRtx={} gaps={} {} ssrc={:?}", origin.elapsed().as_secs_f64(), self.authenticated_packets, self.fec_packets, @@ -5121,6 +5292,7 @@ impl NvstVideoReceiver { .unwrap_or_else(std::sync::PoisonError::into_inner) .len(), self.config.feedback.nack_overflows.load(Ordering::Relaxed), + self.unprotected_retransmissions, self.packet_gap_recoveries, self.config.feedback.frame_invalidation.stats(), self.bound_ssrc, diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs index a5e8a41db..19865a54d 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs @@ -37,7 +37,7 @@ impl NvstFrameType { } } - fn clears_invalidation(self) -> bool { + pub(crate) fn clears_invalidation(self) -> bool { matches!(self, Self::I | Self::IntraRefresh | Self::RefPicI) } } @@ -126,9 +126,21 @@ pub(crate) struct FrameInvalidation { active_since: Option, last_retry_at: Option, backstop_requested: bool, + /// The last assembled frame when a request went out ahead of a give-up. + anticipated: Option, } impl FrameInvalidation { + /// Whether a loss episode is open, i.e. frames are withheld until a recovery frame. + pub(crate) fn active(&self) -> bool { + self.active_since.is_some() + } + + /// Whether recovery is reported by frame range, so a held gap cannot delay it. + pub(crate) fn frame_types_known(&self) -> bool { + self.frame_types_known + } + pub(crate) fn reset(&mut self, feedback: &NvstFeedbackState) { feedback.frame_invalidation.discard(); *self = Self::default(); @@ -144,6 +156,33 @@ impl FrameInvalidation { true } + /// A gap has waited the usual time for its retransmission and is held a little longer. + /// Report the frames at risk now, when the give-up would have, so the server's recovery + /// frame is no later than without the extra wait. Nothing is withheld: if the + /// retransmission completes the gap, its frames are delivered and the server's answer is an + /// ordinary keyframe. If not, the give-up follows with the full range. One request per stall. + pub(crate) fn anticipate(&mut self, revealed_by_frame: u32, feedback: &NvstFeedbackState) { + let Some(last) = self.last_frame else { + return; + }; + if !self.frame_types_known || self.active_since.is_some() || self.anticipated == Some(last) + { + return; + } + self.anticipated = Some(last); + let first = u64::from(last) + 1; + let range = ( + first, + first.max(u64::from(revealed_by_frame).saturating_sub(1)), + ); + feedback.frame_invalidation.queue(range); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-rfi", + &format!("early request frames={}..={}", range.0, range.1), + ); + } + pub(crate) fn poll_backstop( &mut self, now: Instant, @@ -271,5 +310,6 @@ impl FrameInvalidation { self.active_since = None; self.last_retry_at = None; self.backstop_requested = false; + self.anticipated = None; } } diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs index c5b72b343..3987aaa2e 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs @@ -448,3 +448,20 @@ fn resolving_holes_and_readding_ranges_does_not_reset_nack_attempts() { None ); } + +#[test] +fn nacked_packets_are_accepted_unprotected_once_within_the_window() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + feedback.request_nack(100, 102, now); + assert_eq!(feedback.take_nack(now, Some(DEFAULT_NACK_RTT)), Some((100, 102))); + + let answered = now + Duration::from_millis(200); + assert!(feedback.take_retransmission_request(101, answered)); + assert!(!feedback.take_retransmission_request(101, answered), "accepted only once"); + assert!(!feedback.take_retransmission_request(103, answered), "never requested"); + assert!( + !feedback.take_retransmission_request(100, now + RETRANSMISSION_ACCEPT_WINDOW), + "requests expire" + ); +} diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs index 83ef80047..c0460e380 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs @@ -1037,17 +1037,14 @@ fn consecutive_damaged_frames_share_one_nack_window() { assert_eq!(stream.feedback.take_nack(now, None), Some((13, 13))); assert_eq!(stream.feedback.take_nack(now, None), Some((16, 16))); - assert!(stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT).is_empty()); - assert_eq!( - gaps(&stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(1))), - [(10, 10)] - ); - // Serial waits would hold frame 6, and with it the invalidation request, until 158 ms. - let events = stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(3)); + assert!(stream.poll(RETRANSMISSION_WAIT).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(10, 10)]); + // Serial waits would hold frame 6, and with it the invalidation request, three times as long. + let events = stream.poll(RETRANSMISSION_WAIT + ms(3)); assert_eq!(gaps(&events), [(13, 13), (16, 16)]); assert!(delivered(events).is_empty()); assert_eq!(stream.feedback.frame_invalidation.take(), Some((3, 5))); - assert_eq!(stream.fec_frame(7, 2, ms(60)), [(7, true, true)]); + assert_eq!(stream.fec_frame(7, 2, RETRANSMISSION_WAIT + ms(8)), [(7, true, true)]); assert_eq!(stream.receiver.packet_gap_recoveries, 3); assert_eq!( stream.stats(), @@ -1066,7 +1063,7 @@ fn a_recovery_frame_between_two_expired_gaps_is_delivered() { assert!(stream.fec_frame(5, 1, ms(3)).is_empty()); // Both waits are over by the time the queue is polled. The I-frame sits between the // two discontinuities and must reach the decoder. - let events = stream.poll(ms(200)); + let events = stream.poll(RETRANSMISSION_WAIT + ms(20)); assert_eq!(gaps(&events), [(7, 7), (13, 13)]); assert_eq!(delivered(events), [(3, true, true)]); assert_eq!(stream.feedback.frame_invalidation.take(), Some((4, 4))); @@ -1085,10 +1082,7 @@ fn giving_up_on_a_gap_keeps_the_requests_for_later_gaps() { assert_eq!(stream.feedback.take_nack(sent_at, None), Some((7, 7))); assert_eq!(stream.feedback.take_nack(sent_at, None), Some((13, 13))); - assert_eq!( - gaps(&stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(1))), - [(7, 7)] - ); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(7, 7)]); let pending = stream.feedback.pending_nacks.lock().unwrap(); assert_eq!(pending.keys().copied().collect::>(), [13]); assert_eq!(pending[&13].attempts, 1, "the request was not started over"); @@ -1246,3 +1240,126 @@ fn a_stalled_block_requests_every_run_it_needs_at_once() { } assert_eq!(push(16).later_nacks, [(9, 10)]); } + +#[test] +fn a_retransmission_that_misses_the_usual_wait_is_used() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + // At the usual give-up the recovery frame is requested, but the gap stays open. + assert!(stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(1)).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + let [_, retransmitted] = typed_fec_frame(2, 1, 0x41); + assert_eq!( + stream.frames(retransmitted, ms(70)), + [(2, false, true), (3, false, true)] + ); + // The server's answer reaches the decoder as an ordinary keyframe. + assert_eq!(stream.fec_frame(4, 2, ms(90)), [(4, true, true)]); + assert_eq!(stream.receiver.packet_gap_recoveries, 0); + assert_eq!( + stream.stats(), + "rfiSent=0 rfiDropped=0 rfiRecoveredI=0 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0 rfiIdrBackstop=0" + ); +} + +#[test] +fn a_gap_without_its_retransmission_is_given_up_after_the_retransmission_wait() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert!(stream.poll(ms(53)).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + // Only one request per stall goes out ahead of the give-up. + assert!(stream.poll(RETRANSMISSION_WAIT).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), None); + let events = stream.poll(RETRANSMISSION_WAIT + ms(1)); + assert_eq!(gaps(&events), [(7, 7)]); + assert!(delivered(events).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + assert_eq!(stream.fec_frame(4, 2, RETRANSMISSION_WAIT + ms(8)), [(4, true, true)]); +} + +#[test] +fn a_gap_while_a_recovery_is_open_waits_for_its_retransmission_too() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(7, 7)]); + // The recovery frame may itself be the damaged frame, so its gap waits as long. A + // complete recovery frame behind the gap would end the wait at once (next test). + stream.damaged_fec_frame(4, RETRANSMISSION_WAIT + ms(2)); + assert!(stream.fec_frame(5, 1, RETRANSMISSION_WAIT + ms(3)).is_empty()); + assert!(stream.poll(RETRANSMISSION_WAIT * 2 + ms(2)).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT * 2 + ms(3))), [(13, 13)]); +} + +#[test] +fn the_extra_wait_does_not_repeat_the_retransmission_request() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert_eq!(stream.feedback.take_nack(stream.origin + ms(1), None), Some((7, 7))); + // A second request now could not be answered before the extra wait ends. + assert!(stream.poll(ms(60)).is_empty()); + assert_eq!(stream.feedback.take_nack(stream.origin + ms(60), None), None); +} + +#[test] +fn a_recovery_frame_behind_a_gap_ends_an_open_episode_at_once() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(7, 7)]); + // A later frame is damaged too, and the server's recovery frame arrives right behind it. + stream.damaged_fec_frame(4, RETRANSMISSION_WAIT + ms(8)); + let [first, last] = typed_fec_frame(5, 2, 0x65); + assert!(stream.push(first, RETRANSMISSION_WAIT + ms(9)).is_empty()); + let events = stream.push(last, RETRANSMISSION_WAIT + ms(9)); + assert_eq!(gaps(&events), [(13, 13)]); + assert_eq!(delivered(events), [(5, true, true)]); + assert_eq!(stream.feedback.frame_invalidation.take(), None); +} + +#[test] +fn a_recovery_frame_that_arrives_inside_the_extra_wait_ends_it() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(50)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert!(stream.poll(ms(53)).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + // The answer comes back sooner than the last ping suggested: 80 ms, not 103 ms. + assert_eq!(stream.fec_frame(4, 2, ms(80)), [(4, true, true)]); + assert_eq!(stream.receiver.packet_gap_recoveries, 1); +} + +#[test] +fn an_i_frame_behind_a_fresh_gap_does_not_cut_the_retransmission_short() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 2, ms(1)).is_empty()); + let [_, retransmitted] = typed_fec_frame(2, 1, 0x41); + assert_eq!( + stream.frames(retransmitted, ms(40)), + [(2, false, true), (3, true, true)] + ); +}