diff --git a/docs/streamer-comparison/transport-recovery.md b/docs/streamer-comparison/transport-recovery.md index 23ac18f57..712aed6dd 100644 --- a/docs/streamer-comparison/transport-recovery.md +++ b/docs/streamer-comparison/transport-recovery.md @@ -43,6 +43,26 @@ buffered successors to proceed through the existing discontinuity/RFI path. TR13 cites **D** macOS timed source-queue operations and **L** Windows `NACK waiting queue limit time: 52 ms`. +FEC blocks wait for retransmission side by side. A block's 52 ms wait starts +when a packet from a later block arrives, and a hole's when the block behind it +arrives, not when the gap reaches the head of the queue. Each gap is requested +as soon as it is evident, with up to eight NACK reports per 4 ms tick, and a +stalled block asks for every missing data run it needs to reconstruct, because +its wait is not extended for a second round. Giving up on one gap forgets only +that gap's requests. The queue reports one discontinuity at a time, so a +complete frame between two abandoned gaps is still delivered. + +Before this, a damaged block started a fresh 52 ms wait, and its first NACK, +only after the block ahead of it had been resolved. A loss that damaged N +blocks held the queue for N × 52 ms, together with the invalidation request +(queued when the next frame assembles) and the recovery frame the server sends +in reply. In five 2-minute 3840×2160 @ 120 AV1 sessions over a link that drops +packets every 15 s, 21 of 48 loss episodes gave up on two or more gaps in +series before the first invalidation left, and the six longest episodes +(0.65–1.5 s from the first gap to a usable frame) gave up on 15 to 66 gaps +each. The plain reorder queue, used only when a stream carries no FEC layout, +still waits per gap in sequence. + Unified-socket Mjolnir calls the same produced-frame watchdog and publishes the same typed stall and recovery events as the dedicated receiver. Authenticated partial video can no longer keep that route frozen indefinitely while resetting diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs index aac36b959..ce21c11a5 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst.rs @@ -95,6 +95,9 @@ const RTCP_SENDER_SSRC: u32 = 0x4f4e_4f57; // "ONOW" const SRTCP_RR_INTERVAL: Duration = Duration::from_secs(1); const RTCP_RECOVERY_INTERVAL: Duration = Duration::from_millis(4); const NACK_RETRY_INTERVAL: Duration = Duration::from_millis(4); +// Gaps in separate FEC blocks are requested side by side but are not contiguous. One report +// per recovery tick would stagger them by 4 ms each inside a 52 ms wait. +const MAX_NACK_REPORTS_PER_TICK: usize = 8; const DEFAULT_NACK_RTT: Duration = Duration::from_millis(30); const NACK_TRACKING_TIMEOUT: Duration = Duration::from_millis(52); const MAX_NACK_ATTEMPTS: u8 = 3; @@ -225,6 +228,13 @@ const MAX_REORDER_WINDOW: usize = 2_048; // Keep the wait exceptional and bounded, but long enough for a retransmission to beat an IDR // reset. This adds latency only while a sequence gap is open; the normal path still drains at once. const MJOLNIR_REORDER_DEQUEUE_TIMEOUT: Duration = Duration::from_millis(52); +/// Mjolnir answers a NACK 150-200 ms after the gap opens (measured over Starlink at a 27 ms +/// round trip). While a recovery frame can still be requested at the usual time, a gap is +/// held this long so the retransmission can fill it. +const RETRANSMISSION_WAIT: Duration = Duration::from_millis(220); +/// How long after its NACK a packet may still arrive as an unprotected retransmission. +const RETRANSMISSION_ACCEPT_WINDOW: Duration = Duration::from_millis(500); +const MAX_RETRANSMISSION_REQUESTS: usize = 4_096; // The replay filter must be at least as deep as the reorder/NACK window. The old 64-packet bitmap // rejected valid retransmissions after roughly 10 ms on a high-bitrate stream, guaranteeing that // every NACK eventually degraded into a keyframe reset. @@ -648,6 +658,9 @@ pub struct NvstFeedbackState { pub(crate) frame_invalidation: FrameInvalidationFeedback, pending_nacks: Mutex>, nack_overflows: AtomicU64, + /// Packets named in NACKs and when they were first requested. See + /// `take_retransmission_request`. + retransmission_requests: Mutex>, last_sender_frame: AtomicU32, completed_frame_bytes: AtomicU64, pending_frame_acks: Mutex>, @@ -682,6 +695,7 @@ impl Default for NvstFeedbackState { frame_invalidation: FrameInvalidationFeedback::default(), pending_nacks: Mutex::new(BTreeMap::new()), nack_overflows: AtomicU64::new(0), + retransmission_requests: Mutex::new(BTreeMap::new()), last_sender_frame: AtomicU32::new(0), completed_frame_bytes: AtomicU64::new(0), pending_frame_acks: Mutex::new(VecDeque::new()), @@ -1011,6 +1025,19 @@ impl NvstFeedbackState { } } + /// Whether `index` was asked for in a NACK within `RETRANSMISSION_ACCEPT_WINDOW`; consumes + /// the request. The Mjolnir server sends most NACK answers without SRTP protection + /// (payload type 106, no authentication tag), so a packet that fails authentication is + /// accepted only as the answer to a request, once. + fn take_retransmission_request(&self, index: u64, now: Instant) -> bool { + let mut requests = self + .retransmission_requests + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + requests.retain(|_, at| now.saturating_duration_since(*at) < RETRANSMISSION_ACCEPT_WINDOW); + requests.remove(&index).is_some() + } + fn resolve_nack(&self, received_index: u64) -> bool { let mut pending = self .pending_nacks @@ -1355,6 +1382,16 @@ impl NvstFeedbackState { packet.attempts = packet.attempts.saturating_add(1); last = index; } + let mut requests = self + .retransmission_requests + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for index in first..=last { + requests.entry(index).or_insert(now); + } + while requests.len() > MAX_RETRANSMISSION_REQUESTS { + requests.pop_first(); + } Some((first, last)) } @@ -1375,6 +1412,16 @@ impl NvstFeedbackState { .unwrap_or_else(std::sync::PoisonError::into_inner) .clear(); } + + /// Drops requests the reorder queue has given up on. Later gaps keep their own deadline + /// and attempt count instead of being requested again from scratch. + fn forget_nacks_through(&self, last_abandoned_index: u64) { + let mut pending = self + .pending_nacks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *pending = pending.split_off(&last_abandoned_index.saturating_add(1)); + } } /// Shared handle to the NVST feedback plane (cheap to clone, shared across threads). @@ -2595,6 +2642,25 @@ enum SrtpCipher { } impl SrtpReceiver { + /// Takes an unprotected retransmission at `index`. The replay window still rejects a + /// second copy, and the index is committed like an authenticated packet's. + fn accept_unprotected( + &mut self, + datagram: &[u8], + header: RtpHeader, + index: u64, + ) -> Result { + self.replay.check(index)?; + header.payload(datagram).map_err(NvstDropReason::MalformedRtp)?; + self.replay.commit(index); + Ok(RtpPacket { + index, + header, + plaintext: datagram.to_vec(), + origin: RtpPacketOrigin::Authenticated, + }) + } + fn from_material(material: &NvstSrtpMaterial) -> Self { let cipher = match material { NvstSrtpMaterial::AeadAes128Gcm { @@ -3069,6 +3135,7 @@ fn build_rtcp_nack( packet } +/// Returns whether a report was sent, so the caller can send the next pending run. fn send_pending_nack( feedback: &NvstFeedbackState, now: Instant, @@ -3077,20 +3144,20 @@ fn send_pending_nack( mjolnir: bool, sender_ssrc: u32, media_ssrc: u32, -) { +) -> bool { let Some(channel) = (if mjolnir { Some(channels.control_partial) } else { channels.rtcp }) else { - return; + return false; }; if rtc.channel(channel).is_none() { - return; + return false; } let retry_rtt = mjolnir.then(|| feedback.ping(now).unwrap_or(DEFAULT_NACK_RTT)); let Some((first, last)) = feedback.take_nack(now, retry_rtt) else { - return; + return false; }; let admitted = if mjolnir { let missing: Vec = (first..=last).map(|index| index as u16).collect(); @@ -3112,6 +3179,7 @@ fn send_pending_nack( } else { feedback.mark_nack_send_failed(first, last); } + admitted } fn send_pending_frame_invalidation( @@ -3654,6 +3722,8 @@ struct RtpReorderBuffer { struct ReorderResult { ready: Vec, nack: Option<(u64, u64)>, + /// Gaps behind the head of the FEC queue, requested as soon as they are evident. + later_nacks: Vec<(u64, u64)>, recovery: Option, dropped: Option, fec_repaired: usize, @@ -3682,6 +3752,7 @@ impl RtpReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: Some(NvstDropReason::StaleRtpPacket { index }), fec_repaired: 0, @@ -3691,6 +3762,7 @@ impl RtpReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: Some(NvstDropReason::DuplicateRtpPacket { index }), fec_repaired: 0, @@ -3767,6 +3839,7 @@ impl RtpReorderBuffer { ReorderResult { ready, nack, + later_nacks: Vec::new(), recovery, dropped: None, fec_repaired: 0, @@ -3817,6 +3890,12 @@ struct FecBlock { base_index: u64, shards: Vec>, repair_failed: bool, + /// When the block's first packet arrived, or for a block that fills a hole behind buffered + /// successors, when the first of those arrived. A hole in front of it was evident by then. + arrived_at: Instant, + /// When a later block first arrived, or reconstruction failed. From then on, whatever + /// this block still lacks was lost rather than delayed, and its NACK wait is running. + stalled_since: Option, } struct Gf256Tables { @@ -4036,16 +4115,22 @@ fn reconstruct_nvst_cauchy_data( } impl FecBlock { - fn new(layout: FecPacketLayout, packet_index: u64) -> Self { + fn new(layout: FecPacketLayout, packet_index: u64, now: Instant) -> Self { let total = layout.data_shards + layout.parity_shards; Self { layout, base_index: packet_index.saturating_sub(layout.shard_index as u64), shards: (0..total).map(|_| None).collect(), repair_failed: false, + arrived_at: now, + stalled_since: None, } } + fn end_index(&self) -> u64 { + self.base_index + (self.layout.data_shards + self.layout.parity_shards) as u64 + } + fn matches(&self, layout: FecPacketLayout, packet_index: u64) -> bool { self.layout.frame_index == layout.frame_index && self.layout.block_index == layout.block_index @@ -4083,6 +4168,33 @@ impl FecBlock { )) } + /// The missing data runs that retransmission has to fill before the block can be + /// reconstructed. Parity already received covers the rest, so a later run is included + /// only when the earlier ones leave the block short. The block's wait is not extended + /// for a second round, so everything it needs is requested at once. + fn missing_runs_to_complete(&self) -> Vec<(u64, u64)> { + let present = self.shards.iter().flatten().count(); + let mut shortfall = self.layout.data_shards.saturating_sub(present); + let mut runs = Vec::new(); + let mut shard = 0; + while shortfall > 0 && shard < self.layout.data_shards { + if self.shards[shard].is_some() { + shard += 1; + continue; + } + let first = shard; + while shard < self.layout.data_shards && self.shards[shard].is_none() { + shard += 1; + } + runs.push(( + self.base_index + first as u64, + self.base_index + shard as u64 - 1, + )); + shortfall = shortfall.saturating_sub(shard - first); + } + runs + } + fn insert(&mut self, layout: FecPacketLayout, packet: RtpPacket) -> Option { let slot = &mut self.shards[layout.shard_index]; if slot.is_some() { @@ -4101,6 +4213,19 @@ impl FecBlock { self.shards.iter().filter(|shard| shard.is_some()).count() >= self.layout.data_shards } + /// Whether this block opens a frame that ends a loss episode, read from the frame header + /// in the frame's first packet. + fn opens_recovery_frame(&self) -> bool { + self.layout.block_index == 0 + && self.shards[0] + .as_ref() + .and_then(|packet| packet.header.payload(&packet.plaintext).ok()) + .is_some_and(|payload| { + matches!(payload, [version, _, _, kind, ..] if version & 0x7f == 1 + && NvstFrameType::from_wire(kind & 0x0f).clears_invalidation()) + }) + } + fn finish(mut self, shard_len: usize) -> Result<(Vec, usize), NvstDropReason> { let missing_data = self.shards[..self.layout.data_shards] .iter() @@ -4230,7 +4355,11 @@ impl FecBlock { struct FecReorderBuffer { blocks: BTreeMap, completed_through: Option, - gap_wait: Option<(u64, Instant)>, + /// How much longer than `MJOLNIR_REORDER_DEQUEUE_TIMEOUT` a gap is held for its + /// retransmission, set by the receiver. See `overdue`. + grace: Duration, + /// A loss episode is open: frames are withheld until a recovery frame. Set by the receiver. + recovering: bool, shard_len: usize, } @@ -4239,7 +4368,8 @@ impl FecReorderBuffer { Self { blocks: BTreeMap::new(), completed_through: None, - gap_wait: None, + grace: Duration::ZERO, + recovering: false, shard_len: video_packet_size + NVST_FEC_RTP_HEADER_ALLOWANCE, } } @@ -4247,7 +4377,6 @@ impl FecReorderBuffer { fn reset(&mut self) { self.blocks.clear(); self.completed_through = None; - self.gap_wait = None; } fn push(&mut self, packet: RtpPacket, layout: FecPacketLayout, now: Instant) -> ReorderResult { @@ -4259,19 +4388,22 @@ impl FecReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: None, fec_repaired: 0, }; } + let created = !self.blocks.contains_key(&packet_base); let block = self .blocks .entry(packet_base) - .or_insert_with(|| FecBlock::new(layout, packet.index)); + .or_insert_with(|| FecBlock::new(layout, packet.index, now)); if !block.matches(layout, packet.index) { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: Some(NvstDropReason::Unsupported( NvstUnsupportedFeature::FecRepair, @@ -4284,54 +4416,137 @@ impl FecReorderBuffer { return ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped, fec_repaired: 0, }; } - self.poll_deadline(now) + let later_nacks = if created { + self.start_gap_waits(packet_base, now) + } else { + Vec::new() + }; + let mut result = self.poll_deadline(now); + result.later_nacks = later_nacks; + result + } + + /// The frame after the head gap, once the gap has waited the usual 52 ms but is still held + /// for its retransmission. On a long path a retransmission can miss 52 ms by a few + /// milliseconds; the recovery frame requested at 52 ms cannot arrive before 52 ms plus a + /// round trip, so holding the gap that long costs nothing when the retransmission is lost. + fn overdue(&self, now: Instant) -> Option { + let (&base, head) = self.blocks.first_key_value()?; + let between_blocks = self.completed_through.is_some_and(|expected| base > expected); + let since = if between_blocks { + head.arrived_at + } else { + head.stalled_since? + }; + let lost_head = !between_blocks || head.layout.block_index > 0; + (now.saturating_duration_since(since) >= MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + .then_some(head.layout.frame_index + u32::from(lost_head)) + } + + /// Whether the head gap should stop waiting because a complete frame that ends the loss + /// episode is buffered behind it. Nothing in front of that frame can be shown once it is + /// decoded. Only once a recovery frame has been asked for, so an I-frame that merely + /// follows a fresh gap does not cut its retransmission short. + fn recovery_frame_ready(&self, waited: Duration) -> bool { + if !self.recovering && waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT { + return false; + } + let mut awaited = None; + for (&base, block) in &self.blocks { + let layout = block.layout; + let continues = awaited == Some((layout.frame_index, layout.block_index, base)); + awaited = None; + if !(continues || block.opens_recovery_frame()) + || !block.has_enough_shards() + || block.repair_failed + { + continue; + } + if layout.block_index == layout.last_block_index { + return true; + } + awaited = Some((layout.frame_index, layout.block_index + 1, block.end_index())); + } + false + } + + /// A packet from a new block proves that whatever the earlier blocks still lack was + /// skipped, not delayed. Their NACK waits start now and run side by side, so a burst that + /// damages several blocks holds the queue for one NACK window instead of one per block. + /// Returns the gaps to request at once. + fn start_gap_waits(&mut self, new_base: u64, now: Instant) -> Vec<(u64, u64)> { + let mut nacks = Vec::new(); + let mut expected = self.completed_through; + for block in self.blocks.range_mut(..new_base).map(|(_, block)| block) { + if block.stalled_since.is_none() { + block.stalled_since = Some(now); + nacks.append(&mut block.missing_runs_to_complete()); + } + expected = Some(block.end_index()); + } + if let Some(expected) = expected + && new_base > expected + && new_base - expected < MAX_REORDER_WINDOW as u64 + { + nacks.push((expected, new_base - 1)); + } + // A block that fills a hole behind buffered successors was overtaken when the first + // of them arrived. Neither it nor a hole still in front of it gets a fresh wait. + let overtaken_at = self + .blocks + .range(new_base + 1..) + .map(|(_, block)| block.arrived_at) + .min(); + if let (Some(overtaken_at), Some(block)) = (overtaken_at, self.blocks.get_mut(&new_base)) { + block.arrived_at = overtaken_at; + block.stalled_since = Some(overtaken_at); + } + nacks } + /// Reports at most one discontinuity per call, followed by the packets behind it. Waits + /// overlap, so several blocks can run out of time together; the caller polls again while + /// a call makes progress. Packets completed ahead of a discontinuity are delivered first. fn poll_deadline(&mut self, now: Instant) -> ReorderResult { let mut result = ReorderResult { ready: Vec::new(), nack: None, + later_nacks: Vec::new(), recovery: None, dropped: None, fec_repaired: 0, }; - loop { - let Some((&base, head)) = self.blocks.first_key_value() else { - self.gap_wait = None; - break; - }; - + while let Some((&base, head)) = self.blocks.first_key_value() { if let Some(expected) = self.completed_through && base > expected { - result.nack = Some((expected, base - 1)); - let started = match self.gap_wait { - Some((waiting_for, started)) if waiting_for == expected => started, - _ => { - self.gap_wait = Some((expected, now)); - now - } - }; - if now.saturating_duration_since(started) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + let waited = now.saturating_duration_since(head.arrived_at); + if waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + self.grace && base.saturating_sub(expected) < MAX_REORDER_WINDOW as u64 && self.blocks.len() < MAX_PENDING_FEC_BLOCKS + && !self.recovery_frame_ready(waited) { + // In the extra wait the request has expired. A new one could not be + // answered in time and would only have the packets sent twice. + result.nack = (waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + .then_some((expected, base - 1)); + break; + } + if result.recovery.is_some() || !result.ready.is_empty() { break; } result.recovery = Some(NvstRecovery::PacketGap { first_missing_index: expected, last_missing_index: base - 1, }); - result.ready.clear(); - result.nack = None; self.completed_through = Some(base); - self.gap_wait = None; continue; } @@ -4340,8 +4555,7 @@ impl FecReorderBuffer { .blocks .remove(&base) .expect("head FEC block is present"); - let completed_through = - base + (completed.layout.data_shards + completed.layout.parity_shards) as u64; + let completed_through = completed.end_index(); // Only reconstruction can fail. Healthy blocks move their RTP // buffers straight to assembly without copying the video payload. let repair_backup = completed.missing_data_range().map(|_| completed.clone()); @@ -4350,12 +4564,12 @@ impl FecReorderBuffer { result.ready.append(&mut ready); result.fec_repaired += repaired; self.completed_through = Some(completed_through); - self.gap_wait = None; continue; } Err(reason) => { let mut pending = repair_backup.expect("failed repair has missing data"); pending.repair_failed = true; + pending.stalled_since = Some(now); let (first_missing, last_missing) = pending .first_missing_data_run() .expect("failed FEC reconstruction has missing data"); @@ -4371,37 +4585,33 @@ impl FecReorderBuffer { ); self.blocks.insert(base, pending); result.nack = Some((first_missing, last_missing)); - self.gap_wait = Some((first_missing, now)); break; } } } - // A later block proves that the head block has stopped arriving normally. Ask for - // only the first contiguous missing run and retain all subsequent blocks while the - // retransmission is in flight. - if self.blocks.len() == 1 && self.gap_wait.is_none() { + // Until a later block arrives, the head block is still arriving normally. After + // that, ask for the first contiguous missing run and retain all subsequent blocks + // while the retransmission is in flight. + let Some(stalled_since) = head.stalled_since else { break; - } - let (first_missing, last_missing) = head - .first_missing_data_run() - .expect("an incomplete FEC block has missing data"); - result.nack = Some((first_missing, last_missing)); - let started = match self.gap_wait { - Some((waiting_for, started)) if waiting_for == first_missing => started, - _ => { - self.gap_wait = Some((first_missing, now)); - now - } }; let block_span = self .blocks .last_key_value() .map_or(0, |(&last_base, _)| last_base.saturating_sub(base)); - if now.saturating_duration_since(started) < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + let waited = now.saturating_duration_since(stalled_since); + if waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT + self.grace && block_span < MAX_REORDER_WINDOW as u64 && self.blocks.len() < MAX_PENDING_FEC_BLOCKS + && !self.recovery_frame_ready(waited) { + if waited < MJOLNIR_REORDER_DEQUEUE_TIMEOUT { + result.nack = head.first_missing_data_run(); + } + break; + } + if result.recovery.is_some() || !result.ready.is_empty() { break; } @@ -4416,11 +4626,7 @@ impl FecReorderBuffer { first_missing_index, last_missing_index, }); - result.ready.clear(); - result.nack = None; - self.completed_through = - Some(base + (failed.layout.data_shards + failed.layout.parity_shards) as u64); - self.gap_wait = None; + self.completed_through = Some(failed.end_index()); // Emit any complete successors after the assembler has been marked discontinuous. } result @@ -4538,6 +4744,7 @@ pub struct NvstVideoReceiver { fec_repaired_packets: u64, frames_emitted: u64, replay_rejections: u64, + unprotected_retransmissions: u64, stale_packets: u64, recovered_retransmissions: u64, packet_gap_recoveries: u64, @@ -4574,6 +4781,7 @@ impl NvstVideoReceiver { fec_repaired_packets: 0, frames_emitted: 0, replay_rejections: 0, + unprotected_retransmissions: 0, stale_packets: 0, recovered_retransmissions: 0, packet_gap_recoveries: 0, @@ -4719,6 +4927,21 @@ impl NvstVideoReceiver { } } + /// An unprotected answer to one of this receiver's NACKs, on the bound SSRC. + fn unprotected_retransmission(&mut self, datagram: &[u8], now: Instant) -> Option { + let header = RtpHeader::parse(datagram).ok()?; + if self.bound_ssrc.is_some_and(|ssrc| ssrc != header.ssrc) { + return None; + } + let index = self.srtp.replay.guess_packet_index(header.sequence_number).ok()?; + if !self.config.feedback.take_retransmission_request(index, now) { + return None; + } + let packet = self.srtp.accept_unprotected(datagram, header, index).ok()?; + self.unprotected_retransmissions += 1; + Some(packet) + } + pub fn process_datagram( &mut self, source: SocketAddr, @@ -4747,12 +4970,18 @@ impl NvstVideoReceiver { } let packet = match self.srtp.unprotect(datagram) { Ok(packet) => packet, - Err(reason) => { - if reason == NvstDropReason::ReplayRejected { - self.replay_rejections += 1; + Err(reason) => match (reason == NvstDropReason::AuthenticationFailed) + .then(|| self.unprotected_retransmission(datagram, now)) + .flatten() + { + Some(packet) => packet, + None => { + if reason == NvstDropReason::ReplayRejected { + self.replay_rejections += 1; + } + return vec![NvstReceiveEvent::Dropped(reason)]; } - return vec![NvstReceiveEvent::Dropped(reason)]; - } + }, }; if let Some(expected) = self.config.expected_payload_type && packet.header.payload_type != expected @@ -4818,21 +5047,21 @@ impl NvstVideoReceiver { video, ); - let result = if let Some(layout) = FecPacketLayout::from_packet(&packet) { - if layout.shard_index >= layout.data_shards { - self.fec_packets += 1; - } - let result = self.fec_reorder.push(packet, layout, now); - if layout.shard_index < layout.data_shards - && matches!(result.dropped, Some(NvstDropReason::Unsupported(_))) - { - self.invalidate_picture(now); - } - result - } else { - self.reorder.push(packet, now) + let Some(layout) = FecPacketLayout::from_packet(&packet) else { + let result = self.reorder.push(packet, now); + return self.consume_reorder_result(result, now); }; - self.consume_reorder_result(result, now) + if layout.shard_index >= layout.data_shards { + self.fec_packets += 1; + } + self.tune_fec_reorder(); + let result = self.fec_reorder.push(packet, layout, now); + if layout.shard_index < layout.data_shards + && matches!(result.dropped, Some(NvstDropReason::Unsupported(_))) + { + self.invalidate_picture(now); + } + self.consume_fec_result(result, now) } pub fn poll_recovery(&mut self, now: Instant) -> Vec { @@ -4843,11 +5072,50 @@ impl NvstVideoReceiver { .poll_backstop(now, RFI_BACKSTOP_TIMEOUT, &self.config.feedback); let reordered = self.reorder.poll_deadline(now); let mut events = self.consume_reorder_result(reordered, now); + self.tune_fec_reorder(); let fec = self.fec_reorder.poll_deadline(now); - events.extend(self.consume_reorder_result(fec, now)); + events.extend(self.consume_fec_result(fec, now)); events } + fn tune_fec_reorder(&mut self) { + self.fec_reorder.recovering = self.frame_invalidation.active(); + self.fec_reorder.grace = self.retransmission_grace(); + } + + /// Holds a gap until its retransmission can arrive, while frame types are known. The + /// recovery frame is still requested at the usual time (`anticipate`, or the open episode's + /// request), and a complete recovery frame queued behind the gap ends the wait, so holding + /// longer does not delay recovery. + fn retransmission_grace(&self) -> Duration { + if !self.frame_invalidation.frame_types_known() { + return Duration::ZERO; + } + RETRANSMISSION_WAIT.saturating_sub(MJOLNIR_REORDER_DEQUEUE_TIMEOUT) + } + + /// The FEC queue reports one discontinuity per pass, so that packets completed ahead of + /// it are delivered first. Keep polling while a pass makes progress. + fn consume_fec_result( + &mut self, + mut result: ReorderResult, + now: Instant, + ) -> Vec { + let mut events = Vec::new(); + loop { + let progressed = result.recovery.is_some() || !result.ready.is_empty(); + events.extend(self.consume_reorder_result(result, now)); + if !progressed { + if let Some(frame) = self.fec_reorder.overdue(now) { + self.frame_invalidation + .anticipate(frame, &self.config.feedback); + } + return events; + } + result = self.fec_reorder.poll_deadline(now); + } + } + fn consume_reorder_result( &mut self, result: ReorderResult, @@ -4855,7 +5123,9 @@ impl NvstVideoReceiver { ) -> Vec { self.fec_repaired_packets += result.fec_repaired as u64; let mut events = Vec::new(); - if let Some((first_missing_index, last_missing_index)) = result.nack { + for (first_missing_index, last_missing_index) in + result.nack.into_iter().chain(result.later_nacks) + { self.config .feedback .request_nack(first_missing_index, last_missing_index, now); @@ -4869,7 +5139,15 @@ impl NvstVideoReceiver { if let Some(recovery) = result.recovery { self.packet_gap_recoveries += 1; self.invalidate_picture(now); - self.config.feedback.clear_nacks(); + match recovery { + NvstRecovery::PacketGap { + last_missing_index, .. + } => self + .config + .feedback + .forget_nacks_through(last_missing_index), + _ => self.config.feedback.clear_nacks(), + } events.push(NvstReceiveEvent::RecoveryNeeded(recovery)); } for packet in result.ready { @@ -4998,7 +5276,7 @@ impl NvstVideoReceiver { /// depend on when buffered log lines happen to flush. pub fn stats_line(&self, origin: Instant) -> String { format!( - "elapsed={:.1}s auth={} fec={} fecRecovered={} frames={} replay={} stale={} nackRecovered={} nackPending={} nackOverflow={} gaps={} {} ssrc={:?}", + "elapsed={:.1}s auth={} fec={} fecRecovered={} frames={} replay={} stale={} nackRecovered={} nackPending={} nackOverflow={} unprotectedRtx={} gaps={} {} ssrc={:?}", origin.elapsed().as_secs_f64(), self.authenticated_packets, self.fec_packets, @@ -5014,6 +5292,7 @@ impl NvstVideoReceiver { .unwrap_or_else(std::sync::PoisonError::into_inner) .len(), self.config.feedback.nack_overflows.load(Ordering::Relaxed), + self.unprotected_retransmissions, self.packet_gap_recoveries, self.config.feedback.frame_invalidation.stats(), self.bound_ssrc, @@ -8171,15 +8450,19 @@ fn run_nvst_webrtc_bundle( && let Some((media_ssrc, _)) = feedback.stream_snapshot() { if let Some(channels) = input_channels { - send_pending_nack( - &feedback, - now, - &mut rtc, - channels, - mjolnir, - rtcp_sender_ssrc, - media_ssrc, - ); + for _ in 0..MAX_NACK_REPORTS_PER_TICK { + if !send_pending_nack( + &feedback, + now, + &mut rtc, + channels, + mjolnir, + rtcp_sender_ssrc, + media_ssrc, + ) { + break; + } + } } last_recovery_send = now; } @@ -13584,7 +13867,7 @@ mod tests { plaintext, origin: RtpPacketOrigin::Authenticated, }; - let mut block = FecBlock::new(layout(0), base_index); + let mut block = FecBlock::new(layout(0), base_index, Instant::now()); assert_eq!(block.insert(layout(0), packet(0, data[0].clone())), None); assert_eq!(block.insert(layout(2), packet(2, data[2].clone())), None); assert_eq!(block.insert(layout(3), packet(3, data[3].clone())), None); @@ -13678,7 +13961,7 @@ mod tests { plaintext, origin: RtpPacketOrigin::Authenticated, }; - let mut block = FecBlock::new(layout(0), base_index); + let mut block = FecBlock::new(layout(0), base_index, Instant::now()); assert_eq!(block.insert(layout(1), packet(1, parity[0].clone())), None); assert!(block.has_enough_shards()); diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs index a5e8a41db..19865a54d 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_invalidation.rs @@ -37,7 +37,7 @@ impl NvstFrameType { } } - fn clears_invalidation(self) -> bool { + pub(crate) fn clears_invalidation(self) -> bool { matches!(self, Self::I | Self::IntraRefresh | Self::RefPicI) } } @@ -126,9 +126,21 @@ pub(crate) struct FrameInvalidation { active_since: Option, last_retry_at: Option, backstop_requested: bool, + /// The last assembled frame when a request went out ahead of a give-up. + anticipated: Option, } impl FrameInvalidation { + /// Whether a loss episode is open, i.e. frames are withheld until a recovery frame. + pub(crate) fn active(&self) -> bool { + self.active_since.is_some() + } + + /// Whether recovery is reported by frame range, so a held gap cannot delay it. + pub(crate) fn frame_types_known(&self) -> bool { + self.frame_types_known + } + pub(crate) fn reset(&mut self, feedback: &NvstFeedbackState) { feedback.frame_invalidation.discard(); *self = Self::default(); @@ -144,6 +156,33 @@ impl FrameInvalidation { true } + /// A gap has waited the usual time for its retransmission and is held a little longer. + /// Report the frames at risk now, when the give-up would have, so the server's recovery + /// frame is no later than without the extra wait. Nothing is withheld: if the + /// retransmission completes the gap, its frames are delivered and the server's answer is an + /// ordinary keyframe. If not, the give-up follows with the full range. One request per stall. + pub(crate) fn anticipate(&mut self, revealed_by_frame: u32, feedback: &NvstFeedbackState) { + let Some(last) = self.last_frame else { + return; + }; + if !self.frame_types_known || self.active_since.is_some() || self.anticipated == Some(last) + { + return; + } + self.anticipated = Some(last); + let first = u64::from(last) + 1; + let range = ( + first, + first.max(u64::from(revealed_by_frame).saturating_sub(1)), + ); + feedback.frame_invalidation.queue(range); + opennow_streamer_protocol::log::log_async( + "INFO", + "nvst-rfi", + &format!("early request frames={}..={}", range.0, range.1), + ); + } + pub(crate) fn poll_backstop( &mut self, now: Instant, @@ -271,5 +310,6 @@ impl FrameInvalidation { self.active_since = None; self.last_retry_at = None; self.backstop_requested = false; + self.anticipated = None; } } diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs index c5b72b343..3987aaa2e 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_nack_tests.rs @@ -448,3 +448,20 @@ fn resolving_holes_and_readding_ranges_does_not_reset_nack_attempts() { None ); } + +#[test] +fn nacked_packets_are_accepted_unprotected_once_within_the_window() { + let now = Instant::now(); + let feedback = NvstFeedbackState::default(); + feedback.request_nack(100, 102, now); + assert_eq!(feedback.take_nack(now, Some(DEFAULT_NACK_RTT)), Some((100, 102))); + + let answered = now + Duration::from_millis(200); + assert!(feedback.take_retransmission_request(101, answered)); + assert!(!feedback.take_retransmission_request(101, answered), "accepted only once"); + assert!(!feedback.take_retransmission_request(103, answered), "never requested"); + assert!( + !feedback.take_retransmission_request(100, now + RETRANSMISSION_ACCEPT_WINDOW), + "requests expire" + ); +} diff --git a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs index 7cac69fe7..c0460e380 100644 --- a/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs +++ b/native/opennow-streamer/crates/opennow-streamer-transport/src/nvst_recovery_tests.rs @@ -57,9 +57,12 @@ fn multiple_reorder_gaps_do_not_revalidate_post_gap_references() { _ => None, }) .collect(); + // The FEC queue reports each gap separately, so the complete keyframe between the + // two gaps is delivered instead of being discarded with the second one. + let between_gaps: &[(u32, bool, bool)] = if fec { &[(2, true, false)] } else { &[] }; assert_eq!( frames, - [(100, false, false), (101, false, true)], + [between_gaps, &[(100, false, false), (101, false, true)]].concat(), "fec={fec}" ); assert!(receiver.config.feedback.keyframe_request_pending()); @@ -958,3 +961,405 @@ fn rfi_retries_preserve_the_episode_deadline_and_stop_on_recovery() { assert_eq!(stream.feedback.frame_invalidation.take(), None); assert!(!stream.feedback.keyframe_request_pending()); } + +/// One frame as a single FEC block of two data packets; its parity packet (index `3 * frame +/// + 2`) is never sent, so a frame that loses a data packet cannot be reconstructed. +fn typed_fec_frame(frame: u32, frame_type: u8, nal: u8) -> [Vec; 2] { + let base = (frame * 3) as u16; + let with_fec_word = |mut packet: Vec, shard: u32| { + packet[28..32].copy_from_slice(&((50_u32 << 4) | (shard << 12) | (2 << 22)).to_le_bytes()); + packet + }; + [ + with_fec_word( + build_plaintext_rtp( + base, + FLAG_SOF | FLAG_CONTAINS_PIC_DATA, + frame, + &[0x01, 0, 0, frame_type, 0, 0, 0, 0, 0, 0, 0, 1, nal, 0x88], + ), + 0, + ), + with_fec_word( + build_plaintext_rtp(base + 1, FLAG_EOF | FLAG_CONTAINS_PIC_DATA, frame, &[0x99]), + 1, + ), + ] +} + +impl TypedStream { + fn fec_frame(&mut self, frame: u32, frame_type: u8, after: Duration) -> Vec<(u32, bool, bool)> { + let nal = if frame_type == 2 { 0x65 } else { 0x41 }; + let [first, last] = typed_fec_frame(frame, frame_type, nal); + let mut frames = self.frames(first, after); + frames.extend(self.frames(last, after)); + frames + } + + /// Delivers only the first packet of a P-frame. + fn damaged_fec_frame(&mut self, frame: u32, after: Duration) { + let [first, _] = typed_fec_frame(frame, 1, 0x41); + assert!(self.push(first, after).is_empty()); + } + + fn poll(&mut self, after: Duration) -> Vec { + self.receiver.poll_recovery(self.origin + after) + } +} + +fn gaps(events: &[NvstReceiveEvent]) -> Vec<(u64, u64)> { + events + .iter() + .filter_map(|event| match event { + NvstReceiveEvent::RecoveryNeeded(NvstRecovery::PacketGap { + first_missing_index, + last_missing_index, + }) => Some((*first_missing_index, *last_missing_index)), + _ => None, + }) + .collect() +} + +#[test] +fn consecutive_damaged_frames_share_one_nack_window() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + assert_eq!(stream.fec_frame(2, 1, ms(0)), [(2, false, true)]); + // Frames 3, 4 and 5 each lose a packet. Each loss is evident when the next frame starts. + stream.damaged_fec_frame(3, ms(0)); + stream.damaged_fec_frame(4, ms(1)); + stream.damaged_fec_frame(5, ms(2)); + assert!(stream.fec_frame(6, 1, ms(3)).is_empty()); + // All three retransmissions are requested at once, not one per expired wait. + let now = stream.origin + ms(3); + assert_eq!(stream.feedback.take_nack(now, None), Some((10, 10))); + assert_eq!(stream.feedback.take_nack(now, None), Some((13, 13))); + assert_eq!(stream.feedback.take_nack(now, None), Some((16, 16))); + + assert!(stream.poll(RETRANSMISSION_WAIT).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(10, 10)]); + // Serial waits would hold frame 6, and with it the invalidation request, three times as long. + let events = stream.poll(RETRANSMISSION_WAIT + ms(3)); + assert_eq!(gaps(&events), [(13, 13), (16, 16)]); + assert!(delivered(events).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((3, 5))); + assert_eq!(stream.fec_frame(7, 2, RETRANSMISSION_WAIT + ms(8)), [(7, true, true)]); + assert_eq!(stream.receiver.packet_gap_recoveries, 3); + assert_eq!( + stream.stats(), + "rfiSent=0 rfiDropped=1 rfiRecoveredI=1 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0 rfiIdrBackstop=0" + ); +} + +#[test] +fn a_recovery_frame_between_two_expired_gaps_is_delivered() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 2, ms(1)).is_empty()); + stream.damaged_fec_frame(4, ms(2)); + assert!(stream.fec_frame(5, 1, ms(3)).is_empty()); + // Both waits are over by the time the queue is polled. The I-frame sits between the + // two discontinuities and must reach the decoder. + let events = stream.poll(RETRANSMISSION_WAIT + ms(20)); + assert_eq!(gaps(&events), [(7, 7), (13, 13)]); + assert_eq!(delivered(events), [(3, true, true)]); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((4, 4))); +} + +#[test] +fn giving_up_on_a_gap_keeps_the_requests_for_later_gaps() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + stream.damaged_fec_frame(4, ms(30)); + assert!(stream.fec_frame(5, 1, ms(31)).is_empty()); + let sent_at = stream.origin + ms(31); + assert_eq!(stream.feedback.take_nack(sent_at, None), Some((7, 7))); + assert_eq!(stream.feedback.take_nack(sent_at, None), Some((13, 13))); + + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(7, 7)]); + let pending = stream.feedback.pending_nacks.lock().unwrap(); + assert_eq!(pending.keys().copied().collect::>(), [13]); + assert_eq!(pending[&13].attempts, 1, "the request was not started over"); + assert_eq!(pending[&13].last_sent_at, Some(sent_at)); +} + +#[test] +fn fec_holes_wait_from_the_packet_that_revealed_them() { + let now = Instant::now(); + let ms = Duration::from_millis; + let mut reorder = FecReorderBuffer::new(48); + // One data and one parity packet per block: block `n` owns indices `2n` and `2n + 1`. + let data = |block: u64| { + let plaintext = build_plaintext_rtp( + (block * 2) as u16, + FLAG_SOF | FLAG_EOF, + block as u32, + &[0, 0, 0, 1, 0x65], + ); + let packet = RtpPacket { + index: block * 2, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let layout = FecPacketLayout { + frame_index: block as u32, + block_index: 0, + last_block_index: 0, + shard_index: 0, + data_shards: 1, + parity_shards: 1, + }; + (packet, layout) + }; + let indices = |result: &ReorderResult| -> Vec { + result.ready.iter().map(|packet| packet.index).collect() + }; + let (packet, layout) = data(0); + assert_eq!(indices(&reorder.push(packet, layout, now)), [0]); + // Blocks 1 and 3 are lost whole. + let (packet, layout) = data(2); + let first_hole = reorder.push(packet, layout, now); + assert_eq!(first_hole.nack, Some((2, 3))); + let (packet, layout) = data(4); + let second_hole = reorder.push(packet, layout, now + ms(10)); + assert_eq!(second_hole.later_nacks, [(6, 7)]); + assert!(second_hole.ready.is_empty()); + + let at = now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT; + let first = reorder.poll_deadline(at); + assert_eq!( + first.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 2, + last_missing_index: 3 + }) + ); + assert_eq!(indices(&first), [4]); + let waiting = reorder.poll_deadline(at + ms(10) - Duration::from_nanos(1)); + assert_eq!(waiting.recovery, None); + assert_eq!(waiting.nack, Some((6, 7))); + // The second hole was revealed 10 ms after the first and ends 10 ms after it, not 52. + let second = reorder.poll_deadline(at + ms(10)); + assert_eq!( + second.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 6, + last_missing_index: 7 + }) + ); + assert_eq!(indices(&second), [8]); + assert!(reorder.blocks.is_empty()); +} + +#[test] +fn a_block_that_fills_a_hole_does_not_restart_the_wait() { + let now = Instant::now(); + let ms = Duration::from_millis; + let mut reorder = FecReorderBuffer::new(48); + // Two data packets and one parity packet per block: block `n` starts at index `3n`. + let packet = |index: u64| { + let plaintext = build_plaintext_rtp(index as u16, FLAG_SOF, (index / 3) as u32, &[0x55]); + let packet = RtpPacket { + index, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let layout = FecPacketLayout { + frame_index: (index / 3) as u32, + block_index: 0, + last_block_index: 0, + shard_index: (index % 3) as usize, + data_shards: 2, + parity_shards: 1, + }; + (packet, layout) + }; + let mut push = |index: u64, at: Instant| { + let (packet, layout) = packet(index); + reorder.push(packet, layout, at) + }; + push(0, now); + assert_eq!(push(1, now).ready.len(), 2); + // Block 1 is lost whole, then one of its packets is retransmitted 40 ms later. + push(6, now); + assert_eq!(push(7, now).nack, Some((3, 5))); + let partial = push(3, now + ms(40)); + assert_eq!(partial.nack, Some((4, 4))); + assert!(partial.recovery.is_none()); + let just_before = now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT - Duration::from_nanos(1); + assert!(reorder.poll_deadline(just_before).recovery.is_none()); + let expired = reorder.poll_deadline(now + MJOLNIR_REORDER_DEQUEUE_TIMEOUT); + assert_eq!( + expired.recovery, + Some(NvstRecovery::PacketGap { + first_missing_index: 4, + last_missing_index: 4 + }) + ); + assert_eq!(expired.ready.len(), 2); +} + +#[test] +fn a_stalled_block_requests_every_run_it_needs_at_once() { + let now = Instant::now(); + let mut reorder = FecReorderBuffer::new(48); + // Six data packets and two parity packets per block: block `n` starts at index `8n`. + let mut push = |index: u64| { + let plaintext = build_plaintext_rtp(index as u16, FLAG_SOF, (index / 8) as u32, &[0x55]); + let packet = RtpPacket { + index, + header: RtpHeader::parse(&plaintext).unwrap(), + plaintext, + origin: RtpPacketOrigin::Authenticated, + }; + let layout = FecPacketLayout { + frame_index: (index / 8) as u32, + block_index: 0, + last_block_index: 0, + shard_index: (index % 8) as usize, + data_shards: 6, + parity_shards: 2, + }; + reorder.push(packet, layout, now) + }; + // Block 0 keeps data 0 and 3: it is four packets short, in two separate runs. + push(0); + push(3); + // Block 1 keeps data 8, 11, 12, 13 and one parity packet: one more packet completes it. + assert_eq!(push(8).later_nacks, [(1, 2), (4, 5)]); + for index in [11, 12, 13, 14] { + assert!(push(index).later_nacks.is_empty()); + } + assert_eq!(push(16).later_nacks, [(9, 10)]); +} + +#[test] +fn a_retransmission_that_misses_the_usual_wait_is_used() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + // At the usual give-up the recovery frame is requested, but the gap stays open. + assert!(stream.poll(MJOLNIR_REORDER_DEQUEUE_TIMEOUT + ms(1)).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + let [_, retransmitted] = typed_fec_frame(2, 1, 0x41); + assert_eq!( + stream.frames(retransmitted, ms(70)), + [(2, false, true), (3, false, true)] + ); + // The server's answer reaches the decoder as an ordinary keyframe. + assert_eq!(stream.fec_frame(4, 2, ms(90)), [(4, true, true)]); + assert_eq!(stream.receiver.packet_gap_recoveries, 0); + assert_eq!( + stream.stats(), + "rfiSent=0 rfiDropped=0 rfiRecoveredI=0 rfiRecoveredIntraRefresh=0 rfiRecoveredRefPicI=0 rfiIdrBackstop=0" + ); +} + +#[test] +fn a_gap_without_its_retransmission_is_given_up_after_the_retransmission_wait() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert!(stream.poll(ms(53)).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + // Only one request per stall goes out ahead of the give-up. + assert!(stream.poll(RETRANSMISSION_WAIT).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), None); + let events = stream.poll(RETRANSMISSION_WAIT + ms(1)); + assert_eq!(gaps(&events), [(7, 7)]); + assert!(delivered(events).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + assert_eq!(stream.fec_frame(4, 2, RETRANSMISSION_WAIT + ms(8)), [(4, true, true)]); +} + +#[test] +fn a_gap_while_a_recovery_is_open_waits_for_its_retransmission_too() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(7, 7)]); + // The recovery frame may itself be the damaged frame, so its gap waits as long. A + // complete recovery frame behind the gap would end the wait at once (next test). + stream.damaged_fec_frame(4, RETRANSMISSION_WAIT + ms(2)); + assert!(stream.fec_frame(5, 1, RETRANSMISSION_WAIT + ms(3)).is_empty()); + assert!(stream.poll(RETRANSMISSION_WAIT * 2 + ms(2)).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT * 2 + ms(3))), [(13, 13)]); +} + +#[test] +fn the_extra_wait_does_not_repeat_the_retransmission_request() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(30)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert_eq!(stream.feedback.take_nack(stream.origin + ms(1), None), Some((7, 7))); + // A second request now could not be answered before the extra wait ends. + assert!(stream.poll(ms(60)).is_empty()); + assert_eq!(stream.feedback.take_nack(stream.origin + ms(60), None), None); +} + +#[test] +fn a_recovery_frame_behind_a_gap_ends_an_open_episode_at_once() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert_eq!(gaps(&stream.poll(RETRANSMISSION_WAIT + ms(1))), [(7, 7)]); + // A later frame is damaged too, and the server's recovery frame arrives right behind it. + stream.damaged_fec_frame(4, RETRANSMISSION_WAIT + ms(8)); + let [first, last] = typed_fec_frame(5, 2, 0x65); + assert!(stream.push(first, RETRANSMISSION_WAIT + ms(9)).is_empty()); + let events = stream.push(last, RETRANSMISSION_WAIT + ms(9)); + assert_eq!(gaps(&events), [(13, 13)]); + assert_eq!(delivered(events), [(5, true, true)]); + assert_eq!(stream.feedback.frame_invalidation.take(), None); +} + +#[test] +fn a_recovery_frame_that_arrives_inside_the_extra_wait_ends_it() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + stream.feedback.publish_ping(true, stream.origin, ms(50)); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 1, ms(1)).is_empty()); + assert!(stream.poll(ms(53)).is_empty()); + assert_eq!(stream.feedback.frame_invalidation.take(), Some((2, 2))); + // The answer comes back sooner than the last ping suggested: 80 ms, not 103 ms. + assert_eq!(stream.fec_frame(4, 2, ms(80)), [(4, true, true)]); + assert_eq!(stream.receiver.packet_gap_recoveries, 1); +} + +#[test] +fn an_i_frame_behind_a_fresh_gap_does_not_cut_the_retransmission_short() { + let ms = Duration::from_millis; + let mut stream = TypedStream::new(); + assert_eq!(stream.fec_frame(1, 2, ms(0)), [(1, true, true)]); + stream.damaged_fec_frame(2, ms(0)); + assert!(stream.fec_frame(3, 2, ms(1)).is_empty()); + let [_, retransmitted] = typed_fec_frame(2, 1, 0x41); + assert_eq!( + stream.frames(retransmitted, ms(40)), + [(2, false, true), (3, true, true)] + ); +}