From 6a97655469e6c6df1e31d43fc72ff38c32252629 Mon Sep 17 00:00:00 2001 From: IsaFxck Date: Sun, 6 Sep 2026 22:17:05 +0300 Subject: [PATCH 1/2] fix: keep rate admission before preencoded video submission --- include/livekit/video_source.h | 2 ++ .../0001-native-v2-runtime.patch | 32 +++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/include/livekit/video_source.h b/include/livekit/video_source.h index f3f233e..ff9884b 100644 --- a/include/livekit/video_source.h +++ b/include/livekit/video_source.h @@ -135,6 +135,8 @@ class LIVEKIT_API EncodedVideoSource final : public VideoSource { EncodedVideoSource(int width, int height); /// @brief Submit one H.264 access unit synchronously. + /// Callers own bounded raw-input admission and bitrate control before encoding. + /// The SDK does not use its rate dropper on pre-encoded reference frames. /// @param frame Borrowed access-unit bytes and frame metadata. /// @return Whether LiveKit accepted the access unit for delivery. /// @throws std::invalid_argument if the payload is empty. diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index cd0a0fd..1547d9f 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -13,6 +13,21 @@ index 00000000..0dffd207 +FFI read and retain WebRTC frame backpressure for external encoders. The bitrate +request does not change source dimensions or target FPS, and is distinct from a +fresh network-capacity estimate. Reference continuity remains fenced after drops. +diff --git a/.changeset/preencoded-raw-admission.md b/.changeset/preencoded-raw-admission.md +new file mode 100644 +index 00000000..2aeab771 +--- /dev/null ++++ b/.changeset/preencoded-raw-admission.md +@@ -0,0 +1,9 @@ ++--- ++webrtc-sys: patch ++--- ++ ++Keep the bitrate-based frame dropper disabled for pre-encoded video. The source ++owns bounded raw-input admission and applies sender rate feedback before ++encoding. Dropping encoded reference frames merely creates repeated keyframe ++recovery and is not a substitute for source backpressure. The existing ++continuity guard still handles unavoidable missing encoded frames. diff --git a/.changeset/preencoded-reference-continuity.md b/.changeset/preencoded-reference-continuity.md new file mode 100644 index 00000000..d267a689 @@ -1430,7 +1445,7 @@ index 21337794..042ee055 100644 static std::shared_ptr video_to_media( std::shared_ptr track) { diff --git a/webrtc-sys/src/passthrough_video_encoder.cpp b/webrtc-sys/src/passthrough_video_encoder.cpp -index cd45b9e3..1981a4e9 100644 +index cd45b9e3..9d50e638 100644 --- a/webrtc-sys/src/passthrough_video_encoder.cpp +++ b/webrtc-sys/src/passthrough_video_encoder.cpp @@ -26,12 +26,13 @@ @@ -1528,18 +1543,17 @@ index cd45b9e3..1981a4e9 100644 RTC_LOG(LS_ERROR) << "PassthroughVideoEncoder callback failed " << result.error; return WEBRTC_VIDEO_CODEC_ERROR; -@@ -331,6 +350,10 @@ class PassthroughVideoEncoder final : public VideoEncoder { +@@ -331,6 +350,9 @@ class PassthroughVideoEncoder final : public VideoEncoder { info.supports_native_handle = true; info.implementation_name = "LiveKit pre-encoded passthrough"; info.scaling_settings = VideoEncoder::ScalingSettings::kOff; -+ // External encoders may reach their bitrate floor above available -+ // bandwidth. Keep WebRTC backpressure; the continuity guard requests IDR -+ // after a drop. -+ info.has_trusted_rate_controller = false; ++ // Encoded reference frames cannot be dropped to meet a rate allocation. ++ // Forward rate feedback to the source; its raw admission controls pressure. ++ info.has_trusted_rate_controller = true; info.is_hardware_accelerated = false; info.supports_simulcast = false; info.preferred_pixel_formats = {VideoFrameBuffer::Type::kNative}; -@@ -338,8 +361,7 @@ class PassthroughVideoEncoder final : public VideoEncoder { +@@ -338,8 +360,7 @@ class PassthroughVideoEncoder final : public VideoEncoder { } private: @@ -1549,7 +1563,7 @@ index cd45b9e3..1981a4e9 100644 std::optional request; { webrtc::MutexLock lock(&rate_control_mutex_); -@@ -357,10 +379,13 @@ class PassthroughVideoEncoder final : public VideoEncoder { +@@ -357,10 +378,13 @@ class PassthroughVideoEncoder final : public VideoEncoder { VideoCodecType codec_type_; VideoCodec codec_; EncodedImageCallback* encoded_image_callback_ = nullptr; @@ -1564,7 +1578,7 @@ index cd45b9e3..1981a4e9 100644 }; } // namespace -@@ -388,8 +413,8 @@ PassthroughVideoEncoderFactory::GetSupportedFormats() const { +@@ -388,8 +412,8 @@ PassthroughVideoEncoderFactory::GetSupportedFormats() const { return supported_formats_; } From 1c43b0ec24642bf3cac98ae9d103026b763f3db8 Mon Sep 17 00:00:00 2001 From: IsaFxck Date: Mon, 7 Sep 2026 14:34:03 +0300 Subject: [PATCH 2/2] fix(video): flush superseded pacing packets at keyframes --- .../0001-native-v2-runtime.patch | 56 +++++++++++++++++-- patches/client-sdk-rust/README.md | 17 ++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index 1547d9f..fa52a2b 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -13,6 +13,22 @@ index 00000000..0dffd207 +FFI read and retain WebRTC frame backpressure for external encoders. The bitrate +request does not change source dimensions or target FPS, and is distinct from a +fresh network-capacity estimate. Reference continuity remains fenced after drops. +diff --git a/.changeset/preencoded-keyframe-pacing.md b/.changeset/preencoded-keyframe-pacing.md +new file mode 100644 +index 00000000..c87c547b +--- /dev/null ++++ b/.changeset/preencoded-keyframe-pacing.md +@@ -0,0 +1,10 @@ ++--- ++webrtc-sys: patch ++--- ++ ++Enable WebRTC keyframe queue flushing in the zero-playout-delay factory. When the first packet ++of a new keyframe arrives, superseded queued packets for that video stream and ++its RTX stream are removed unless another keyframe is still being paced. ++Audio and unrelated streams retain their queues. This bounds recovery work ++without dropping arbitrary encoded references, changing capture cadence, or ++replacing an encoder or publication. diff --git a/.changeset/preencoded-raw-admission.md b/.changeset/preencoded-raw-admission.md new file mode 100644 index 00000000..2aeab771 @@ -1700,7 +1716,7 @@ index 6b6f6028..c8fe0fb2 100644 observer_->on_ice_candidates_removed(std::move(vec)); } diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp -index fe76067f..02463b1c 100644 +index fe76067f..c6d2b1c7 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -19,21 +19,21 @@ @@ -1739,7 +1755,7 @@ index fe76067f..02463b1c 100644 #include "rtc_base/thread.h" #include "webrtc-sys/src/peer_connection.rs.h" #include "webrtc-sys/src/peer_connection_factory.rs.h" -@@ -49,6 +52,27 @@ +@@ -49,25 +52,52 @@ namespace livekit_ffi { namespace { @@ -1767,7 +1783,35 @@ index fe76067f..02463b1c 100644 constexpr char kForcePlayoutDelayFieldTrial[] = "WebRTC-ForcePlayoutDelay/min_ms:0,max_ms:0/"; constexpr char kForcePlayoutDelayValue[] = "min_ms:0,max_ms:0"; -@@ -83,14 +107,24 @@ PeerConnectionFactory::PeerConnectionFactory( + +-class ZeroPlayoutDelayFieldTrials final : public webrtc::FieldTrialsView { ++class RealtimeMediaFieldTrials final : public webrtc::FieldTrialsView { + public: + std::string Lookup(absl::string_view key) const override { ++ // A new random-access frame supersedes queued dependent video. WebRTC ++ // flushes only that stream and its RTX packets, and never interrupts a ++ // keyframe already being paced. Audio and other streams keep their queues. ++ if (key == "WebRTC-Pacer-KeyframeFlushing") { ++ return "Enabled"; ++ } + return key == "WebRTC-ForcePlayoutDelay" ? kForcePlayoutDelayValue : ""; + } + + std::unique_ptr CreateCopy() const override { +- return std::make_unique(); ++ return std::make_unique(); + } + }; + + webrtc::Environment CreateEnvironment(bool zero_playout_delay) { + if (zero_playout_delay) { + return webrtc::CreateEnvironment( +- std::make_unique()); ++ std::make_unique()); + } + return webrtc::CreateEnvironment(); + } +@@ -83,14 +113,24 @@ PeerConnectionFactory::PeerConnectionFactory( PeerConnectionFactory::PeerConnectionFactory( std::shared_ptr rtc_runtime, bool zero_playout_delay) @@ -1795,7 +1839,7 @@ index fe76067f..02463b1c 100644 dependencies.env = env_; if (zero_playout_delay) { -@@ -111,9 +145,12 @@ PeerConnectionFactory::PeerConnectionFactory( +@@ -111,9 +151,12 @@ PeerConnectionFactory::PeerConnectionFactory( std::move(std::make_unique()); dependencies.video_decoder_factory = std::move(std::make_unique()); @@ -1811,7 +1855,7 @@ index fe76067f..02463b1c 100644 webrtc::EnableMedia(dependencies); peer_factory_ = -@@ -130,17 +167,26 @@ PeerConnectionFactory::~PeerConnectionFactory() { +@@ -130,17 +173,26 @@ PeerConnectionFactory::~PeerConnectionFactory() { peer_factory_ = nullptr; audio_device_ = nullptr; @@ -1841,7 +1885,7 @@ index fe76067f..02463b1c 100644 throw std::runtime_error(serialize_error(to_error(webrtc::RTCError( webrtc::RTCErrorType::INTERNAL_ERROR, "failed to initialize pc")))); } -@@ -197,7 +243,8 @@ RtpCapabilities PeerConnectionFactory::rtp_receiver_capabilities( +@@ -197,7 +249,8 @@ RtpCapabilities PeerConnectionFactory::rtp_receiver_capabilities( static_cast(type))); } diff --git a/patches/client-sdk-rust/README.md b/patches/client-sdk-rust/README.md index e73e152..7d02d3b 100644 --- a/patches/client-sdk-rust/README.md +++ b/patches/client-sdk-rust/README.md @@ -1,5 +1,22 @@ # Native v2 Rust patch +The keyframe pacing correction enables WebRTC's existing +`WebRTC-Pacer-KeyframeFlushing` behavior in the zero-playout-delay factory +environment; the default factory remains unchanged. On the first +packet of a new keyframe, the pacer removes superseded queued video and RTX +packets for that stream. A keyframe already being paced is preserved; audio and +other stream queues are unaffected. It does not impose a frame quota or change +encoder, source, publication identity, or the selected screen preset. + +The pinned WebRTC implementation and its `FlushesPacketsOnKeyFrames` test are +in [pacing_controller.cc](https://webrtc.googlesource.com/src/+/refs/branch-heads/7559/modules/pacing/pacing_controller.cc) +and [pacing_controller_unittest.cc](https://webrtc.googlesource.com/src/+/refs/branch-heads/7559/modules/pacing/pacing_controller_unittest.cc). +Two app #139 preview-stall candidate runs passed the unchanged full-interval +receiver threshold: p95 132/133 ms, maximum gap 532/467 ms, zero invalid CRC +markers, continuous screen audio/remote voice, and unchanged publication and +encoder identity. These candidate runs do not replace qualification of the +published SDK pin in the application repository. + The v1.10.0-syrnike.9 candidate has normalized tree `8f5c3220de048f23a8f17b4a65653be3c5b11179`. Session shutdown now cancels and joins publisher negotiation tasks before closing signaling and peer