diff --git a/include/livekit/video_source.h b/include/livekit/video_source.h index 904d136..f3f233e 100644 --- a/include/livekit/video_source.h +++ b/include/livekit/video_source.h @@ -144,6 +144,13 @@ class LIVEKIT_API EncodedVideoSource final : public VideoSource { /// @brief Return and clear a pending PLI/FIR key-frame request. /// @return Whether the application encoder must produce a key frame. bool takeKeyFrameRequest(); + + /// @brief Consume the newest WebRTC sender bitrate allocation. + /// @return Latest bitrate in bits per second, or no value when unchanged. + /// Zero is a valid allocation under congestion. This is sender control state, + /// not a fresh network-capacity measurement; it never changes the source FPS. + /// @throws std::runtime_error if the FFI request fails. + [[nodiscard]] std::optional takeBitrateRequest(); }; } // namespace livekit diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index 8d9af23..cd0a0fd 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -1,3 +1,18 @@ +diff --git a/.changeset/preencoded-bitrate-feedback.md b/.changeset/preencoded-bitrate-feedback.md +new file mode 100644 +index 00000000..0dffd207 +--- /dev/null ++++ b/.changeset/preencoded-bitrate-feedback.md +@@ -0,0 +1,9 @@ ++--- ++livekit-ffi: patch ++webrtc-sys: patch ++--- ++ ++Expose the current pre-encoded sender bitrate allocation through a coalesced ++FFI read and retain WebRTC frame backpressure for external encoders. The bitrate ++request does not change source dimensions or target FPS, and is distinct from a ++fresh network-capacity estimate. Reference continuity remains fenced after drops. diff --git a/.changeset/preencoded-reference-continuity.md b/.changeset/preencoded-reference-continuity.md new file mode 100644 index 00000000..d267a689 @@ -99,42 +114,44 @@ index 00a7a919..b033b780 100644 self.handle.sender() } diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto -index 53ea782b..74276479 100644 +index 53ea782b..1d03c91c 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto -@@ -89,6 +89,8 @@ message FfiRequest { +@@ -89,6 +89,9 @@ message FfiRequest { CaptureVideoFrameRequest capture_video_frame = 22; VideoConvertRequest video_convert = 23; VideoStreamFromParticipantRequest video_stream_from_participant = 24; + CaptureEncodedVideoFrameRequest capture_encoded_video_frame = 87; + TakeVideoKeyFrameRequestRequest take_video_key_frame_request = 88; ++ TakeVideoBitrateRequestRequest take_video_bitrate_request = 89; // Audio NewAudioStreamRequest new_audio_stream = 25; -@@ -184,7 +186,7 @@ message FfiRequest { +@@ -184,7 +187,7 @@ message FfiRequest { // Room event ready signal ReadyForRoomEventRequest ready_for_room_event = 83; - // NEXT_ID: 87 -+ // NEXT_ID: 89 ++ // NEXT_ID: 90 } } -@@ -221,6 +223,8 @@ message FfiResponse { +@@ -221,6 +224,9 @@ message FfiResponse { CaptureVideoFrameResponse capture_video_frame = 22; VideoConvertResponse video_convert = 23; VideoStreamFromParticipantResponse video_stream_from_participant = 24; + CaptureEncodedVideoFrameResponse capture_encoded_video_frame = 87; + TakeVideoKeyFrameRequestResponse take_video_key_frame_request = 88; ++ TakeVideoBitrateRequestResponse take_video_bitrate_request = 89; // Audio NewAudioStreamResponse new_audio_stream = 25; -@@ -313,7 +317,7 @@ message FfiResponse { +@@ -313,7 +319,7 @@ message FfiResponse { // Room event ready signal ReadyForRoomEventResponse ready_for_room_event = 82; - // NEXT_ID: 87 -+ // NEXT_ID: 89 ++ // NEXT_ID: 90 } } @@ -151,10 +168,10 @@ index df08ed5b..ab1068a5 100644 // Controls how the encoder degrades quality when bandwidth is constrained. diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto -index a4b610e0..eadb22ca 100644 +index a4b610e0..82b60f0c 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto -@@ -82,6 +82,22 @@ message CaptureVideoFrameRequest { +@@ -82,6 +82,27 @@ message CaptureVideoFrameRequest { message CaptureVideoFrameResponse {} @@ -173,11 +190,16 @@ index a4b610e0..eadb22ca 100644 + +message TakeVideoKeyFrameRequestRequest { required uint64 source_handle = 1; } +message TakeVideoKeyFrameRequestResponse { required bool requested = 1; } ++ ++// Current sender allocation, coalesced until read. This is a bitrate command, ++// not an estimate of available network capacity or a request to change FPS. ++message TakeVideoBitrateRequestRequest { required uint64 source_handle = 1; } ++message TakeVideoBitrateRequestResponse { optional uint64 target_bitrate_bps = 1; } + message VideoConvertRequest { optional bool flip_y = 1; required VideoBufferInfo buffer = 2; -@@ -207,6 +223,7 @@ message VideoSourceResolution { +@@ -207,6 +228,7 @@ message VideoSourceResolution { enum VideoSourceType { VIDEO_SOURCE_NATIVE = 0; @@ -208,10 +230,10 @@ index 4922add5..281dcfc7 100644 for (proto_backend, expected) in cases { diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs -index 8ee56988..80b299b7 100644 +index 8ee56988..95f1c87c 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs -@@ -520,6 +520,27 @@ unsafe fn on_capture_video_frame( +@@ -520,6 +520,37 @@ unsafe fn on_capture_video_frame( Ok(proto::CaptureVideoFrameResponse::default()) } @@ -235,11 +257,21 @@ index 8ee56988..80b299b7 100644 + let source = server.retrieve_handle::(request.source_handle)?; + Ok(proto::TakeVideoKeyFrameRequestResponse { requested: source.take_key_frame_request()? }) +} ++ ++fn on_take_video_bitrate_request( ++ server: &'static FfiServer, ++ request: proto::TakeVideoBitrateRequestRequest, ++) -> FfiResult { ++ let source = server.retrieve_handle::(request.source_handle)?; ++ Ok(proto::TakeVideoBitrateRequestResponse { ++ target_bitrate_bps: source.take_bitrate_request()?, ++ }) ++} + /// Convert a video frame /// /// # Safety: The user must ensure that the pointers/len provided are valid -@@ -1390,6 +1411,12 @@ pub fn handle_request( +@@ -1390,6 +1421,13 @@ pub fn handle_request( } Request::NewVideoSource(req) => on_new_video_source(server, req)?.into(), Request::CaptureVideoFrame(req) => unsafe { on_capture_video_frame(server, req)?.into() }, @@ -249,11 +281,12 @@ index 8ee56988..80b299b7 100644 + Request::TakeVideoKeyFrameRequest(req) => { + on_take_video_key_frame_request(server, req)?.into() + } ++ Request::TakeVideoBitrateRequest(req) => on_take_video_bitrate_request(server, req)?.into(), Request::VideoConvert(req) => unsafe { on_video_convert(server, req)?.into() }, Request::NewAudioStream(req) => on_new_audio_stream(server, req)?.into(), Request::NewAudioSource(req) => on_new_audio_source(server, req)?.into(), diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs -index ee1410a7..ab0284f6 100644 +index ee1410a7..1b280e2d 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -16,9 +16,13 @@ use super::{colorcvt, FfiHandle}; @@ -285,7 +318,7 @@ index ee1410a7..ab0284f6 100644 _ => return Err(FfiError::InvalidRequest("unsupported video source type".into())), }; -@@ -94,6 +105,57 @@ impl FfiVideoSource { +@@ -94,6 +105,71 @@ impl FfiVideoSource { } Ok(()) } @@ -339,6 +372,20 @@ index ee1410a7..ab0284f6 100644 + RtcVideoSource::Native(ref source) => Ok(source.take_keyframe_request()), + _ => Err(FfiError::InvalidRequest("unsupported video source type".into())), + } ++ } ++ ++ /// Consume the latest pre-encoded sender bitrate allocation, if changed. ++ pub(crate) fn take_bitrate_request(&self) -> FfiResult> { ++ if self.source_type != proto::VideoSourceType::VideoSourceNativeEncoded { ++ return Err(FfiError::InvalidRequest("video source is not pre-encoded".into())); ++ } ++ match self.source { ++ #[cfg(not(target_arch = "wasm32"))] ++ RtcVideoSource::Native(ref source) => { ++ Ok(source.take_rate_control_request().map(|value| value.target_bitrate_bps)) ++ } ++ _ => Err(FfiError::InvalidRequest("unsupported video source type".into())), ++ } + } } @@ -1383,7 +1430,7 @@ index 21337794..042ee055 100644 static std::shared_ptr video_to_media( std::shared_ptr track) { diff --git a/webrtc-sys/src/passthrough_video_encoder.cpp b/webrtc-sys/src/passthrough_video_encoder.cpp -index cd45b9e3..15fab01e 100644 +index cd45b9e3..1981a4e9 100644 --- a/webrtc-sys/src/passthrough_video_encoder.cpp +++ b/webrtc-sys/src/passthrough_video_encoder.cpp @@ -26,12 +26,13 @@ @@ -1481,15 +1528,18 @@ index cd45b9e3..15fab01e 100644 RTC_LOG(LS_ERROR) << "PassthroughVideoEncoder callback failed " << result.error; return WEBRTC_VIDEO_CODEC_ERROR; -@@ -331,6 +350,7 @@ class PassthroughVideoEncoder final : public VideoEncoder { +@@ -331,6 +350,10 @@ class PassthroughVideoEncoder final : public VideoEncoder { info.supports_native_handle = true; info.implementation_name = "LiveKit pre-encoded passthrough"; info.scaling_settings = VideoEncoder::ScalingSettings::kOff; -+ info.has_trusted_rate_controller = true; ++ // External encoders may reach their bitrate floor above available ++ // bandwidth. Keep WebRTC backpressure; the continuity guard requests IDR ++ // after a drop. ++ info.has_trusted_rate_controller = false; info.is_hardware_accelerated = false; info.supports_simulcast = false; info.preferred_pixel_formats = {VideoFrameBuffer::Type::kNative}; -@@ -338,8 +358,7 @@ class PassthroughVideoEncoder final : public VideoEncoder { +@@ -338,8 +361,7 @@ class PassthroughVideoEncoder final : public VideoEncoder { } private: @@ -1499,7 +1549,7 @@ index cd45b9e3..15fab01e 100644 std::optional request; { webrtc::MutexLock lock(&rate_control_mutex_); -@@ -357,10 +376,13 @@ class PassthroughVideoEncoder final : public VideoEncoder { +@@ -357,10 +379,13 @@ class PassthroughVideoEncoder final : public VideoEncoder { VideoCodecType codec_type_; VideoCodec codec_; EncodedImageCallback* encoded_image_callback_ = nullptr; @@ -1514,7 +1564,7 @@ index cd45b9e3..15fab01e 100644 }; } // namespace -@@ -388,8 +410,8 @@ PassthroughVideoEncoderFactory::GetSupportedFormats() const { +@@ -388,8 +413,8 @@ PassthroughVideoEncoderFactory::GetSupportedFormats() const { return supported_formats_; } diff --git a/src/tests/unit/test_video_source.cpp b/src/tests/unit/test_video_source.cpp index 8546bd5..af086bc 100644 --- a/src/tests/unit/test_video_source.cpp +++ b/src/tests/unit/test_video_source.cpp @@ -19,6 +19,7 @@ #include #include +#include #include namespace livekit::test { @@ -49,12 +50,21 @@ TEST_F(VideoSourceTest, PreEncodedSourceHasStrictEncodedInterface) { EXPECT_EQ(source.height(), 480); EXPECT_NE(source.ffiHandleId(), 0u); EXPECT_FALSE(source.takeKeyFrameRequest()); + EXPECT_FALSE(source.takeBitrateRequest().has_value()); constexpr std::array access_unit{0, 0, 0, 1, 0x65}; const EncodedVideoFrame frame{access_unit.data(), access_unit.size(), 1'000, true, {}}; // Acceptance transfers bytes to the source; it does not validate the H.264 // bitstream or imply that a subscriber has decoded it. EXPECT_TRUE(source.captureFrame(frame)); + // Without a sender there is no allocation. Polling consumes no frame state. + const auto polling_start = std::chrono::steady_clock::now(); + for (int poll = 0; poll < 1000; ++poll) { + EXPECT_FALSE(source.takeBitrateRequest().has_value()); + } + RecordProperty("bitrate_poll_1000_us", std::to_string(std::chrono::duration_cast( + std::chrono::steady_clock::now() - polling_start) + .count())); EXPECT_THROW(source.captureFrame(EncodedVideoFrame{}), std::invalid_argument); const EncodedVideoFrame empty_frame{access_unit.data(), 0, 1'000, true, {}}; EXPECT_THROW(source.captureFrame(empty_frame), std::invalid_argument); diff --git a/src/video_source.cpp b/src/video_source.cpp index b82a27e..5958ecc 100644 --- a/src/video_source.cpp +++ b/src/video_source.cpp @@ -105,4 +105,17 @@ bool EncodedVideoSource::takeKeyFrameRequest() { return resp.take_video_key_frame_request().requested(); } +std::optional EncodedVideoSource::takeBitrateRequest() { + if (ffiHandleId() == 0) return std::nullopt; + proto::FfiRequest req; + req.mutable_take_video_bitrate_request()->set_source_handle(ffiHandleId()); + const proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + if (!resp.has_take_video_bitrate_request()) { + throw std::runtime_error("FfiResponse missing take_video_bitrate_request"); + } + const auto& allocation = resp.take_video_bitrate_request(); + if (!allocation.has_target_bitrate_bps()) return std::nullopt; + return allocation.target_bitrate_bps(); +} + } // namespace livekit