From 6a2a4322f88e67d926805ffb49177d8189a10703 Mon Sep 17 00:00:00 2001 From: IsaFxck Date: Sun, 11 Oct 2026 10:21:33 +0300 Subject: [PATCH] feat: application video decoder with zero-copy external frames Applications can install a C ABI video decoder (setExternalVideoDecoder). For H.264 the WebRTC decoder factory puts it in front of the internal FFmpeg decoder via the software fallback wrapper. Its frames are native buffers; video streams opened with external_frames receive them as VideoBufferType::External without any CPU conversion or copy, other streams read them back through read_i420. Co-Authored-By: Claude Opus 5.5 --- include/livekit/external_video_decoder.h | 74 +++ include/livekit/livekit.h | 1 + include/livekit/video_frame.h | 12 +- include/livekit/video_stream.h | 4 + .../0001-native-v2-runtime.patch | 606 +++++++++++++++++- patches/client-sdk-rust/README.md | 10 + src/livekit.cpp | 5 + src/video_frame.cpp | 12 + src/video_stream.cpp | 2 + src/video_utils.cpp | 4 + 10 files changed, 725 insertions(+), 5 deletions(-) create mode 100644 include/livekit/external_video_decoder.h diff --git a/include/livekit/external_video_decoder.h b/include/livekit/external_video_decoder.h new file mode 100644 index 0000000..d534f3b --- /dev/null +++ b/include/livekit/external_video_decoder.h @@ -0,0 +1,74 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "livekit/visibility.h" + +// Application-provided video decoder, e.g. a GPU decoder whose frames stay on +// the application's device. WebRTC keeps its own decoder as the fallback when +// create() returns null or decode() asks for it. Layout must match +// webrtc-sys/include/livekit/external_video_decoder.h. +extern "C" { + +#define LK_EXTERNAL_VIDEO_DECODER_ABI 1 + +enum LkExternalVideoCodec : std::int32_t { + LK_EXTERNAL_VIDEO_CODEC_H264 = 0, +}; + +enum LkExternalDecodeResult : std::int32_t { + LK_EXTERNAL_DECODE_FRAME = 0, ///< `out->frame` holds a new reference + LK_EXTERNAL_DECODE_NO_FRAME = 1, ///< accepted, nothing to show yet + LK_EXTERNAL_DECODE_ERROR = 2, ///< undecodable: WebRTC requests a keyframe + LK_EXTERNAL_DECODE_FALLBACK = 3, ///< unusable: switch to the built-in decoder +}; + +struct LkExternalVideoFrame { + void* frame; ///< opaque, released with release_frame + std::int32_t width; + std::int32_t height; +}; + +/// Every function may run on any WebRTC decoder thread; one decoder instance +/// is never called concurrently. The table must outlive the SDK. +struct LkExternalVideoDecoder { + std::uint32_t abi_version; + void* user; + void* (*create)(void* user, std::int32_t codec); + void (*destroy)(void* decoder); + std::int32_t (*decode)(void* decoder, const std::uint8_t* data, std::size_t size, + std::int32_t keyframe, LkExternalVideoFrame* out); + void (*release_frame)(void* frame); + /// CPU copy for streams that did not request external frames; 0 on success. + std::int32_t (*read_i420)(void* frame, std::uint8_t* y, std::int32_t stride_y, + std::uint8_t* u, std::int32_t stride_u, std::uint8_t* v, + std::int32_t stride_v); +}; + +} // extern "C" + +namespace livekit { + +/// Installs (or with null removes) the decoder for video tracks subscribed +/// afterwards. Video streams opened with `external_frames` receive its frames +/// as VideoBufferType::External without a CPU copy. +LIVEKIT_API void setExternalVideoDecoder(const LkExternalVideoDecoder* decoder); + +} // namespace livekit diff --git a/include/livekit/livekit.h b/include/livekit/livekit.h index d0aaee8..267e1c1 100644 --- a/include/livekit/livekit.h +++ b/include/livekit/livekit.h @@ -22,6 +22,7 @@ #include "livekit/audio_stream.h" #include "livekit/build.h" #include "livekit/e2ee.h" +#include "livekit/external_video_decoder.h" #include "livekit/local_audio_track.h" #include "livekit/local_participant.h" #include "livekit/local_track_publication.h" diff --git a/include/livekit/video_frame.h b/include/livekit/video_frame.h index e13c7b8..0e47f3a 100644 --- a/include/livekit/video_frame.h +++ b/include/livekit/video_frame.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -25,8 +26,9 @@ namespace livekit { -/// Mirror of WebRTC video buffer type -enum class VideoBufferType { RGBA = 0, ABGR, ARGB, BGRA, RGB24, I420, I420A, I422, I444, I010, NV12 }; +/// Mirror of WebRTC video buffer type. External frames come from an +/// application decoder (see setExternalVideoDecoder) and carry no pixels. +enum class VideoBufferType { RGBA = 0, ABGR, ARGB, BGRA, RGB24, I420, I420A, I422, I444, I010, NV12, External }; /// @brief Plane layout metadata for a VideoFrame buffer. struct VideoPlaneInfo { @@ -69,6 +71,10 @@ class LIVEKIT_API VideoFrame { const std::uint8_t* data() const noexcept { return data_.data(); } std::size_t dataSize() const noexcept { return data_.size(); } + /// The decoder's frame for VideoBufferType::External, valid while this + /// VideoFrame lives; null otherwise. + void* externalFrame() const noexcept { return external_frame_; } + /// Compute plane layout for this frame (Y/U/V, UV, etc.), in terms of /// pointers & sizes relative to this frame's backing buffer. /// @@ -111,6 +117,8 @@ class LIVEKIT_API VideoFrame { int height_; VideoBufferType type_; std::vector data_; + void* external_frame_ = nullptr; + std::shared_ptr external_owner_; }; } // namespace livekit diff --git a/include/livekit/video_stream.h b/include/livekit/video_stream.h index 1554fee..ea08ab3 100644 --- a/include/livekit/video_stream.h +++ b/include/livekit/video_stream.h @@ -77,6 +77,10 @@ class LIVEKIT_API VideoStream { /// Preferred pixel format for frames delivered by read(). The FFI layer /// converts into this format if supported (e.g., RGBA, BGRA, I420, ...). VideoBufferType format{VideoBufferType::RGBA}; + + /// Deliver frames of the external decoder (setExternalVideoDecoder) as + /// VideoBufferType::External instead of converting them to `format`. + bool external_frames{false}; }; /// Factory: create a VideoStream bound to a specific Track diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index 479ce8c..8ca6006 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -586,6 +586,25 @@ index 3bd4fb59..a6a089bc 100644 pub fn sender(&self) -> rtp_sender::RtpSender { rtp_sender::RtpSender { handle: RtpSender { sys_handle: self.sys_handle.sender() } } } +diff --git a/libwebrtc/src/native/video_frame.rs b/libwebrtc/src/native/video_frame.rs +index 1f9f62f5..59102793 100644 +--- a/libwebrtc/src/native/video_frame.rs ++++ b/libwebrtc/src/native/video_frame.rs +@@ -161,6 +161,14 @@ impl NativeBuffer { + } + } + ++ /// The application frame of an external decoder, if this buffer holds one. ++ pub fn external_frame(&self) -> Option { ++ match vfb_sys::ffi::native_buffer_external_frame(&self.sys_handle) { ++ 0 => None, ++ frame => Some(frame), ++ } ++ } ++ + #[cfg(any(target_os = "macos", target_os = "ios"))] + pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void { + unsafe { vfb_sys::ffi::native_buffer_to_platform_image_buffer(&self.sys_handle) as *mut _ } diff --git a/libwebrtc/src/rtp_sender.rs b/libwebrtc/src/rtp_sender.rs index 1bdf96ad..ca60a36c 100644 --- a/libwebrtc/src/rtp_sender.rs @@ -635,6 +654,36 @@ index 00a7a919..b033b780 100644 pub fn sender(&self) -> RtpSender { self.handle.sender() } +diff --git a/libwebrtc/src/video_frame.rs b/libwebrtc/src/video_frame.rs +index 7a4234e3..880e2825 100644 +--- a/libwebrtc/src/video_frame.rs ++++ b/libwebrtc/src/video_frame.rs +@@ -590,6 +590,12 @@ pub mod native { + pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void { + self.handle.get_cv_pixel_buffer() + } ++ ++ /// The application frame when an external decoder produced this ++ /// buffer (see `lk_set_external_video_decoder`). ++ pub fn external_frame(&self) -> Option { ++ self.handle.external_frame() ++ } + } + + pub trait VideoFrameBufferExt: VideoBuffer { +diff --git a/livekit-ffi/include/livekit_ffi.h b/livekit-ffi/include/livekit_ffi.h +index 15f64dfa..025438bc 100644 +--- a/livekit-ffi/include/livekit_ffi.h ++++ b/livekit-ffi/include/livekit_ffi.h +@@ -37,6 +37,8 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len, + + bool livekit_ffi_drop_handle(FfiHandleId handle_id); + ++void livekit_ffi_set_external_video_decoder(const void *decoder); ++ + } // extern "C" + + #endif // livekit_ffi diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index 53ea782b..1d03c91c 100644 --- a/livekit-ffi/protocol/ffi.proto @@ -690,10 +739,28 @@ index df08ed5b..ab1068a5 100644 // Controls how the encoder degrades quality when bandwidth is constrained. diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto -index a4b610e0..82b60f0c 100644 +index a4b610e0..0e177673 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto -@@ -82,6 +82,27 @@ message CaptureVideoFrameRequest { +@@ -37,6 +37,9 @@ message NewVideoStreamRequest { + // unbounded buffering, for only one of them can increase end-to-end latency + // for that stream and cause audio/video drift. + optional uint32 queue_size_frames = 5; ++ // Deliver frames of the application's external decoder as EXTERNAL buffers ++ // (the decoder's frame in data_ptr) instead of converting them to `format`. ++ optional bool external_frames = 6; + } + message NewVideoStreamResponse { required OwnedVideoStream stream = 1; } + +@@ -56,6 +59,7 @@ message VideoStreamFromParticipantRequest { + // unbounded buffering, for only one of them can increase end-to-end latency + // for that stream and cause audio/video drift. + optional uint32 queue_size_frames = 6; ++ optional bool external_frames = 7; + } + + message VideoStreamFromParticipantResponse { required OwnedVideoStream stream = 1;} +@@ -82,6 +86,27 @@ message CaptureVideoFrameRequest { message CaptureVideoFrameResponse {} @@ -721,7 +788,16 @@ index a4b610e0..82b60f0c 100644 message VideoConvertRequest { optional bool flip_y = 1; required VideoBufferInfo buffer = 2; -@@ -207,6 +228,7 @@ message VideoSourceResolution { +@@ -133,6 +158,8 @@ enum VideoBufferType { + I444 = 8; + I010 = 9; + NV12 = 10; ++ // Frame of the application's external decoder; carries no pixels. ++ EXTERNAL = 11; + } + + message VideoBufferInfo { +@@ -207,6 +234,7 @@ message VideoSourceResolution { enum VideoSourceType { VIDEO_SOURCE_NATIVE = 0; @@ -729,6 +805,30 @@ index a4b610e0..82b60f0c 100644 } message VideoSourceInfo { +diff --git a/livekit-ffi/src/cabi.rs b/livekit-ffi/src/cabi.rs +index d9e83f13..e8ab3b62 100644 +--- a/livekit-ffi/src/cabi.rs ++++ b/livekit-ffi/src/cabi.rs +@@ -108,6 +108,19 @@ pub extern "C" fn livekit_ffi_drop_handle(handle_id: FfiHandleId) -> bool { + FFI_SERVER.drop_handle(handle_id) + } + ++/// Installs the application's video decoder; null removes it. Decoders created ++/// afterwards use it, with WebRTC's own decoder as the fallback. ++/// ++/// # Safety ++/// ++/// `decoder` is null or points to an `LkExternalVideoDecoder` (see ++/// `webrtc-sys/include/livekit/external_video_decoder.h`) that outlives every ++/// video track subscribed after this call. ++#[no_mangle] ++pub unsafe extern "C" fn livekit_ffi_set_external_video_decoder(decoder: *const std::ffi::c_void) { ++ webrtc_sys::external_video_decoder::lk_set_external_video_decoder(decoder); ++} ++ + #[no_mangle] + pub extern "C" fn livekit_ffi_dispose() { + FFI_SERVER.async_runtime.block_on(FFI_SERVER.dispose()); diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs index 4922add5..281dcfc7 100644 --- a/livekit-ffi/src/conversion/room.rs @@ -1006,6 +1106,36 @@ index 00000000..a13fa9f4 + } + 0 +} +diff --git a/livekit-ffi/src/server/colorcvt/cvtimpl.rs b/livekit-ffi/src/server/colorcvt/cvtimpl.rs +index 95a52e26..cb63a157 100644 +--- a/livekit-ffi/src/server/colorcvt/cvtimpl.rs ++++ b/livekit-ffi/src/server/colorcvt/cvtimpl.rs +@@ -34,6 +34,9 @@ pub unsafe fn cvt( + proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y), + proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y), + proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y), ++ proto::VideoBufferType::External => { ++ Err(FfiError::InvalidRequest("external frames carry no pixels".into())) ++ } + } + } + +diff --git a/livekit-ffi/src/server/colorcvt/mod.rs b/livekit-ffi/src/server/colorcvt/mod.rs +index bb1ca6fc..fbe9b165 100644 +--- a/livekit-ffi/src/server/colorcvt/mod.rs ++++ b/livekit-ffi/src/server/colorcvt/mod.rs +@@ -119,6 +119,11 @@ pub unsafe fn to_libwebrtc_buffer(info: proto::VideoBufferInfo) -> BoxVideoBuffe + duv.copy_from_slice(data_uv); + Box::new(nv12) as BoxVideoBuffer + } ++ proto::VideoBufferType::External => { ++ // Decoder output only; it carries no pixels to capture. ++ log::error!("external frames cannot be captured"); ++ Box::new(I420Buffer::new_black(width, height)) as BoxVideoBuffer ++ } + } + } + diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 8ee56988..95f1c87c 100644 --- a/livekit-ffi/src/server/requests.rs @@ -1167,6 +1297,80 @@ index ee1410a7..1b280e2d 100644 } #[cfg(test)] +diff --git a/livekit-ffi/src/server/video_stream.rs b/livekit-ffi/src/server/video_stream.rs +index 2cc69ae7..f1697d29 100644 +--- a/livekit-ffi/src/server/video_stream.rs ++++ b/livekit-ffi/src/server/video_stream.rs +@@ -82,6 +82,7 @@ impl FfiVideoStream { + handle_id, + new_stream.format.and_then(|_| Some(new_stream.format())), + new_stream.normalize_stride.unwrap_or(true), ++ new_stream.external_frames.unwrap_or(false), + NativeVideoStream::with_options(rtc_track, options), + self_dropped_rx, + server.watch_handle_dropped(new_stream.track_handle), +@@ -135,6 +136,7 @@ impl FfiVideoStream { + stream_handle: FfiHandleId, + dst_type: Option, + normalize_stride: bool, ++ external_frames: bool, + mut native_stream: NativeVideoStream, + mut self_dropped_rx: oneshot::Receiver<()>, + mut handle_dropped_rx: oneshot::Receiver<()>, +@@ -156,13 +158,31 @@ impl FfiVideoStream { + let metadata = frame_metadata_to_proto(frame.frame_metadata); + let timestamp_us = frame.timestamp_us; + let rotation = proto::VideoRotation::from(frame.rotation).into(); +- let Ok((buffer, info)) = colorcvt::to_video_buffer_info(frame.buffer, dst_type, normalize_stride) else { +- log::error!("video stream failed to convert video frame to {:?}", dst_type); +- continue; +- }; +- ++ let external = external_frames ++ .then(|| frame.buffer.as_native().and_then(|native| native.external_frame())) ++ .flatten(); + let handle_id = server.next_id(); +- server.store_handle(handle_id, buffer); ++ let info = if let Some(external) = external { ++ // The stored buffer keeps the decoder frame alive until the ++ // client drops the handle; no pixels leave the decoder. ++ let info = proto::VideoBufferInfo { ++ r#type: proto::VideoBufferType::External.into(), ++ width: frame.buffer.width(), ++ height: frame.buffer.height(), ++ data_ptr: external as u64, ++ stride: None, ++ components: Vec::new(), ++ }; ++ server.store_handle(handle_id, frame.buffer); ++ info ++ } else { ++ let Ok((buffer, info)) = colorcvt::to_video_buffer_info(frame.buffer, dst_type, normalize_stride) else { ++ log::error!("video stream failed to convert video frame to {:?}", dst_type); ++ continue; ++ }; ++ server.store_handle(handle_id, buffer); ++ info ++ }; + + + if let Err(err) = server.send_event( +@@ -224,6 +244,7 @@ impl FfiVideoStream { + + let track_source = request.track_source(); + let normalize_stride = request.normalize_stride.unwrap_or(true); ++ let external_frames = request.external_frames.unwrap_or(false); + let queue_size_frames = request.queue_size_frames.map(|capacity| capacity as usize); + let (track_tx, mut track_rx) = mpsc::channel::(1); + let (track_finished_tx, track_finished_rx) = broadcast::channel::(1); +@@ -268,6 +289,7 @@ impl FfiVideoStream { + stream_handle, + dst_type, + normalize_stride, ++ external_frames, + NativeVideoStream::with_options(rtc_track, options), + c_rx, + handle_dropped_rx, diff --git a/livekit/src/room/e2ee/manager.rs b/livekit/src/room/e2ee/manager.rs index 5ee35cd3..ff3b7c8c 100644 --- a/livekit/src/room/e2ee/manager.rs @@ -2115,6 +2319,18 @@ index 09bf854a..b63a55e5 100644 + assert!(weak.upgrade().is_none()); } } +diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs +index adabf9cf..202b52c7 100644 +--- a/webrtc-sys/build.rs ++++ b/webrtc-sys/build.rs +@@ -88,6 +88,7 @@ fn main() { + "src/video_encoder_factory.cpp", + "src/passthrough_video_encoder.cpp", + "src/video_decoder_factory.cpp", ++ "src/external_video_decoder.cpp", + "src/synthetic_audio_device.cpp", + "src/adm_proxy.cpp", + "src/audio_resampler.cpp", diff --git a/webrtc-sys/include/livekit/encoded_reference_continuity.h b/webrtc-sys/include/livekit/encoded_reference_continuity.h new file mode 100644 index 00000000..d71b0f0e @@ -2197,6 +2413,103 @@ index 40e5fece..f0e7c560 100644 webrtc::scoped_refptr payload_; std::shared_ptr> keyframe_request_flag_; std::shared_ptr rate_control_state_; +diff --git a/webrtc-sys/include/livekit/external_video_decoder.h b/webrtc-sys/include/livekit/external_video_decoder.h +new file mode 100644 +index 00000000..adff581c +--- /dev/null ++++ b/webrtc-sys/include/livekit/external_video_decoder.h +@@ -0,0 +1,91 @@ ++/* ++ * Copyright 2026 LiveKit, Inc. ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++#pragma once ++ ++#include ++#include ++ ++#include "api/video/video_frame_buffer.h" ++#include "api/video_codecs/video_decoder.h" ++ ++// C ABI of an application-provided video decoder. The application keeps the ++// table alive for the process lifetime. Every function may be called from any ++// WebRTC decoder thread; one decoder instance is never used concurrently. ++// Layout must match include/livekit/external_video_decoder.h in client-sdk-cpp. ++extern "C" { ++ ++#define LK_EXTERNAL_VIDEO_DECODER_ABI 1 ++ ++enum LkExternalVideoCodec : int32_t { ++ LK_EXTERNAL_VIDEO_CODEC_H264 = 0, ++}; ++ ++enum LkExternalDecodeResult : int32_t { ++ LK_EXTERNAL_DECODE_FRAME = 0, // `frame` holds a new reference ++ LK_EXTERNAL_DECODE_NO_FRAME = 1, // accepted, nothing to show yet ++ LK_EXTERNAL_DECODE_ERROR = 2, // corrupt or undecodable: request a keyframe ++ LK_EXTERNAL_DECODE_FALLBACK = 3, // unusable: switch to the built-in decoder ++}; ++ ++struct LkExternalVideoFrame { ++ void* frame; // opaque, released with release_frame ++ int32_t width; ++ int32_t height; ++}; ++ ++struct LkExternalVideoDecoder { ++ uint32_t abi_version; ++ void* user; ++ // Returns null when the codec or the machine is unsupported; the built-in ++ // decoder is used then. ++ void* (*create)(void* user, int32_t codec); ++ void (*destroy)(void* decoder); ++ int32_t (*decode)(void* decoder, ++ const uint8_t* data, ++ size_t size, ++ int32_t keyframe, ++ LkExternalVideoFrame* out); ++ void (*release_frame)(void* frame); ++ // CPU copy for consumers that did not ask for external frames. Nonzero on ++ // failure. ++ int32_t (*read_i420)(void* frame, ++ uint8_t* y, ++ int32_t stride_y, ++ uint8_t* u, ++ int32_t stride_u, ++ uint8_t* v, ++ int32_t stride_v); ++}; ++ ++// Null clears the decoder. Affects decoders created after the call. ++void lk_set_external_video_decoder(const LkExternalVideoDecoder* decoder); ++ ++} // extern "C" ++ ++namespace livekit_ffi { ++ ++const LkExternalVideoDecoder* ExternalVideoDecoder(); ++ ++// Wraps the application decoder; null when it does not support `codec`. ++std::unique_ptr CreateExternalVideoDecoder( ++ int32_t codec); ++ ++// The application frame carried by a native buffer of an external decoder, or ++// null for any other buffer. ++void* ExternalFrameOf(const webrtc::VideoFrameBuffer& buffer); ++ ++} // namespace livekit_ffi diff --git a/webrtc-sys/include/livekit/peer_connection.h b/webrtc-sys/include/livekit/peer_connection.h index 50c545c7..f9df1357 100644 --- a/webrtc-sys/include/livekit/peer_connection.h @@ -2248,6 +2561,18 @@ index abf4f5ab..d49ab199 100644 }; std::shared_ptr create_peer_connection_factory(); +diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h +index cb419fd3..4f09d876 100644 +--- a/webrtc-sys/include/livekit/video_frame_buffer.h ++++ b/webrtc-sys/include/livekit/video_frame_buffer.h +@@ -222,6 +222,7 @@ std::unique_ptr new_nv12_buffer(int width, int height, int stride_y, + + std::unique_ptr new_native_buffer_from_platform_image_buffer(PlatformImageBuffer *buffer); + PlatformImageBuffer* native_buffer_to_platform_image_buffer(const std::unique_ptr &); ++size_t native_buffer_external_frame(const std::unique_ptr& buffer); + + static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) { + return yuv; diff --git a/webrtc-sys/include/livekit/video_track.h b/webrtc-sys/include/livekit/video_track.h index 21337794..042ee055 100644 --- a/webrtc-sys/include/livekit/video_track.h @@ -2389,6 +2714,206 @@ index 67ed2701..23a3979d 100644 + ); } } +diff --git a/webrtc-sys/src/external_video_decoder.cpp b/webrtc-sys/src/external_video_decoder.cpp +new file mode 100644 +index 00000000..79dcb9c0 +--- /dev/null ++++ b/webrtc-sys/src/external_video_decoder.cpp +@@ -0,0 +1,173 @@ ++/* ++ * Copyright 2026 LiveKit, Inc. ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++#include "livekit/external_video_decoder.h" ++ ++#include ++ ++#include "api/make_ref_counted.h" ++#include "api/video/i420_buffer.h" ++#include "api/video/video_frame.h" ++#include "modules/video_coding/include/video_error_codes.h" ++#include "rtc_base/logging.h" ++ ++namespace { ++std::atomic g_decoder{nullptr}; ++constexpr char kStorage[] = "livekit-external-decoder"; ++} // namespace ++ ++extern "C" void lk_set_external_video_decoder( ++ const LkExternalVideoDecoder* decoder) { ++ if (decoder && decoder->abi_version != LK_EXTERNAL_VIDEO_DECODER_ABI) { ++ RTC_LOG(LS_ERROR) << "External video decoder ABI " << decoder->abi_version ++ << " is not " << LK_EXTERNAL_VIDEO_DECODER_ABI; ++ decoder = nullptr; ++ } ++ g_decoder.store(decoder); ++} ++ ++namespace livekit_ffi { ++namespace { ++ ++// Owns one application frame reference; the frame stays on the application's ++// device until the last consumer drops this buffer. ++class ExternalFrameBuffer : public webrtc::VideoFrameBuffer { ++ public: ++ ExternalFrameBuffer(const LkExternalVideoDecoder* decoder, ++ const LkExternalVideoFrame& frame) ++ : decoder_(decoder), frame_(frame) {} ++ ~ExternalFrameBuffer() override { decoder_->release_frame(frame_.frame); } ++ ++ Type type() const override { return Type::kNative; } ++ int width() const override { return frame_.width; } ++ int height() const override { return frame_.height; } ++ std::string storage_representation() const override { return kStorage; } ++ void* frame() const { return frame_.frame; } ++ ++ webrtc::scoped_refptr ToI420() override { ++ auto i420 = webrtc::I420Buffer::Create(frame_.width, frame_.height); ++ if (decoder_->read_i420(frame_.frame, i420->MutableDataY(), i420->StrideY(), ++ i420->MutableDataU(), i420->StrideU(), ++ i420->MutableDataV(), i420->StrideV()) != 0) { ++ RTC_LOG(LS_WARNING) << "External decoder frame readback failed"; ++ webrtc::I420Buffer::SetBlack(i420.get()); ++ } ++ return i420; ++ } ++ ++ private: ++ const LkExternalVideoDecoder* decoder_; ++ const LkExternalVideoFrame frame_; ++}; ++ ++class ExternalVideoDecoderAdapter : public webrtc::VideoDecoder { ++ public: ++ ExternalVideoDecoderAdapter(const LkExternalVideoDecoder* decoder, ++ int32_t codec) ++ : decoder_(decoder), codec_(codec) {} ++ ~ExternalVideoDecoderAdapter() override { Release(); } ++ ++ bool Configure(const Settings&) override { ++ Release(); ++ instance_ = decoder_->create(decoder_->user, codec_); ++ return instance_ != nullptr; ++ } ++ ++ int32_t Decode(const webrtc::EncodedImage& input, ++ int64_t /*render_time_ms*/) override { ++ if (!instance_ || !callback_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; ++ if (!input.data() || input.size() == 0) return WEBRTC_VIDEO_CODEC_ERROR; ++ LkExternalVideoFrame frame{}; ++ const auto result = decoder_->decode( ++ instance_, input.data(), input.size(), ++ input.FrameType() == webrtc::VideoFrameType::kVideoFrameKey, &frame); ++ switch (result) { ++ case LK_EXTERNAL_DECODE_FRAME: ++ break; ++ case LK_EXTERNAL_DECODE_NO_FRAME: ++ return WEBRTC_VIDEO_CODEC_OK; ++ case LK_EXTERNAL_DECODE_FALLBACK: ++ RTC_LOG(LS_WARNING) << "External video decoder requested fallback"; ++ return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; ++ default: ++ return WEBRTC_VIDEO_CODEC_ERROR; ++ } ++ if (!frame.frame) return WEBRTC_VIDEO_CODEC_ERROR; ++ if (frame.width <= 0 || frame.height <= 0) { ++ decoder_->release_frame(frame.frame); ++ return WEBRTC_VIDEO_CODEC_ERROR; ++ } ++ auto decoded = ++ webrtc::VideoFrame::Builder() ++ .set_video_frame_buffer( ++ webrtc::make_ref_counted(decoder_, frame)) ++ .set_rtp_timestamp(input.RtpTimestamp()) ++ .set_color_space(input.ColorSpace()) ++ .set_rotation(input.rotation()) ++ .build(); ++ callback_->Decoded(decoded, std::nullopt, std::nullopt); ++ return WEBRTC_VIDEO_CODEC_OK; ++ } ++ ++ int32_t RegisterDecodeCompleteCallback( ++ webrtc::DecodedImageCallback* callback) override { ++ callback_ = callback; ++ return WEBRTC_VIDEO_CODEC_OK; ++ } ++ ++ int32_t Release() override { ++ if (instance_) decoder_->destroy(instance_); ++ instance_ = nullptr; ++ return WEBRTC_VIDEO_CODEC_OK; ++ } ++ ++ DecoderInfo GetDecoderInfo() const override { ++ DecoderInfo info; ++ info.implementation_name = "ExternalHardware"; ++ info.is_hardware_accelerated = true; ++ return info; ++ } ++ ++ const char* ImplementationName() const override { return "ExternalHardware"; } ++ ++ private: ++ const LkExternalVideoDecoder* decoder_; ++ const int32_t codec_; ++ void* instance_ = nullptr; ++ webrtc::DecodedImageCallback* callback_ = nullptr; ++}; ++ ++} // namespace ++ ++const LkExternalVideoDecoder* ExternalVideoDecoder() { ++ return g_decoder.load(); ++} ++ ++std::unique_ptr CreateExternalVideoDecoder( ++ int32_t codec) { ++ const auto* decoder = g_decoder.load(); ++ if (!decoder) return nullptr; ++ return std::make_unique(decoder, codec); ++} ++ ++void* ExternalFrameOf(const webrtc::VideoFrameBuffer& buffer) { ++ if (buffer.type() != webrtc::VideoFrameBuffer::Type::kNative || ++ buffer.storage_representation() != kStorage) ++ return nullptr; ++ return static_cast(buffer).frame(); ++} ++ ++} // namespace livekit_ffi +diff --git a/webrtc-sys/src/lib.rs b/webrtc-sys/src/lib.rs +index d33322f1..3ef31583 100644 +--- a/webrtc-sys/src/lib.rs ++++ b/webrtc-sys/src/lib.rs +@@ -43,6 +43,16 @@ pub mod video_track; + pub mod webrtc; + pub mod yuv_helper; + ++/// Application video decoder, see `include/livekit/external_video_decoder.h`. ++pub mod external_video_decoder { ++ extern "C" { ++ /// # Safety ++ /// `decoder` is null or a valid `LkExternalVideoDecoder` that outlives ++ /// every decoder created after this call. ++ pub fn lk_set_external_video_decoder(decoder: *const std::ffi::c_void); ++ } ++} ++ + pub const MEDIA_TYPE_VIDEO: &str = "video"; + pub const MEDIA_TYPE_AUDIO: &str = "audio"; + pub const MEDIA_TYPE_DATA: &str = "data"; diff --git a/webrtc-sys/src/passthrough_video_encoder.cpp b/webrtc-sys/src/passthrough_video_encoder.cpp index cd45b9e3..9d50e638 100644 --- a/webrtc-sys/src/passthrough_video_encoder.cpp @@ -2839,6 +3364,81 @@ index e97bed2d..24ae4db9 100644 } uint32_t RtpSender::ssrc() const { +diff --git a/webrtc-sys/src/video_decoder_factory.cpp b/webrtc-sys/src/video_decoder_factory.cpp +index d22ae70d..676b8e5a 100644 +--- a/webrtc-sys/src/video_decoder_factory.cpp ++++ b/webrtc-sys/src/video_decoder_factory.cpp +@@ -20,6 +20,8 @@ + #include "api/environment/environment.h" + #include "api/video_codecs/av1_profile.h" + #include "api/video_codecs/sdp_video_format.h" ++#include "api/video_codecs/video_decoder_software_fallback_wrapper.h" ++#include "livekit/external_video_decoder.h" + #include "livekit/objc_video_factory.h" + #include "media/base/media_constants.h" + #include "modules/video_coding/codecs/h264/include/h264.h" +@@ -105,7 +107,7 @@ std::vector VideoDecoderFactory::GetSupportedFormats() + for (const webrtc::SdpVideoFormat& format : + webrtc::SupportedVP9DecoderCodecs()) + formats.push_back(format); +- if (internal_h264_decoder_works_) { ++ if (internal_h264_decoder_works_ || ExternalVideoDecoder()) { + for (const webrtc::SdpVideoFormat& h264_format : + webrtc::SupportedH264DecoderCodecs()) { + formats.push_back(h264_format); +@@ -136,6 +138,16 @@ VideoDecoderFactory::CodecSupport VideoDecoderFactory::QueryCodecSupport( + + std::unique_ptr VideoDecoderFactory::Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) { ++ // The application decoder comes first; WebRTC's software decoder takes over ++ // when it declines the stream or reports itself unusable. ++ if (absl::EqualsIgnoreCase(format.name, webrtc::kH264CodecName) && ++ ExternalVideoDecoder()) { ++ auto external = CreateExternalVideoDecoder(LK_EXTERNAL_VIDEO_CODEC_H264); ++ if (!internal_h264_decoder_works_) return external; ++ return webrtc::CreateVideoDecoderSoftwareFallbackWrapper( ++ env, webrtc::H264Decoder::Create(), std::move(external)); ++ } ++ + for (const auto& factory : factories_) { + for (const auto& supported_format : factory->GetSupportedFormats()) { + if (supported_format.IsSameCodec(format)) +diff --git a/webrtc-sys/src/video_frame_buffer.cpp b/webrtc-sys/src/video_frame_buffer.cpp +index 13b91cec..b64ffc0a 100644 +--- a/webrtc-sys/src/video_frame_buffer.cpp ++++ b/webrtc-sys/src/video_frame_buffer.cpp +@@ -17,6 +17,7 @@ + #include "livekit/video_frame_buffer.h" + + #include "api/make_ref_counted.h" ++#include "livekit/external_video_decoder.h" + + namespace livekit_ffi { + +@@ -374,4 +375,9 @@ PlatformImageBuffer* native_buffer_to_platform_image_buffer( + + #endif + ++size_t native_buffer_external_frame( ++ const std::unique_ptr& buffer) { ++ return reinterpret_cast(ExternalFrameOf(*buffer->get())); ++} ++ + } // namespace livekit_ffi +diff --git a/webrtc-sys/src/video_frame_buffer.rs b/webrtc-sys/src/video_frame_buffer.rs +index 498f8c05..95933411 100644 +--- a/webrtc-sys/src/video_frame_buffer.rs ++++ b/webrtc-sys/src/video_frame_buffer.rs +@@ -160,6 +160,9 @@ pub mod ffi { + buffer: &UniquePtr, + ) -> *mut PlatformImageBuffer; + ++ /// The application frame of an external decoder buffer, or 0. ++ fn native_buffer_external_frame(buffer: &UniquePtr) -> usize; ++ + unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer; + unsafe fn biyuv_to_vfb(yuv: *const BiplanarYuvBuffer) -> *const VideoFrameBuffer; + unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer; diff --git a/webrtc-sys/src/video_track.cpp b/webrtc-sys/src/video_track.cpp index ba89c298..333a890f 100644 --- a/webrtc-sys/src/video_track.cpp diff --git a/patches/client-sdk-rust/README.md b/patches/client-sdk-rust/README.md index 7d02d3b..ec21871 100644 --- a/patches/client-sdk-rust/README.md +++ b/patches/client-sdk-rust/README.md @@ -66,3 +66,13 @@ See [the deterministic socket-retirement reproduction](../../docs/ice-socket-ret The unsuccessful tag v1.10.0-syrnike.3 referenced the unpublished Rust commit and produced no release assets. v1.10.0-syrnike.4 is the reproducible successor. + +The v1.10.0-syrnike.16 candidate adds an application video decoder +(`webrtc-sys/include/livekit/external_video_decoder.h`, exported as +`livekit_ffi_set_external_video_decoder`). For H.264 the decoder factory puts +it in front of the internal FFmpeg decoder through WebRTC's software fallback +wrapper; declining a stream or returning `LK_EXTERNAL_DECODE_FALLBACK` keeps +decoding in software. Its frames are native buffers carrying the application's +frame; video streams created with `external_frames` deliver them as +`VideoBufferType::EXTERNAL` without conversion, other streams read them back +through `read_i420`. diff --git a/src/livekit.cpp b/src/livekit.cpp index 6b76759..fb9870f 100644 --- a/src/livekit.cpp +++ b/src/livekit.cpp @@ -17,6 +17,7 @@ #include "livekit/livekit.h" #include "ffi_client.h" +#include "livekit_ffi.h" #include "lk_log.h" namespace livekit { @@ -29,6 +30,10 @@ bool initialize(const LogLevel& level) { return ffi_client.initialize(false); } +void setExternalVideoDecoder(const LkExternalVideoDecoder* decoder) { + livekit_ffi_set_external_video_decoder(decoder); +} + bool isInitialized() { return FfiClient::instance().isInitialized(); } void shutdown() { diff --git a/src/video_frame.cpp b/src/video_frame.cpp index 6f90c6b..63ab3e6 100644 --- a/src/video_frame.cpp +++ b/src/video_frame.cpp @@ -319,6 +319,18 @@ VideoFrame VideoFrame::fromOwnedInfo(const proto::OwnedVideoBuffer& owned) { const int height = static_cast(info.height()); const VideoBufferType type = fromProto(info.type()); + if (type == VideoBufferType::External) { + // No pixels to copy: the FFI handle keeps the decoder frame alive. + VideoFrame frame; + frame.width_ = width; + frame.height_ = height; + frame.type_ = type; + // NOLINTNEXTLINE(performance-no-int-to-ptr) + frame.external_frame_ = reinterpret_cast(info.data_ptr()); + frame.external_owner_ = std::make_shared(static_cast(owned.handle().id())); + return frame; + } + std::vector buffer; if (info.components_size() > 0) { diff --git a/src/video_stream.cpp b/src/video_stream.cpp index effd3f6..edfd882 100644 --- a/src/video_stream.cpp +++ b/src/video_stream.cpp @@ -139,6 +139,7 @@ void VideoStream::initFromTrack(const std::shared_ptr& track, const Optio new_video_stream->set_type(proto::VideoStreamType::VIDEO_STREAM_NATIVE); new_video_stream->set_normalize_stride(true); new_video_stream->set_format(toProto(options.format)); + new_video_stream->set_external_frames(options.external_frames); auto resp = FfiClient::instance().sendRequest(req); if (!resp.has_new_video_stream()) { @@ -166,6 +167,7 @@ void VideoStream::initFromParticipant(Participant& participant, TrackSource trac vs->set_track_source(static_cast(track_source)); vs->set_normalize_stride(true); vs->set_format(toProto(options.format)); + vs->set_external_frames(options.external_frames); auto resp = FfiClient::instance().sendRequest(req); // Adjust field names to match your proto exactly: diff --git a/src/video_utils.cpp b/src/video_utils.cpp index faa3fba..4b80289 100644 --- a/src/video_utils.cpp +++ b/src/video_utils.cpp @@ -52,6 +52,8 @@ proto::VideoBufferType toProto(const VideoBufferType t) { return proto::VideoBufferType::I010; case VideoBufferType::NV12: return proto::VideoBufferType::NV12; + case VideoBufferType::External: + return proto::VideoBufferType::EXTERNAL; default: throw std::runtime_error("Unknown VideoBufferType in toProto"); } @@ -82,6 +84,8 @@ VideoBufferType fromProto(const proto::VideoBufferType t) { return VideoBufferType::I010; case proto::VideoBufferType::NV12: return VideoBufferType::NV12; + case proto::VideoBufferType::EXTERNAL: + return VideoBufferType::External; default: throw std::runtime_error("Unknown proto::VideoBufferType in fromProto"); }