diff --git a/Cargo.lock b/Cargo.lock index bd0a2d86..f364aadf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7950,6 +7950,7 @@ dependencies = [ "flate2", "fslock", "futures-util", + "getrandom 0.4.3", "hex 0.4.3", "ico", "if-addrs", diff --git a/enginefs/src/hls.rs b/enginefs/src/hls.rs index 9efc9759..fd7743dd 100644 --- a/enginefs/src/hls.rs +++ b/enginefs/src/hls.rs @@ -235,7 +235,7 @@ impl HlsEngine { "Spawning ffmpeg probe command with analyzeduration={} probesize={} path={}", analyzeduration, probesize, - file_path + redact_source_capabilities(file_path) ); let mut child = cmd.spawn().context("Failed to spawn ffmpeg")?; let stderr = child.stderr.take().context("Failed to capture stderr")?; @@ -694,14 +694,14 @@ impl HlsEngine { // Output format: MPEG-TS for HLS with copyts to preserve timestamps cmd.args(["-mpegts_copyts", "1", "-f", "mpegts", "pipe:1"]); - tracing::debug!("FFmpeg video command (HLS V2): {:?}", cmd); + tracing::debug!("Starting FFmpeg video command for HLS V2"); #[allow(clippy::zombie_processes)] let mut child = cmd .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .with_context(|| format!("Failed to spawn ffmpeg for video segment: {:?}", cmd))?; + .context("Failed to spawn ffmpeg for video segment")?; // Spawn a task to log stderr in background (for debugging) if let Some(stderr) = child.stderr.take() { @@ -711,7 +711,10 @@ impl HlsEngine { let mut line = String::new(); while reader.read_line(&mut line).await.unwrap_or(0) > 0 { if !line.trim().is_empty() { - tracing::warn!("FFmpeg video stderr: {}", line.trim()); + tracing::warn!( + "FFmpeg video stderr: {}", + redact_source_capabilities(line.trim()) + ); } line.clear(); } @@ -795,14 +798,14 @@ impl HlsEngine { // Output format: MPEG-TS for HLS cmd.args(["-mpegts_copyts", "1", "-f", "mpegts", "pipe:1"]); - tracing::debug!("FFmpeg audio command (HLS V2): {:?}", cmd); + tracing::debug!("Starting FFmpeg audio command for HLS V2"); #[allow(clippy::zombie_processes)] let mut child = cmd .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .with_context(|| format!("Failed to spawn ffmpeg for audio segment: {:?}", cmd))?; + .context("Failed to spawn ffmpeg for audio segment")?; if let Some(stderr) = child.stderr.take() { tokio::spawn(async move { @@ -811,7 +814,10 @@ impl HlsEngine { let mut line = String::new(); while reader.read_line(&mut line).await.unwrap_or(0) > 0 { if !line.trim().is_empty() { - tracing::warn!("FFmpeg audio stderr: {}", line.trim()); + tracing::warn!( + "FFmpeg audio stderr: {}", + redact_source_capabilities(line.trim()) + ); } line.clear(); } @@ -937,6 +943,36 @@ fn text_mentions_high_bit_depth(value: &str) -> bool { || value.contains("yuv444p12") } +fn redact_source_capabilities(input: &str) -> std::borrow::Cow<'_, str> { + const MARKER: &str = "cap="; + const CREDENTIAL_LENGTH: usize = 64; + + let mut remainder = input; + let mut output = String::new(); + let mut redacted = false; + while let Some(index) = remainder.find(MARKER) { + let credential_start = index + MARKER.len(); + let credential = remainder + .as_bytes() + .get(credential_start..credential_start + CREDENTIAL_LENGTH); + if credential.is_some_and(|bytes| bytes.iter().all(u8::is_ascii_hexdigit)) { + output.push_str(&remainder[..credential_start]); + output.push_str(""); + remainder = &remainder[credential_start + CREDENTIAL_LENGTH..]; + redacted = true; + } else { + output.push_str(&remainder[..credential_start]); + remainder = &remainder[credential_start..]; + } + } + if redacted { + output.push_str(remainder); + std::borrow::Cow::Owned(output) + } else { + std::borrow::Cow::Borrowed(input) + } +} + #[cfg(test)] mod tests { use super::*; @@ -994,4 +1030,20 @@ mod tests { } } } + + #[test] + fn ffmpeg_diagnostics_redact_transcoding_source_capabilities() { + let credential = "a".repeat(64); + let diagnostic = format!( + "unable to open http://127.0.0.1:11470/_transcoding/source?cap={credential}: denied" + ); + let redacted = redact_source_capabilities(&diagnostic); + + assert!(redacted.contains("cap=")); + assert!(!redacted.contains(&credential)); + assert_eq!( + redact_source_capabilities("ordinary diagnostic"), + "ordinary diagnostic" + ); + } } diff --git a/enginefs/tests/hls_playlist_allocations.rs b/enginefs/tests/hls_playlist_allocations.rs index 6645cc73..2813fd97 100644 --- a/enginefs/tests/hls_playlist_allocations.rs +++ b/enginefs/tests/hls_playlist_allocations.rs @@ -1,16 +1,29 @@ use enginefs::hls::{HlsEngine, ProbeResult}; use std::alloc::{GlobalAlloc, Layout, System}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::cell::Cell; +use std::sync::{ + Arc, Barrier, + atomic::{AtomicUsize, Ordering}, +}; struct CountingAllocator; -static COUNTING: AtomicBool = AtomicBool::new(false); +thread_local! { + static COUNTING: Cell = const { Cell::new(false) }; +} + static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); static REALLOCATIONS: AtomicUsize = AtomicUsize::new(0); +fn is_counting() -> bool { + COUNTING + .try_with(|counting| counting.get()) + .unwrap_or(false) +} + unsafe impl GlobalAlloc for CountingAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - if COUNTING.load(Ordering::Relaxed) { + if is_counting() { ALLOCATIONS.fetch_add(1, Ordering::Relaxed); } unsafe { System.alloc(layout) } @@ -21,7 +34,7 @@ unsafe impl GlobalAlloc for CountingAllocator { } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if COUNTING.load(Ordering::Relaxed) { + if is_counting() { REALLOCATIONS.fetch_add(1, Ordering::Relaxed); } unsafe { System.realloc(ptr, layout, new_size) } @@ -32,14 +45,14 @@ unsafe impl GlobalAlloc for CountingAllocator { static GLOBAL: CountingAllocator = CountingAllocator; fn start_counting() { - COUNTING.store(false, Ordering::SeqCst); + COUNTING.with(|counting| counting.set(false)); ALLOCATIONS.store(0, Ordering::SeqCst); REALLOCATIONS.store(0, Ordering::SeqCst); - COUNTING.store(true, Ordering::SeqCst); + COUNTING.with(|counting| counting.set(true)); } fn stop_counting() -> (usize, usize) { - COUNTING.store(false, Ordering::SeqCst); + COUNTING.with(|counting| counting.set(false)); ( ALLOCATIONS.load(Ordering::SeqCst), REALLOCATIONS.load(Ordering::SeqCst), @@ -48,6 +61,30 @@ fn stop_counting() -> (usize, usize) { #[test] fn two_hour_playlist_avoids_per_segment_temporary_allocations() { + let allocation_started = Arc::new(Barrier::new(2)); + let allocation_finished = Arc::new(Barrier::new(2)); + let worker = { + let allocation_started = Arc::clone(&allocation_started); + let allocation_finished = Arc::clone(&allocation_finished); + std::thread::spawn(move || { + allocation_started.wait(); + let unrelated = Box::new([0_u8; 1_024]); + std::hint::black_box(&unrelated); + allocation_finished.wait(); + }) + }; + + start_counting(); + allocation_started.wait(); + allocation_finished.wait(); + let unrelated_counts = stop_counting(); + worker.join().expect("allocation worker must finish"); + assert_eq!( + unrelated_counts, + (0, 0), + "allocator measurements must ignore unrelated test-harness threads" + ); + let probe = ProbeResult { duration: 7_200.0, container: "test".to_string(), diff --git a/server/Cargo.toml b/server/Cargo.toml index 2ec82234..a6d1448c 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -31,6 +31,7 @@ serde_json = "1.0.151" anyhow = "1.0.104" tokio-util = { version = "0.7.19", features = ["io", "compat", "rt"] } hex = "0.4.3" +getrandom = "0.4.3" librqbit = { version = "9.0.0", optional = true } if-addrs = "0.15.0" ipnet = "2.12.1" diff --git a/server/src/diagnostics/logging.rs b/server/src/diagnostics/logging.rs index 7ee7ca78..3ae7b410 100644 --- a/server/src/diagnostics/logging.rs +++ b/server/src/diagnostics/logging.rs @@ -79,6 +79,13 @@ pub(crate) struct SanitizedRequestTarget { } pub(crate) fn sanitize_request_target(uri: &axum::http::Uri) -> SanitizedRequestTarget { + if uri.path() == "/_transcoding/source" { + return SanitizedRequestTarget { + uri: "/_transcoding/source?".to_owned(), + path: "/_transcoding/source".to_owned(), + query: String::new(), + }; + } if uri.path().starts_with("/proxy") { return SanitizedRequestTarget { uri: "/proxy/".to_owned(), @@ -446,6 +453,18 @@ mod tests { assert_eq!(target.query, ""); } + #[test] + fn transcoding_source_capabilities_are_redacted_for_every_logging_path() { + let uri: Uri = "/_transcoding/source?cap=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .unwrap(); + let target = sanitize_request_target(&uri); + assert_eq!(target.uri, "/_transcoding/source?"); + assert_eq!(target.path, "/_transcoding/source"); + assert_eq!(target.query, ""); + assert!(!target.uri.contains("aaaaaaaa")); + } + #[test] fn ordinary_request_targets_keep_diagnostic_context() { let uri: Uri = "/heartbeat?probe=1".parse().unwrap(); diff --git a/server/src/lib.rs b/server/src/lib.rs index 9a6987f5..e21be48e 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -575,13 +575,19 @@ async fn run_inner( Arc::new(network_security::SystemClock), listeners, )); - state.proxy_runtime = Arc::new(network_security::ProxyRuntime::new( + let proxy_runtime = Arc::new(network_security::ProxyRuntime::new( network_security::ProxyPolicySettings { allow_private_network_sources: settings.allow_private_network_sources, allow_invalid_proxy_tls_certificates: settings.allow_invalid_proxy_tls_certificates, }, validator, )); + state.proxy_runtime = proxy_runtime.clone(); + state.source_broker = Arc::new(transcoding::source::SourceBroker::new( + state.stream_engine(), + bound_http_addr, + proxy_runtime, + )); #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))] { @@ -971,6 +977,10 @@ pub fn build_router(state: AppState) -> Router { "/stream/{infoHash}/{fileIdx}", get(routes::stream::stream_video).head(routes::stream::head_stream_video), ) + .route( + "/_transcoding/source", + get(transcoding::source::route_source).head(transcoding::source::route_source), + ) .route( "/{infoHash}/{fileIdx}", get(routes::stream::stream_video).head(routes::stream::head_stream_video), @@ -1084,6 +1094,7 @@ pub fn build_router_with_listeners(mut state: AppState, listeners: Vec { + inner: R, + _source: crate::transcoding::ValidatedMediaSource, +} + +impl AsyncRead for SourceHeldReader { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(context, buffer) + } +} + #[derive(Deserialize)] pub struct ProbeQuery { #[serde(rename = "mediaURL")] @@ -64,6 +86,150 @@ fn stremio_probe_json( }) } +fn rational_as_f64(rate: crate::transcoding::RationalRate) -> f64 { + f64::from(rate.numerator()) / f64::from(rate.denominator().get()) +} + +fn stremio_probe_json_from_document( + info_hash: &str, + file_idx: usize, + probe: &crate::transcoding::ProbeDocument, +) -> serde_json::Value { + use crate::transcoding::MediaStreamDescriptor; + + let streams = probe + .streams() + .iter() + .map(|stream| match stream { + MediaStreamDescriptor::Video(video) => serde_json::json!({ + "index": video.index(), + "track": "video", + "codec": video.codec_display(), + "channels": 0, + "width": video.width(), + "height": video.height(), + "fps": video.nominal_frame_rate().or(video.average_frame_rate()).map(rational_as_f64), + "bitrate": video.bitrate(), + "lang": video.language(), + "default": video.disposition().default, + "profile": video.profile_display(), + }), + MediaStreamDescriptor::Audio(audio) => serde_json::json!({ + "index": audio.index(), + "track": "audio", + "codec": audio.codec_display(), + "channels": audio.channels().unwrap_or(2), + "width": null, + "height": null, + "fps": null, + "bitrate": audio.bitrate(), + "lang": audio.language(), + "default": audio.disposition().default, + "profile": audio.profile_display(), + }), + MediaStreamDescriptor::Subtitle(subtitle) => serde_json::json!({ + "index": subtitle.index(), + "track": "subtitle", + "codec": subtitle.codec_display(), + "channels": 0, + "width": null, + "height": null, + "fps": null, + "bitrate": null, + "lang": subtitle.language(), + "default": subtitle.disposition().default, + "profile": null, + }), + MediaStreamDescriptor::Other { + index, + track_display, + codec_display, + .. + } => serde_json::json!({ + "index": index, + "track": track_display.as_str(), + "codec": codec_display.as_str(), + "channels": 0, + "width": null, + "height": null, + "fps": null, + "bitrate": null, + "lang": null, + "default": false, + "profile": null, + }), + }) + .collect::>(); + + serde_json::json!({ + "infoHash": info_hash, + "fileIdx": file_idx, + "format": { "name": stremio_format_name(probe.container_display()) }, + "duration": probe.duration_micros().map(|micros| micros as f64 / 1_000_000.0).unwrap_or(0.0), + "streams": streams, + }) +} + +fn schedule_typed_probe_observation( + state: AppState, + info_hash: String, + file_idx: usize, + legacy: enginefs::hls::ProbeResult, +) { + static ADMISSION: OnceLock> = OnceLock::new(); + let admission = ADMISSION.get_or_init(|| Arc::new(tokio::sync::Semaphore::new(1))); + let Ok(permit) = admission.clone().try_acquire_owned() else { + return; + }; + tokio::spawn(async move { + let _permit = permit; + observe_typed_probe_once(&state, &info_hash, file_idx, &legacy).await; + }); +} + +async fn observe_typed_probe_once( + state: &AppState, + info_hash: &str, + file_idx: usize, + legacy: &enginefs::hls::ProbeResult, +) { + let source = match state + .source_broker + .issue_engine_source( + info_hash, + file_idx, + PlaybackIntent::InternalProbe, + Duration::from_secs(30 * 60), + ) + .await + { + Ok(source) => source, + Err(error) => { + tracing::debug!(error = %error, "typed HLS probe observation could not issue source"); + return; + } + }; + match crate::transcoding::probe_media(&state.transcoding, &source).await { + Ok(descriptor) => { + let legacy_json = stremio_probe_json(info_hash, file_idx, legacy); + let typed_json = + stremio_probe_json_from_document(info_hash, file_idx, descriptor.probe()); + if typed_json == legacy_json { + tracing::debug!("typed HLS probe observation matched legacy compatibility DTO"); + } else { + tracing::debug!( + legacy_streams = legacy.streams.len(), + typed_streams = descriptor.probe().streams().len(), + "typed HLS probe observation differed from legacy compatibility DTO" + ); + } + } + Err(error) => { + tracing::debug!(error = %error, "typed HLS probe observation was unavailable"); + } + } +} + /// Probe endpoint using mediaURL query parameter (for Stremio compatibility) /// GET /hlsv2/probe?mediaURL=http://127.0.0.1:11470/{infoHash}/{fileIdx}? pub async fn probe_by_url( @@ -110,6 +276,8 @@ pub async fn probe_by_url( } }; + schedule_typed_probe_observation(state.clone(), info_hash.clone(), file_idx, probe.clone()); + let elapsed = start.elapsed(); tracing::info!( info_hash = %info_hash, @@ -492,14 +660,71 @@ async fn get_segment( .unwrap_or(false) }; - let transcode_input_path = if is_fully_downloaded { - engine - .handle - .get_file_path(file_idx) + let validated_source = if is_fully_downloaded { + match state + .source_broker + .issue_completed_file(&info_hash, file_idx) .await - .unwrap_or_else(|| stream_url.clone()) + { + Ok(source) => source, + Err(crate::transcoding::SourceError::NotFound) => match state + .source_broker + .issue_engine_source( + &info_hash, + file_idx, + if seg_index == 0 { + enginefs::backend::priorities::PlaybackIntent::HlsInitial + } else { + enginefs::backend::priorities::PlaybackIntent::HlsSeek + }, + Duration::from_secs(30 * 60), + ) + .await + { + Ok(source) => source, + Err(error) => { + return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + } + }, + Err(error) => { + return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + } + } } else { - stream_url.clone() + match state + .source_broker + .issue_engine_source( + &info_hash, + file_idx, + if seg_index == 0 { + enginefs::backend::priorities::PlaybackIntent::HlsInitial + } else { + enginefs::backend::priorities::PlaybackIntent::HlsSeek + }, + Duration::from_secs(30 * 60), + ) + .await + { + Ok(source) => source, + Err(error) => { + return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + } + } + }; + let transcode_input_path = match validated_source.input_argument() { + Ok(input) => match input.into_string() { + Ok(input) => input, + Err(_) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Media source path is not valid Unicode", + ) + .into_response(); + } + }, + Err(error) => { + return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + } }; let probe = match engine.get_probe_result(file_idx, &stream_url).await { @@ -530,11 +755,11 @@ async fn get_segment( let mut child = if let Some(audio_idx) = audio_track_idx { // Audio-only segment tracing::debug!( - "Transcoding audio segment: track={}, segment={}, start={:.2}s, input={}", + "Transcoding audio segment: track={}, segment={}, start={:.2}s, source_id={}", audio_idx, seg_index, start, - transcode_input_path + validated_source.id() ); match enginefs::hls::HlsEngine::transcode_audio_segment( &transcode_input_path, @@ -635,7 +860,10 @@ async fn get_segment( None => return (StatusCode::INTERNAL_SERVER_ERROR, "No stdout").into_response(), }; - let stream = ReaderStream::new(stdout); + let stream = ReaderStream::new(SourceHeldReader { + inner: stdout, + _source: validated_source, + }); let body = Body::from_stream(stream); tracing::info!( @@ -678,6 +906,7 @@ pub async fn get_probe( Ok(p) => p, Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), }; + schedule_typed_probe_observation(state.clone(), info_hash.clone(), file_idx, probe.clone()); Json(probe).into_response() } @@ -1037,3 +1266,55 @@ fn hls_v2_segment_alias(resource: &str) -> Option { None } } + +#[cfg(test)] +mod typed_probe_tests { + use super::*; + + #[test] + fn typed_probe_projection_matches_the_legacy_compatibility_dto_fixture() { + let typed = crate::transcoding::parse_probe_document(include_bytes!( + "../../tests/fixtures/ffprobe/compatibility.json" + )) + .expect("parse typed compatibility fixture"); + let legacy = enginefs::hls::ProbeResult { + duration: 12.5, + container: "matroska".to_owned(), + streams: vec![ + enginefs::hls::VideoStream { + index: 0, + codec_type: "video".to_owned(), + codec_name: "h264".to_owned(), + width: Some(1280), + height: Some(720), + channels: None, + bitrate: Some(3_000_000), + fps: Some(30.0), + lang: Some("eng".to_owned()), + is_default: true, + profile: Some("High".to_owned()), + pix_fmt: None, + }, + enginefs::hls::VideoStream { + index: 1, + codec_type: "audio".to_owned(), + codec_name: "aac".to_owned(), + width: None, + height: None, + channels: Some(2), + bitrate: Some(128_000), + fps: None, + lang: Some("eng".to_owned()), + is_default: true, + profile: Some("LC".to_owned()), + pix_fmt: None, + }, + ], + }; + + assert_eq!( + stremio_probe_json_from_document("fixture-hash", 7, &typed), + stremio_probe_json("fixture-hash", 7, &legacy) + ); + } +} diff --git a/server/src/routes/proxy.rs b/server/src/routes/proxy.rs index 08eaad96..15eb817f 100644 --- a/server/src/routes/proxy.rs +++ b/server/src/routes/proxy.rs @@ -45,7 +45,7 @@ const PLAYLIST_LIFETIME_DEADLINE: Duration = Duration::from_secs(120); const PROXY_BODY_CHUNK_SIZE: usize = 64 * 1024; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ProxyError { +pub(crate) enum ProxyError { InvalidRequest, Blocked, Capacity, @@ -53,6 +53,39 @@ enum ProxyError { Cancelled, } +pub(crate) struct FetchedMediaSource { + pub(crate) response: reqwest::Response, + pub(crate) _lease: ProxyProducerLease, +} + +pub(crate) async fn fetch_media_source( + runtime: &ProxyRuntime, + target: Url, + method: Method, + range: Option<&HeaderValue>, +) -> Result { + if !matches!(method, Method::GET | Method::HEAD) { + return Err(ProxyError::InvalidRequest); + } + let context = runtime + .try_request_for_peer(None) + .map_err(|_| ProxyError::Capacity)?; + let request = ParsedProxyRequest { + target, + request_headers: HeaderMap::new(), + response_headers: HeaderMap::new(), + }; + let mut incoming = HeaderMap::new(); + if let Some(range) = range { + incoming.insert(header::RANGE, range.clone()); + } + let fetched = fetch_with_redirects(runtime, &context, &request, method, &incoming).await?; + Ok(FetchedMediaSource { + response: fetched.response, + _lease: context.into_producer_lease(), + }) +} + impl From for ProxyError { fn from(value: DestinationError) -> Self { match value { diff --git a/server/src/state.rs b/server/src/state.rs index 8ce11d96..cf9f4036 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -404,6 +404,7 @@ pub struct AppState { pub(crate) proxy_runtime: Arc, pub(crate) settings_persistence: Arc, pub transcoding: Arc, + pub(crate) source_broker: Arc, } impl AppState { @@ -517,6 +518,13 @@ impl AppState { socket: default_http_addr, }], )); + let proxy_runtime = Arc::new(ProxyRuntime::new(proxy_policy, validator)); + + let source_broker = Arc::new(crate::transcoding::source::SourceBroker::new( + engine.clone(), + default_http_addr, + proxy_runtime.clone(), + )); Self { engine, @@ -535,9 +543,10 @@ impl AppState { nzb_sessions: Arc::new(dashmap::DashMap::new()), devices: Arc::new(RwLock::new(Vec::new())), settings_control: SettingsControl::ephemeral(), - proxy_runtime: Arc::new(ProxyRuntime::new(proxy_policy, validator)), + proxy_runtime, settings_persistence: Arc::new(SettingsPersistenceCoordinator::new(initial_settings)), transcoding, + source_broker, } } diff --git a/server/src/transcoding/codec.rs b/server/src/transcoding/codec.rs new file mode 100644 index 00000000..e2d0a464 --- /dev/null +++ b/server/src/transcoding/codec.rs @@ -0,0 +1,310 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum InputVideoCodec { + H264, + Hevc, + Av1, + Vp9, + Mpeg2, + Vc1, + OtherProbed, +} + +impl InputVideoCodec { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + match value.unwrap_or_default().to_ascii_lowercase().as_str() { + "h264" | "avc" => Self::H264, + "hevc" | "h265" => Self::Hevc, + "av1" => Self::Av1, + "vp9" => Self::Vp9, + "mpeg2video" | "mpeg2" => Self::Mpeg2, + "vc1" => Self::Vc1, + _ => Self::OtherProbed, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum OutputVideoCodec { + H264, + Hevc, + Av1, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ContainerKind { + MatroskaWebm, + MovMp4, + MpegTs, + Avi, + Ogg, + Flv, + Mpeg, + OtherProbed, + Unknown, +} + +impl ContainerKind { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + let value = value.unwrap_or_default().to_ascii_lowercase(); + if value.is_empty() || value == "unknown" || value == "n/a" { + Self::Unknown + } else if value + .split(',') + .any(|part| matches!(part, "matroska" | "webm")) + { + Self::MatroskaWebm + } else if value + .split(',') + .any(|part| matches!(part, "mov" | "mp4" | "m4a" | "3gp" | "3g2" | "mj2")) + { + Self::MovMp4 + } else if value + .split(',') + .any(|part| matches!(part, "mpegts" | "mpegtsraw")) + { + Self::MpegTs + } else if value == "avi" { + Self::Avi + } else if value == "ogg" { + Self::Ogg + } else if value == "flv" { + Self::Flv + } else if value + .split(',') + .any(|part| matches!(part, "mpeg" | "mpegvideo")) + { + Self::Mpeg + } else { + Self::OtherProbed + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SampleEntry { + Avc1, + Avc3, + Hvc1, + Hev1, + Av01, + Vp09, + Mp4a, + OtherProbed, + Unknown, +} + +impl SampleEntry { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + match value.unwrap_or_default().to_ascii_lowercase().as_str() { + "" | "unknown" | "[0][0][0][0]" => Self::Unknown, + "avc1" => Self::Avc1, + "avc3" => Self::Avc3, + "hvc1" => Self::Hvc1, + "hev1" => Self::Hev1, + "av01" => Self::Av01, + "vp09" => Self::Vp09, + "mp4a" => Self::Mp4a, + _ => Self::OtherProbed, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum VideoProfile { + H264Baseline, + H264Main, + H264High, + H264High10, + HevcMain, + HevcMain10, + Av1Main, + Vp9Profile0, + Vp9Profile2, + Mpeg2Main, + Vc1Advanced, + OtherProbed, + Unknown, +} + +impl VideoProfile { + pub(crate) fn from_probe(codec: InputVideoCodec, value: Option<&str>) -> Self { + let normalized = value + .unwrap_or_default() + .to_ascii_lowercase() + .replace([' ', '_', '-'], ""); + if normalized.is_empty() { + return Self::Unknown; + } + match (codec, normalized.as_str()) { + (InputVideoCodec::H264, "baseline" | "constrainedbaseline") => Self::H264Baseline, + (InputVideoCodec::H264, "main") => Self::H264Main, + (InputVideoCodec::H264, "high") => Self::H264High, + (InputVideoCodec::H264, "high10" | "high10intra") => Self::H264High10, + (InputVideoCodec::Hevc, "main") => Self::HevcMain, + (InputVideoCodec::Hevc, "main10") => Self::HevcMain10, + (InputVideoCodec::Av1, "main") => Self::Av1Main, + (InputVideoCodec::Vp9, "profile0" | "0") => Self::Vp9Profile0, + (InputVideoCodec::Vp9, "profile2" | "2") => Self::Vp9Profile2, + (InputVideoCodec::Mpeg2, "main") => Self::Mpeg2Main, + (InputVideoCodec::Vc1, "advanced") => Self::Vc1Advanced, + _ => Self::OtherProbed, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PixelFormat { + Yuv420p, + Yuv420p10le, + Yuv422p, + Yuv422p10le, + Yuv444p, + Yuv444p10le, + Nv12, + P010le, + Gray8, + Gray10le, + OtherProbed, + Unknown, +} + +impl PixelFormat { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + match value.unwrap_or_default().to_ascii_lowercase().as_str() { + "" | "unknown" | "n/a" => Self::Unknown, + "yuv420p" => Self::Yuv420p, + "yuv420p10le" => Self::Yuv420p10le, + "yuv422p" => Self::Yuv422p, + "yuv422p10le" => Self::Yuv422p10le, + "yuv444p" => Self::Yuv444p, + "yuv444p10le" => Self::Yuv444p10le, + "nv12" => Self::Nv12, + "p010le" | "p010" => Self::P010le, + "gray" | "gray8" => Self::Gray8, + "gray10le" => Self::Gray10le, + _ => Self::OtherProbed, + } + } + + pub(crate) const fn inferred_bit_depth(self) -> Option { + match self { + Self::Yuv420p | Self::Yuv422p | Self::Yuv444p | Self::Nv12 | Self::Gray8 => Some(8), + Self::Yuv420p10le + | Self::Yuv422p10le + | Self::Yuv444p10le + | Self::P010le + | Self::Gray10le => Some(10), + Self::OtherProbed | Self::Unknown => None, + } + } + + pub(crate) const fn chroma(self) -> ChromaSubsampling { + match self { + Self::Yuv420p | Self::Yuv420p10le | Self::Nv12 | Self::P010le => { + ChromaSubsampling::Cs420 + } + Self::Yuv422p | Self::Yuv422p10le => ChromaSubsampling::Cs422, + Self::Yuv444p | Self::Yuv444p10le => ChromaSubsampling::Cs444, + Self::Gray8 | Self::Gray10le => ChromaSubsampling::Monochrome, + Self::OtherProbed | Self::Unknown => ChromaSubsampling::Unknown, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ChromaSubsampling { + Cs420, + Cs422, + Cs444, + Monochrome, + Unknown, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FieldOrder { + Progressive, + TopFirst, + BottomFirst, + Interlaced, + Unknown, +} + +impl FieldOrder { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + match value.unwrap_or_default().to_ascii_lowercase().as_str() { + "progressive" => Self::Progressive, + "tt" | "tb" | "top" | "top_first" => Self::TopFirst, + "bb" | "bt" | "bottom" | "bottom_first" => Self::BottomFirst, + "interlaced" => Self::Interlaced, + _ => Self::Unknown, + } + } +} + +macro_rules! probed_color_enum { + ($name:ident { $($variant:ident => $value:literal),+ $(,)? }) => { + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub enum $name { $($variant,)+ OtherProbed, Unknown } + + impl $name { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + match value.unwrap_or_default().to_ascii_lowercase().as_str() { + "" | "unknown" | "reserved" | "n/a" => Self::Unknown, + $($value => Self::$variant,)+ + _ => Self::OtherProbed, + } + } + } + }; +} + +probed_color_enum!(ColorPrimaries { + Bt709 => "bt709", + Bt2020 => "bt2020", + Smpte170m => "smpte170m", + Smpte432 => "smpte432", +}); +probed_color_enum!(ColorTransfer { + Bt709 => "bt709", + Smpte2084 => "smpte2084", + AribStdB67 => "arib-std-b67", + Iec6196621 => "iec61966-2-1", +}); +probed_color_enum!(ColorMatrix { + Bt709 => "bt709", + Bt2020Nc => "bt2020nc", + Bt2020C => "bt2020c", + Smpte170m => "smpte170m", + Rgb => "gbr", +}); + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ColorRange { + Limited, + Full, + OtherProbed, + Unknown, +} + +impl ColorRange { + pub(crate) fn from_probe(value: Option<&str>) -> Self { + match value.unwrap_or_default().to_ascii_lowercase().as_str() { + "tv" | "mpeg" | "limited" => Self::Limited, + "pc" | "jpeg" | "full" => Self::Full, + "" | "unknown" | "n/a" => Self::Unknown, + _ => Self::OtherProbed, + } + } +} diff --git a/server/src/transcoding/integration_tests.rs b/server/src/transcoding/integration_tests.rs index b1083d59..2edd9f29 100644 --- a/server/src/transcoding/integration_tests.rs +++ b/server/src/transcoding/integration_tests.rs @@ -1409,6 +1409,194 @@ async fn unavailable_service_is_side_effect_free_and_exposes_disabled_status() { assert_eq!(supervisor.active_processes(), 0); } +async fn fake_probe_service( + root: &Path, + cancellation: CancellationToken, +) -> (Arc, Arc) { + let config = isolated_config().with_explicit_root(root.to_path_buf()); + let supervisor = Arc::new(ProcessSupervisor::new(cancellation)); + let runtime = resolve_runtime(&config, &supervisor) + .await + .expect("resolve fake probe runtime"); + ( + Arc::new(TranscodingService::resolved( + config, + supervisor.clone(), + runtime, + )), + supervisor, + ) +} + +fn synthetic_probe_source(root: &Path, version: &str) -> crate::transcoding::ValidatedMediaSource { + let media = root.join("media.fixture"); + fs::write(&media, b"fixture media bytes").expect("write synthetic media source"); + crate::transcoding::ValidatedMediaSource::synthetic_fixture_path( + "synthetic-probe-source", + version, + media, + ) + .expect("construct sealed synthetic source") +} + +#[tokio::test] +async fn bounded_ffprobe_process_parses_output_single_flights_and_invalidates_by_version() { + let _guard = PROCESS_TEST_LOCK.lock().await; + let directory = tempfile::tempdir().expect("probe runtime root"); + let root = jellyfin_root(directory.path(), "probe-success"); + fs::write( + root.join("ffprobe.probe"), + include_bytes!("../../tests/fixtures/ffprobe/compatibility.json"), + ) + .expect("write probe output"); + let (service, supervisor) = fake_probe_service(&root, CancellationToken::new()).await; + let source = Arc::new(synthetic_probe_source(&root, "version-one")); + let mut tasks = Vec::new(); + for _ in 0..8 { + let service = service.clone(); + let source = source.clone(); + tasks.push(tokio::spawn(async move { + crate::transcoding::probe_media(&service, &source) + .await + .expect("probe through paired runtime") + })); + } + for task in tasks { + assert_eq!( + task.await + .expect("join probe") + .probe() + .selected_video_stream(), + Some(0) + ); + } + assert_eq!(fs::read_to_string(root.join("probe-count")).unwrap(), "1"); + + let changed = synthetic_probe_source(&root, "version-two"); + crate::transcoding::probe_media(&service, &changed) + .await + .expect("changed source version reprobes"); + assert_eq!(fs::read_to_string(root.join("probe-count")).unwrap(), "2"); + assert_eq!(supervisor.active_processes(), 0); +} + +#[tokio::test] +async fn bounded_ffprobe_process_rejects_nonzero_and_output_limit_failures() { + let _guard = PROCESS_TEST_LOCK.lock().await; + for (name, setup, expected) in [ + ( + "nonzero", + ("probe-exit-code", b"7".as_slice()), + crate::transcoding::ProbeErrorCode::NonZeroExit, + ), + ( + "stdout-limit", + ( + "ffprobe.probe", + &vec![b'o'; crate::transcoding::probe::MAX_PROBE_STDOUT_BYTES + 1], + ), + crate::transcoding::ProbeErrorCode::OutputTooLarge, + ), + ( + "stderr-limit", + ("probe-stderr", &vec![b'e'; 1024 * 1024 + 1]), + crate::transcoding::ProbeErrorCode::ProcessFailure, + ), + ] { + let directory = tempfile::tempdir().expect("probe failure runtime root"); + let root = jellyfin_root(directory.path(), name); + fs::write(root.join(setup.0), setup.1).expect("write probe failure control"); + let (service, supervisor) = fake_probe_service(&root, CancellationToken::new()).await; + let source = synthetic_probe_source(&root, "failure-version"); + let error = crate::transcoding::probe_media(&service, &source) + .await + .expect_err("probe failure must fail closed"); + assert_eq!(error.code(), expected, "failure scenario {name}"); + assert_eq!(supervisor.active_processes(), 0, "failure scenario {name}"); + } +} + +#[tokio::test] +async fn bounded_ffprobe_process_cancellation_terminates_the_stalled_child() { + let _guard = PROCESS_TEST_LOCK.lock().await; + let directory = tempfile::tempdir().expect("probe cancellation runtime root"); + let root = jellyfin_root(directory.path(), "probe-cancel"); + fs::write(root.join("probe-stall"), b"stall").expect("enable probe stall"); + let cancellation = CancellationToken::new(); + let (service, supervisor) = fake_probe_service(&root, cancellation.clone()).await; + let source = Arc::new(synthetic_probe_source(&root, "cancel-version")); + let probe = { + let service = service.clone(); + let source = source.clone(); + tokio::spawn(async move { crate::transcoding::probe_media(&service, &source).await }) + }; + let marker = root.join("probe-started"); + tokio::time::timeout(Duration::from_secs(5), async { + while !marker.is_file() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("stalled probe process started"); + cancellation.cancel(); + let error = tokio::time::timeout(Duration::from_secs(10), probe) + .await + .expect("cancelled probe completes") + .expect("join cancelled probe") + .expect_err("cancelled probe fails"); + assert_eq!(error.code(), crate::transcoding::ProbeErrorCode::Cancelled); + supervisor + .wait_for_idle(Duration::from_secs(10)) + .await + .expect("cancelled probe process reaped"); + assert_eq!(supervisor.active_processes(), 0); +} + +#[tokio::test] +async fn bounded_ffprobe_process_stall_hits_the_probe_inactivity_deadline_and_reaps() { + let _guard = PROCESS_TEST_LOCK.lock().await; + let directory = tempfile::tempdir().expect("probe deadline runtime root"); + let root = jellyfin_root(directory.path(), "probe-deadline"); + fs::write(root.join("probe-stall"), b"stall").expect("enable probe stall"); + let (service, supervisor) = fake_probe_service(&root, CancellationToken::new()).await; + let source = Arc::new(synthetic_probe_source(&root, "deadline-version")); + let probe = { + let service = service.clone(); + let source = source.clone(); + tokio::spawn(async move { crate::transcoding::probe_media(&service, &source).await }) + }; + let marker = root.join("probe-started"); + tokio::time::timeout(Duration::from_secs(5), async { + while !marker.is_file() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("stalled probe process started"); + + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(31)).await; + for _ in 0..15 { + if probe.is_finished() { + break; + } + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + } + tokio::time::resume(); + let error = tokio::time::timeout(Duration::from_secs(10), probe) + .await + .expect("deadline probe cleanup completes") + .expect("join deadline probe") + .expect_err("stalled probe fails"); + assert_eq!(error.code(), crate::transcoding::ProbeErrorCode::Inactivity); + supervisor + .wait_for_idle(Duration::from_secs(10)) + .await + .expect("deadline probe process reaped"); + assert_eq!(supervisor.active_processes(), 0); +} + #[cfg(windows)] #[tokio::test] async fn explicit_unc_and_device_paths_are_rejected_without_a_probe() { @@ -1476,6 +1664,9 @@ fn main() { if runtime_query(&args) { return; } + if probe_query(&args) { + return; + } let mode = args.first().and_then(|arg| arg.to_str()); match mode { Some("--emit") => { @@ -1548,6 +1739,36 @@ fn main() { } } +fn probe_query(args: &[OsString]) -> bool { + if !args.iter().any(|argument| argument == "-show_format") { + return false; + } + let root = env::current_dir().unwrap(); + fs::write(root.join("probe-started"), b"started").unwrap(); + let counter = root.join("probe-count"); + let count = fs::read_to_string(&counter) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0) + .saturating_add(1); + fs::write(counter, count.to_string()).unwrap(); + if root.join("probe-stall").is_file() { + loop { thread::sleep(Duration::from_secs(1)); } + } + if let Ok(stderr) = fs::read(root.join("probe-stderr")) { + io::stderr().write_all(&stderr).unwrap(); + } + if let Ok(stdout) = fs::read(root.join("ffprobe.probe")) { + io::stdout().write_all(&stdout).unwrap(); + } else { + io::stdout().write_all(b"{\"format\":{},\"streams\":[]}").unwrap(); + } + if let Ok(code) = fs::read_to_string(root.join("probe-exit-code")) { + std::process::exit(code.trim().parse().unwrap()); + } + true +} + fn runtime_query(args: &[OsString]) -> bool { let Some(query) = args.first().and_then(|arg| arg.to_str()) else { return false; diff --git a/server/src/transcoding/mod.rs b/server/src/transcoding/mod.rs index 0b4beb9e..2401d2c5 100644 --- a/server/src/transcoding/mod.rs +++ b/server/src/transcoding/mod.rs @@ -1,5 +1,7 @@ +pub mod codec; pub mod error; pub mod model; +pub mod probe; /// Raw process construction is internal to the transcoding runtime. /// /// ```compile_fail @@ -13,13 +15,28 @@ pub mod runtime; pub mod runtime_manifest; #[cfg(any(unix, test))] pub(crate) mod snapshot_helper; +pub mod source; +pub use codec::{ + ChromaSubsampling, ColorMatrix, ColorPrimaries, ColorRange, ColorTransfer, ContainerKind, + FieldOrder, InputVideoCodec, OutputVideoCodec, PixelFormat, SampleEntry, VideoProfile, +}; pub use error::{FailureCode, TranscodeFailure}; pub use model::{ AccelerationClass, AccelerationMode, BackendKind, CapabilityState, DeviceClass, DeviceId, FrameRateClass, KeyframeStrategy, MediaDescriptor, OutputContract, PresetIntent, RateControlEnvelope, RateControlIntent, RationalRate, StageKind, StageMode, TranscodePlan, - TranscodeRequest, ValidatedMediaSource, VideoStage, + TranscodeRequest, VideoStage, +}; +pub use probe::{ + AudioStreamDescriptor, ChapterDescriptor, ColorDescriptor, ContentLightMetadata, + DolbyVisionMetadata, HdrMetadata, MasteringDisplayMetadata, MediaStreamDescriptor, + ProbeDocument, ProbeError, ProbeErrorCode, ProbeRational, SafeProbeText, StreamDisposition, + SubtitleStreamDescriptor, VideoStreamDescriptor, parse_probe_document, probe_media, +}; +pub use source::{ + CompletedFileSource, EngineSource, FixtureSource, RemoteSourceHandle, SourceActivitySnapshot, + SourceBroker, SourceError, SourceProtocolPolicy, ValidatedMediaSource, issue_engine_source, }; #[cfg(test)] diff --git a/server/src/transcoding/model.rs b/server/src/transcoding/model.rs index 4c93e0d3..e68a80b6 100644 --- a/server/src/transcoding/model.rs +++ b/server/src/transcoding/model.rs @@ -425,185 +425,38 @@ pub enum KeyframeStrategy { TimeForced { segment_duration_ms: NonZeroU32 }, } -/// A media source capability issued by a trusted internal owner. -/// -/// Route/external code cannot manufacture a trusted source from JSON: -/// -/// ```compile_fail -/// use stream_server::transcoding::ValidatedMediaSource; -/// -/// let _: ValidatedMediaSource = serde_json::from_str( -/// r#"{"approvedRemote":{"id":"route-text"}}"#, -/// ).unwrap(); -/// ``` -/// -/// It also cannot call the internal issuance seams directly: -/// -/// ```compile_fail -/// use stream_server::transcoding::ValidatedMediaSource; -/// -/// let _ = ValidatedMediaSource::approved_remote("route-text"); -/// ``` -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum ValidatedMediaSource { - CompletedFile(CompletedFileSource), - EngineLoopback(EngineLoopbackSource), - ApprovedRemote(ApprovedRemoteSource), - SyntheticFixture(SyntheticFixtureSource), -} - -#[allow( - dead_code, - reason = "sealed issuance seams are consumed by the source broker and verifier in later planned tasks" -)] -impl ValidatedMediaSource { - pub(in crate::transcoding) fn completed_file( - id: impl Into, - ) -> Result { - Ok(Self::CompletedFile(CompletedFileSource::new(id)?)) - } - - pub(in crate::transcoding) fn engine_loopback( - id: impl Into, - ) -> Result { - Ok(Self::EngineLoopback(EngineLoopbackSource::new(id)?)) - } - - pub(in crate::transcoding) fn approved_remote( - id: impl Into, - ) -> Result { - Ok(Self::ApprovedRemote(ApprovedRemoteSource::new(id)?)) - } - - pub(in crate::transcoding) fn synthetic_fixture( - id: impl Into, - ) -> Result { - Ok(Self::SyntheticFixture(SyntheticFixtureSource::new(id)?)) - } - - pub fn id(&self) -> &str { - match self { - Self::CompletedFile(source) => source.id(), - Self::EngineLoopback(source) => source.id(), - Self::ApprovedRemote(source) => source.id(), - Self::SyntheticFixture(source) => source.id(), - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct CompletedFileSource { - id: SourceId, -} - -impl CompletedFileSource { - fn new(id: impl Into) -> Result { - Ok(Self { - id: SourceId::new(id)?, - }) - } - - pub fn id(&self) -> &str { - self.id.as_str() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct EngineLoopbackSource { - id: SourceId, -} - -impl EngineLoopbackSource { - fn new(id: impl Into) -> Result { - Ok(Self { - id: SourceId::new(id)?, - }) - } - - pub fn id(&self) -> &str { - self.id.as_str() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct ApprovedRemoteSource { - id: SourceId, -} - -impl ApprovedRemoteSource { - fn new(id: impl Into) -> Result { - Ok(Self { - id: SourceId::new(id)?, - }) - } - - pub fn id(&self) -> &str { - self.id.as_str() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SyntheticFixtureSource { - id: SourceId, -} - -impl SyntheticFixtureSource { - fn new(id: impl Into) -> Result { - Ok(Self { - id: SourceId::new(id)?, - }) - } - - pub fn id(&self) -> &str { - self.id.as_str() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] -#[serde(transparent)] -struct SourceId(String); - -impl SourceId { - fn new(value: impl Into) -> Result { - let value = value.into(); - if is_safe_identifier(&value) { - Ok(Self(value)) - } else { - Err(ModelValidationError::new("invalid media source id")) - } - } - - fn as_str(&self) -> &str { - &self.0 - } -} - #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] #[serde(rename_all = "camelCase")] pub struct MediaDescriptor { - source: ValidatedMediaSource, - frame_rate_class: FrameRateClass, + source: super::source::ValidatedMediaSource, + probe: super::probe::ProbeDocument, } impl MediaDescriptor { - pub fn new(source: ValidatedMediaSource, frame_rate_class: FrameRateClass) -> Self { - Self { - source, - frame_rate_class, - } + pub(crate) fn from_probe( + source: super::source::ValidatedMediaSource, + probe: super::probe::ProbeDocument, + ) -> Self { + Self { source, probe } } - pub fn source(&self) -> &ValidatedMediaSource { + pub fn source(&self) -> &super::source::ValidatedMediaSource { &self.source } + pub fn probe(&self) -> &super::probe::ProbeDocument { + &self.probe + } + pub fn frame_rate_class(&self) -> FrameRateClass { - self.frame_rate_class + self.probe + .selected_video() + .map(super::probe::VideoStreamDescriptor::frame_rate_class) + .unwrap_or(FrameRateClass::Unknown) + } + + pub fn media_signature(&self) -> String { + self.probe.media_signature() } } @@ -634,14 +487,14 @@ impl OutputContract { #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] #[serde(rename_all = "camelCase")] pub struct TranscodeRequest { - source: ValidatedMediaSource, + source: super::source::ValidatedMediaSource, output: OutputContract, acceleration_mode: AccelerationMode, } impl TranscodeRequest { pub fn new( - source: ValidatedMediaSource, + source: super::source::ValidatedMediaSource, output: OutputContract, acceleration_mode: AccelerationMode, ) -> Self { @@ -652,7 +505,7 @@ impl TranscodeRequest { } } - pub fn source(&self) -> &ValidatedMediaSource { + pub fn source(&self) -> &super::source::ValidatedMediaSource { &self.source } @@ -774,6 +627,7 @@ fn gcd(mut left: u32, mut right: u32) -> u32 { #[cfg(test)] mod tests { use super::*; + use crate::transcoding::ValidatedMediaSource; use serde::{Serialize, de::DeserializeOwned}; use std::num::NonZeroU32; diff --git a/server/src/transcoding/probe.rs b/server/src/transcoding/probe.rs new file mode 100644 index 00000000..f0504e3d --- /dev/null +++ b/server/src/transcoding/probe.rs @@ -0,0 +1,1789 @@ +use super::{ + codec::{ + ChromaSubsampling, ColorMatrix, ColorPrimaries, ColorRange, ColorTransfer, ContainerKind, + FieldOrder, InputVideoCodec, PixelFormat, SampleEntry, VideoProfile, + }, + model::{FrameRateClass, MediaDescriptor, RationalRate}, + process::{ProcessErrorCode, StdoutPolicy}, + runtime::{RuntimeCommand, RuntimeCommandError, RuntimeExecutable, TranscodingService}, + source::{SourceActivitySnapshot, SourceError, ValidatedMediaSource}, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + ffi::OsString, + fmt, + num::NonZeroU32, + sync::{Arc, Mutex}, + time::Duration, +}; +use tokio::{sync::Notify, time::Instant}; +use tokio_util::sync::CancellationToken; + +pub const MAX_PROBE_STDOUT_BYTES: usize = 8 * 1024 * 1024; +pub const MAX_PROBE_STDERR_BYTES: usize = 1024 * 1024; +const MAX_JSON_DEPTH: usize = 32; +const MAX_JSON_NODES: usize = 65_536; +const MAX_JSON_STRING_BYTES: usize = 4_096; +const MAX_STREAMS: usize = 128; +const MAX_CHAPTERS: usize = 2_048; +const MAX_SIDE_DATA_PER_STREAM: usize = 64; +const PROBE_INACTIVITY: Duration = Duration::from_secs(30); +const PROBE_STARVATION_DEFAULT: Duration = Duration::from_secs(10 * 60); +const PROBE_HARD_DEADLINE: Duration = Duration::from_secs(30 * 60); +const CACHE_TTL: Duration = Duration::from_secs(5 * 60); +const CACHE_MAX_ENTRIES: usize = 128; +const CACHE_MAX_WEIGHT: usize = 16 * 1024 * 1024; +const CACHE_MAX_IN_FLIGHT: usize = 16; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProbeErrorCode { + OutputTooLarge, + MalformedOutput, + LimitExceeded, + RuntimeUnavailable, + SourceInvalid, + ProcessFailure, + NonZeroExit, + Inactivity, + SourceStarvation, + OverallDeadline, + CapacityExceeded, + Cancelled, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProbeError { + code: ProbeErrorCode, +} + +impl ProbeError { + const fn new(code: ProbeErrorCode) -> Self { + Self { code } + } + + pub const fn code(&self) -> ProbeErrorCode { + self.code + } +} + +impl fmt::Display for ProbeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self.code { + ProbeErrorCode::OutputTooLarge => "probe output exceeded its byte limit", + ProbeErrorCode::MalformedOutput => "probe output was malformed", + ProbeErrorCode::LimitExceeded => "probe metadata exceeded a structural limit", + ProbeErrorCode::RuntimeUnavailable => "paired media runtime is unavailable", + ProbeErrorCode::SourceInvalid => "validated media source is no longer usable", + ProbeErrorCode::ProcessFailure => "media probe process failed", + ProbeErrorCode::NonZeroExit => "media probe exited unsuccessfully", + ProbeErrorCode::Inactivity => "media probe made no source progress", + ProbeErrorCode::SourceStarvation => "media probe source remained unavailable", + ProbeErrorCode::OverallDeadline => "media probe exceeded its hard overall deadline", + ProbeErrorCode::CapacityExceeded => "media probe admission capacity is exhausted", + ProbeErrorCode::Cancelled => "media probe was cancelled", + }) + } +} + +impl std::error::Error for ProbeError {} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(transparent)] +pub struct SafeProbeText(String); + +impl SafeProbeText { + fn parse(value: Option<&str>, fallback: &'static str) -> Result { + let value = value.filter(|value| !value.is_empty()).unwrap_or(fallback); + if value.len() > 256 || value.chars().any(char::is_control) { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + Ok(Self(value.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProbeRational { + numerator: i64, + denominator: u64, +} + +impl ProbeRational { + pub fn numerator(self) -> i64 { + self.numerator + } + + pub fn denominator(self) -> u64 { + self.denominator + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamDisposition { + pub default: bool, + pub forced: bool, + pub hearing_impaired: bool, + pub visual_impaired: bool, + pub attached_picture: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ColorDescriptor { + pub primaries: ColorPrimaries, + pub transfer: ColorTransfer, + pub matrix: ColorMatrix, + pub range: ColorRange, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MasteringDisplayMetadata { + pub red_x: Option, + pub red_y: Option, + pub green_x: Option, + pub green_y: Option, + pub blue_x: Option, + pub blue_y: Option, + pub white_point_x: Option, + pub white_point_y: Option, + pub min_luminance: Option, + pub max_luminance: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ContentLightMetadata { + pub max_content: Option, + pub max_average: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DolbyVisionMetadata { + pub profile: Option, + pub level: Option, + pub rpu_present: Option, + pub enhancement_layer_present: Option, + pub base_layer_present: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HdrMetadata { + mastering_display: Option, + content_light: Option, + dolby_vision: Option, +} + +impl HdrMetadata { + pub fn mastering_display(&self) -> Option<&MasteringDisplayMetadata> { + self.mastering_display.as_ref() + } + + pub fn content_light(&self) -> Option { + self.content_light + } + + pub fn dolby_vision(&self) -> Option { + self.dolby_vision + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct VideoStreamDescriptor { + index: u32, + codec: InputVideoCodec, + codec_display: SafeProbeText, + sample_entry: SampleEntry, + codec_tag: Option, + profile: VideoProfile, + profile_display: Option, + level: Option, + start_micros: Option, + duration_micros: Option, + width: Option, + height: Option, + sample_aspect_ratio: Option, + display_aspect_ratio: Option, + pixel_format: PixelFormat, + pixel_format_display: Option, + bit_depth: Option, + chroma: ChromaSubsampling, + nominal_frame_rate: Option, + average_frame_rate: Option, + frame_rate_class: FrameRateClass, + stream_time_base: Option, + codec_time_base: Option, + field_order: FieldOrder, + rotation_degrees: Option, + color: ColorDescriptor, + hdr: HdrMetadata, + bitrate: Option, + language: Option, + disposition: StreamDisposition, +} + +impl VideoStreamDescriptor { + pub fn index(&self) -> u32 { + self.index + } + pub fn codec(&self) -> InputVideoCodec { + self.codec + } + pub fn codec_display(&self) -> &str { + self.codec_display.as_str() + } + pub fn sample_entry(&self) -> SampleEntry { + self.sample_entry + } + pub fn codec_tag(&self) -> Option<&str> { + self.codec_tag.as_ref().map(SafeProbeText::as_str) + } + pub fn profile(&self) -> VideoProfile { + self.profile + } + pub fn profile_display(&self) -> Option<&str> { + self.profile_display.as_ref().map(SafeProbeText::as_str) + } + pub fn level(&self) -> Option { + self.level + } + pub fn pixel_format(&self) -> PixelFormat { + self.pixel_format + } + pub fn pixel_format_display(&self) -> Option<&str> { + self.pixel_format_display + .as_ref() + .map(SafeProbeText::as_str) + } + pub fn bit_depth(&self) -> Option { + self.bit_depth + } + pub fn chroma(&self) -> ChromaSubsampling { + self.chroma + } + pub fn sample_aspect_ratio(&self) -> Option { + self.sample_aspect_ratio + } + pub fn display_aspect_ratio(&self) -> Option { + self.display_aspect_ratio + } + pub fn nominal_frame_rate(&self) -> Option { + self.nominal_frame_rate + } + pub fn average_frame_rate(&self) -> Option { + self.average_frame_rate + } + pub fn frame_rate_class(&self) -> FrameRateClass { + self.frame_rate_class + } + pub fn stream_time_base(&self) -> Option { + self.stream_time_base + } + pub fn codec_time_base(&self) -> Option { + self.codec_time_base + } + pub fn field_order(&self) -> FieldOrder { + self.field_order + } + pub fn color(&self) -> ColorDescriptor { + self.color + } + pub fn rotation_degrees(&self) -> Option { + self.rotation_degrees + } + pub fn hdr(&self) -> &HdrMetadata { + &self.hdr + } + pub fn width(&self) -> Option { + self.width + } + pub fn height(&self) -> Option { + self.height + } + pub fn bitrate(&self) -> Option { + self.bitrate + } + pub fn start_micros(&self) -> Option { + self.start_micros + } + pub fn duration_micros(&self) -> Option { + self.duration_micros + } + pub fn language(&self) -> Option<&str> { + self.language.as_ref().map(SafeProbeText::as_str) + } + pub fn disposition(&self) -> StreamDisposition { + self.disposition + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AudioStreamDescriptor { + index: u32, + codec_display: SafeProbeText, + codec_tag: Option, + profile_display: Option, + start_micros: Option, + duration_micros: Option, + stream_time_base: Option, + sample_rate: Option, + channels: Option, + channel_layout: Option, + bitrate: Option, + language: Option, + disposition: StreamDisposition, +} + +impl AudioStreamDescriptor { + pub fn index(&self) -> u32 { + self.index + } + pub fn codec_display(&self) -> &str { + self.codec_display.as_str() + } + pub fn codec_tag(&self) -> Option<&str> { + self.codec_tag.as_ref().map(SafeProbeText::as_str) + } + pub fn profile_display(&self) -> Option<&str> { + self.profile_display.as_ref().map(SafeProbeText::as_str) + } + pub fn start_micros(&self) -> Option { + self.start_micros + } + pub fn duration_micros(&self) -> Option { + self.duration_micros + } + pub fn stream_time_base(&self) -> Option { + self.stream_time_base + } + pub fn sample_rate(&self) -> Option { + self.sample_rate + } + pub fn channels(&self) -> Option { + self.channels + } + pub fn channel_layout(&self) -> Option<&str> { + self.channel_layout.as_ref().map(SafeProbeText::as_str) + } + pub fn bitrate(&self) -> Option { + self.bitrate + } + pub fn language(&self) -> Option<&str> { + self.language.as_ref().map(SafeProbeText::as_str) + } + pub fn disposition(&self) -> StreamDisposition { + self.disposition + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SubtitleStreamDescriptor { + index: u32, + codec_display: SafeProbeText, + codec_tag: Option, + start_micros: Option, + duration_micros: Option, + stream_time_base: Option, + language: Option, + disposition: StreamDisposition, +} + +impl SubtitleStreamDescriptor { + pub fn index(&self) -> u32 { + self.index + } + pub fn codec_display(&self) -> &str { + self.codec_display.as_str() + } + pub fn codec_tag(&self) -> Option<&str> { + self.codec_tag.as_ref().map(SafeProbeText::as_str) + } + pub fn start_micros(&self) -> Option { + self.start_micros + } + pub fn duration_micros(&self) -> Option { + self.duration_micros + } + pub fn stream_time_base(&self) -> Option { + self.stream_time_base + } + pub fn language(&self) -> Option<&str> { + self.language.as_ref().map(SafeProbeText::as_str) + } + pub fn disposition(&self) -> StreamDisposition { + self.disposition + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MediaStreamDescriptor { + Video(Box), + Audio(AudioStreamDescriptor), + Subtitle(SubtitleStreamDescriptor), + Other { + index: u32, + track_display: SafeProbeText, + codec_display: SafeProbeText, + codec_tag: Option, + start_micros: Option, + duration_micros: Option, + stream_time_base: Option, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ChapterDescriptor { + id: i64, + start_micros: Option, + end_micros: Option, + title: Option, +} + +impl ChapterDescriptor { + pub fn id(&self) -> i64 { + self.id + } + pub fn start_micros(&self) -> Option { + self.start_micros + } + pub fn end_micros(&self) -> Option { + self.end_micros + } + pub fn title(&self) -> Option<&str> { + self.title.as_ref().map(SafeProbeText::as_str) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProbeDocument { + container: ContainerKind, + container_display: SafeProbeText, + start_micros: Option, + duration_micros: Option, + bitrate: Option, + streams: Vec, + chapters: Vec, + selected_video_stream: Option, + selected_audio_stream: Option, +} + +impl ProbeDocument { + pub fn container(&self) -> ContainerKind { + self.container + } + pub fn container_display(&self) -> &str { + self.container_display.as_str() + } + pub fn start_micros(&self) -> Option { + self.start_micros + } + pub fn duration_micros(&self) -> Option { + self.duration_micros + } + pub fn bitrate(&self) -> Option { + self.bitrate + } + pub fn streams(&self) -> &[MediaStreamDescriptor] { + &self.streams + } + pub fn chapters(&self) -> &[ChapterDescriptor] { + &self.chapters + } + pub fn selected_video_stream(&self) -> Option { + self.selected_video_stream + } + pub fn selected_audio_stream(&self) -> Option { + self.selected_audio_stream + } + + pub fn video_streams(&self) -> impl Iterator { + self.streams.iter().filter_map(|stream| match stream { + MediaStreamDescriptor::Video(video) => Some(video.as_ref()), + _ => None, + }) + } + + pub fn audio_streams(&self) -> impl Iterator { + self.streams.iter().filter_map(|stream| match stream { + MediaStreamDescriptor::Audio(audio) => Some(audio), + _ => None, + }) + } + + pub fn subtitle_streams(&self) -> impl Iterator { + self.streams.iter().filter_map(|stream| match stream { + MediaStreamDescriptor::Subtitle(subtitle) => Some(subtitle), + _ => None, + }) + } + + pub fn selected_video(&self) -> Option<&VideoStreamDescriptor> { + let selected = self.selected_video_stream?; + self.video_streams().find(|video| video.index == selected) + } + + pub fn selected_audio(&self) -> Option<&AudioStreamDescriptor> { + let selected = self.selected_audio_stream?; + self.audio_streams().find(|audio| audio.index == selected) + } + + pub fn media_signature(&self) -> String { + let selected_video = self.selected_video().map(TypedVideoSignature::from); + let signature = TypedMediaSignature { + container: self.container, + selected_video, + }; + let bytes = serde_json::to_vec(&signature) + .expect("typed media signature serialization is infallible"); + hex::encode(Sha256::digest(bytes)) + } + + pub(crate) fn estimated_weight(&self) -> usize { + serde_json::to_vec(self).map_or(MAX_PROBE_STDOUT_BYTES, |bytes| bytes.len()) + } +} + +#[derive(Serialize)] +struct TypedMediaSignature<'a> { + container: ContainerKind, + selected_video: Option>, +} + +#[derive(Serialize)] +struct TypedVideoSignature<'a> { + index: u32, + codec: InputVideoCodec, + sample_entry: SampleEntry, + profile: VideoProfile, + level: Option, + width: Option, + height: Option, + sample_aspect_ratio: Option, + display_aspect_ratio: Option, + pixel_format: PixelFormat, + bit_depth: Option, + chroma: ChromaSubsampling, + nominal_frame_rate: Option, + average_frame_rate: Option, + frame_rate_class: FrameRateClass, + stream_time_base: Option, + codec_time_base: Option, + field_order: FieldOrder, + rotation_degrees: Option, + color: ColorDescriptor, + hdr: &'a HdrMetadata, +} + +impl<'a> From<&'a VideoStreamDescriptor> for TypedVideoSignature<'a> { + fn from(video: &'a VideoStreamDescriptor) -> Self { + Self { + index: video.index, + codec: video.codec, + sample_entry: video.sample_entry, + profile: video.profile, + level: video.level, + width: video.width, + height: video.height, + sample_aspect_ratio: video.sample_aspect_ratio, + display_aspect_ratio: video.display_aspect_ratio, + pixel_format: video.pixel_format, + bit_depth: video.bit_depth, + chroma: video.chroma, + nominal_frame_rate: video.nominal_frame_rate, + average_frame_rate: video.average_frame_rate, + frame_rate_class: video.frame_rate_class, + stream_time_base: video.stream_time_base, + codec_time_base: video.codec_time_base, + field_order: video.field_order, + rotation_degrees: video.rotation_degrees, + color: video.color, + hdr: &video.hdr, + } + } +} + +#[derive(Deserialize)] +struct RawProbe { + #[serde(default)] + format: RawFormat, + #[serde(default)] + streams: Vec, + #[serde(default)] + chapters: Vec, +} + +#[derive(Default, Deserialize)] +struct RawFormat { + format_name: Option, + start_time: Option, + duration: Option, + bit_rate: Option, +} + +#[derive(Deserialize)] +struct RawStream { + index: u32, + codec_type: Option, + codec_name: Option, + codec_tag_string: Option, + profile: Option, + level: Option, + start_time: Option, + duration: Option, + width: Option, + height: Option, + sample_aspect_ratio: Option, + display_aspect_ratio: Option, + pix_fmt: Option, + bits_per_raw_sample: Option, + bits_per_sample: Option, + r_frame_rate: Option, + avg_frame_rate: Option, + time_base: Option, + codec_time_base: Option, + field_order: Option, + color_range: Option, + color_space: Option, + color_transfer: Option, + color_primaries: Option, + bit_rate: Option, + sample_rate: Option, + channels: Option, + channel_layout: Option, + #[serde(default)] + tags: HashMap, + #[serde(default)] + disposition: HashMap, + #[serde(default, alias = "side_data")] + side_data_list: Vec, +} + +#[derive(Deserialize)] +struct RawChapter { + id: i64, + start_time: Option, + end_time: Option, + #[serde(default)] + tags: HashMap, +} + +pub fn parse_probe_document(bytes: &[u8]) -> Result { + if bytes.len() > MAX_PROBE_STDOUT_BYTES { + return Err(ProbeError::new(ProbeErrorCode::OutputTooLarge)); + } + validate_json_shape(bytes)?; + let value: serde_json::Value = serde_json::from_slice(bytes) + .map_err(|_| ProbeError::new(ProbeErrorCode::MalformedOutput))?; + validate_json_value(&value, 0, &mut 0)?; + let raw: RawProbe = serde_json::from_value(value) + .map_err(|_| ProbeError::new(ProbeErrorCode::MalformedOutput))?; + if raw.streams.len() > MAX_STREAMS || raw.chapters.len() > MAX_CHAPTERS { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + + let mut streams = Vec::with_capacity(raw.streams.len()); + let mut stream_indices = HashSet::with_capacity(raw.streams.len()); + for raw_stream in raw.streams { + if !stream_indices.insert(raw_stream.index) { + return Err(ProbeError::new(ProbeErrorCode::MalformedOutput)); + } + if raw_stream.tags.len() > 64 || raw_stream.side_data_list.len() > MAX_SIDE_DATA_PER_STREAM + { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + streams.push(parse_stream(raw_stream)?); + } + streams.sort_by_key(stream_index); + let chapters = raw + .chapters + .into_iter() + .map(parse_chapter) + .collect::, _>>()?; + let selected_video_stream = select_stream(&streams, true); + let selected_audio_stream = select_stream(&streams, false); + + Ok(ProbeDocument { + container: ContainerKind::from_probe(raw.format.format_name.as_deref()), + container_display: SafeProbeText::parse(raw.format.format_name.as_deref(), "unknown")?, + start_micros: parse_signed_micros(raw.format.start_time.as_deref()), + duration_micros: parse_unsigned_micros(raw.format.duration.as_deref()), + bitrate: parse_u64_text(raw.format.bit_rate.as_deref()), + streams, + chapters, + selected_video_stream, + selected_audio_stream, + }) +} + +fn validate_json_shape(bytes: &[u8]) -> Result<(), ProbeError> { + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + let mut string_len = 0usize; + for &byte in bytes { + if in_string { + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + in_string = false; + string_len = 0; + } else { + string_len = string_len.saturating_add(1); + if string_len > MAX_JSON_STRING_BYTES { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + } + continue; + } + match byte { + b'"' => in_string = true, + b'{' | b'[' => { + depth = depth.saturating_add(1); + if depth > MAX_JSON_DEPTH { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + } + b'}' | b']' => depth = depth.saturating_sub(1), + _ => {} + } + } + if in_string || depth != 0 { + return Err(ProbeError::new(ProbeErrorCode::MalformedOutput)); + } + Ok(()) +} + +fn validate_json_value( + value: &serde_json::Value, + depth: usize, + nodes: &mut usize, +) -> Result<(), ProbeError> { + if depth > MAX_JSON_DEPTH { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + *nodes = nodes.saturating_add(1); + if *nodes > MAX_JSON_NODES { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + match value { + serde_json::Value::String(value) if value.len() > MAX_JSON_STRING_BYTES => { + Err(ProbeError::new(ProbeErrorCode::LimitExceeded)) + } + serde_json::Value::Array(values) => values + .iter() + .try_for_each(|value| validate_json_value(value, depth + 1, nodes)), + serde_json::Value::Object(values) => values + .values() + .try_for_each(|value| validate_json_value(value, depth + 1, nodes)), + _ => Ok(()), + } +} + +fn parse_stream(raw: RawStream) -> Result { + let disposition = parse_disposition(&raw.disposition); + let language = optional_safe_text(raw.tags.get("language"))?; + match raw.codec_type.as_deref() { + Some("video") => { + let codec = InputVideoCodec::from_probe(raw.codec_name.as_deref()); + let pixel_format = PixelFormat::from_probe(raw.pix_fmt.as_deref()); + let nominal_frame_rate = parse_rate(raw.r_frame_rate.as_deref()); + let average_frame_rate = parse_rate(raw.avg_frame_rate.as_deref()); + let frame_rate_class = match (nominal_frame_rate, average_frame_rate) { + (Some(nominal), Some(average)) if nominal == average => FrameRateClass::Constant, + (Some(_), Some(_)) => FrameRateClass::Variable, + _ => FrameRateClass::Unknown, + }; + let (side_data_rotation, hdr) = parse_side_data(&raw.side_data_list)?; + let rotation_degrees = side_data_rotation.or_else(|| { + raw.tags + .get("rotate") + .and_then(|value| value.parse::().ok()) + .filter(|value| (-360..=360).contains(value)) + }); + let bit_depth = parse_u8_text(raw.bits_per_raw_sample.as_deref()) + .or(raw.bits_per_sample) + .filter(|depth| (1..=16).contains(depth)) + .or_else(|| pixel_format.inferred_bit_depth()); + Ok(MediaStreamDescriptor::Video(Box::new( + VideoStreamDescriptor { + index: raw.index, + codec, + codec_display: SafeProbeText::parse(raw.codec_name.as_deref(), "unknown")?, + sample_entry: SampleEntry::from_probe(raw.codec_tag_string.as_deref()), + codec_tag: optional_safe_text(raw.codec_tag_string.as_ref())?, + profile: VideoProfile::from_probe(codec, raw.profile.as_deref()), + profile_display: optional_safe_text(raw.profile.as_ref())?, + level: raw.level, + start_micros: parse_signed_micros(raw.start_time.as_deref()), + duration_micros: parse_unsigned_micros(raw.duration.as_deref()), + width: raw.width.filter(|value| *value > 0), + height: raw.height.filter(|value| *value > 0), + sample_aspect_ratio: parse_ratio(raw.sample_aspect_ratio.as_deref()), + display_aspect_ratio: parse_ratio(raw.display_aspect_ratio.as_deref()), + pixel_format, + pixel_format_display: optional_safe_text(raw.pix_fmt.as_ref())?, + bit_depth, + chroma: pixel_format.chroma(), + nominal_frame_rate, + average_frame_rate, + frame_rate_class, + stream_time_base: parse_ratio(raw.time_base.as_deref()), + codec_time_base: parse_ratio(raw.codec_time_base.as_deref()), + field_order: FieldOrder::from_probe(raw.field_order.as_deref()), + rotation_degrees, + color: ColorDescriptor { + primaries: ColorPrimaries::from_probe(raw.color_primaries.as_deref()), + transfer: ColorTransfer::from_probe(raw.color_transfer.as_deref()), + matrix: ColorMatrix::from_probe(raw.color_space.as_deref()), + range: ColorRange::from_probe(raw.color_range.as_deref()), + }, + hdr, + bitrate: parse_u64_text(raw.bit_rate.as_deref()), + language, + disposition, + }, + ))) + } + Some("audio") => Ok(MediaStreamDescriptor::Audio(AudioStreamDescriptor { + index: raw.index, + codec_display: SafeProbeText::parse(raw.codec_name.as_deref(), "unknown")?, + codec_tag: optional_safe_text(raw.codec_tag_string.as_ref())?, + profile_display: optional_safe_text(raw.profile.as_ref())?, + start_micros: parse_signed_micros(raw.start_time.as_deref()), + duration_micros: parse_unsigned_micros(raw.duration.as_deref()), + stream_time_base: parse_ratio(raw.time_base.as_deref()), + sample_rate: parse_u32_text(raw.sample_rate.as_deref()), + channels: raw.channels, + channel_layout: optional_safe_text(raw.channel_layout.as_ref())?, + bitrate: parse_u64_text(raw.bit_rate.as_deref()), + language, + disposition, + })), + Some("subtitle") => Ok(MediaStreamDescriptor::Subtitle(SubtitleStreamDescriptor { + index: raw.index, + codec_display: SafeProbeText::parse(raw.codec_name.as_deref(), "unknown")?, + codec_tag: optional_safe_text(raw.codec_tag_string.as_ref())?, + start_micros: parse_signed_micros(raw.start_time.as_deref()), + duration_micros: parse_unsigned_micros(raw.duration.as_deref()), + stream_time_base: parse_ratio(raw.time_base.as_deref()), + language, + disposition, + })), + _ => Ok(MediaStreamDescriptor::Other { + index: raw.index, + track_display: SafeProbeText::parse(raw.codec_type.as_deref(), "other")?, + codec_display: SafeProbeText::parse(raw.codec_name.as_deref(), "unknown")?, + codec_tag: optional_safe_text(raw.codec_tag_string.as_ref())?, + start_micros: parse_signed_micros(raw.start_time.as_deref()), + duration_micros: parse_unsigned_micros(raw.duration.as_deref()), + stream_time_base: parse_ratio(raw.time_base.as_deref()), + }), + } +} + +fn parse_chapter(raw: RawChapter) -> Result { + if raw.tags.len() > 64 { + return Err(ProbeError::new(ProbeErrorCode::LimitExceeded)); + } + Ok(ChapterDescriptor { + id: raw.id, + start_micros: parse_signed_micros(raw.start_time.as_deref()), + end_micros: parse_signed_micros(raw.end_time.as_deref()), + title: optional_safe_text(raw.tags.get("title"))?, + }) +} + +fn select_stream(streams: &[MediaStreamDescriptor], video: bool) -> Option { + let matching = |stream: &MediaStreamDescriptor| match (video, stream) { + (true, MediaStreamDescriptor::Video(stream)) if !stream.disposition.attached_picture => { + Some((stream.index, stream.disposition.default)) + } + (false, MediaStreamDescriptor::Audio(stream)) => { + Some((stream.index, stream.disposition.default)) + } + _ => None, + }; + streams + .iter() + .filter_map(matching) + .find(|(_, is_default)| *is_default) + .or_else(|| streams.iter().filter_map(matching).next()) + .map(|(index, _)| index) +} + +fn stream_index(stream: &MediaStreamDescriptor) -> u32 { + match stream { + MediaStreamDescriptor::Video(stream) => stream.index, + MediaStreamDescriptor::Audio(stream) => stream.index, + MediaStreamDescriptor::Subtitle(stream) => stream.index, + MediaStreamDescriptor::Other { index, .. } => *index, + } +} + +fn parse_disposition(raw: &HashMap) -> StreamDisposition { + let enabled = |name: &str| raw.get(name).copied().unwrap_or_default() == 1; + StreamDisposition { + default: enabled("default"), + forced: enabled("forced"), + hearing_impaired: enabled("hearing_impaired"), + visual_impaired: enabled("visual_impaired"), + attached_picture: enabled("attached_pic"), + } +} + +fn parse_side_data(values: &[serde_json::Value]) -> Result<(Option, HdrMetadata), ProbeError> { + let mut rotation = None; + let mut hdr = HdrMetadata::default(); + for value in values { + let Some(object) = value.as_object() else { + continue; + }; + let kind = object + .get("side_data_type") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if let Some(raw_rotation) = object.get("rotation").and_then(serde_json::Value::as_i64) { + let parsed = i16::try_from(raw_rotation) + .ok() + .filter(|value| (-360..=360).contains(value)); + if rotation.is_some() && parsed != rotation { + return Err(ProbeError::new(ProbeErrorCode::MalformedOutput)); + } + rotation = parsed.or(rotation); + } + match kind { + "Mastering display metadata" => { + if hdr.mastering_display.is_some() { + return Err(ProbeError::new(ProbeErrorCode::MalformedOutput)); + } + let rational = |name: &str| { + object + .get(name) + .and_then(serde_json::Value::as_str) + .and_then(|value| parse_ratio(Some(value))) + }; + hdr.mastering_display = Some(MasteringDisplayMetadata { + red_x: rational("red_x"), + red_y: rational("red_y"), + green_x: rational("green_x"), + green_y: rational("green_y"), + blue_x: rational("blue_x"), + blue_y: rational("blue_y"), + white_point_x: rational("white_point_x"), + white_point_y: rational("white_point_y"), + min_luminance: rational("min_luminance"), + max_luminance: rational("max_luminance"), + }); + } + "Content light level metadata" => { + if hdr.content_light.is_some() { + return Err(ProbeError::new(ProbeErrorCode::MalformedOutput)); + } + hdr.content_light = Some(ContentLightMetadata { + max_content: json_u32(object.get("max_content")), + max_average: json_u32(object.get("max_average")), + }); + } + "DOVI configuration record" | "Dolby Vision configuration record" => { + if hdr.dolby_vision.is_some() { + return Err(ProbeError::new(ProbeErrorCode::MalformedOutput)); + } + hdr.dolby_vision = Some(DolbyVisionMetadata { + profile: json_u8(object.get("dv_profile")), + level: json_u8(object.get("dv_level")), + rpu_present: json_bool_flag(object.get("rpu_present_flag")), + enhancement_layer_present: json_bool_flag(object.get("el_present_flag")), + base_layer_present: json_bool_flag(object.get("bl_present_flag")), + }); + } + _ => {} + } + } + Ok((rotation, hdr)) +} + +fn json_u32(value: Option<&serde_json::Value>) -> Option { + value + .and_then(serde_json::Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) +} +fn json_u8(value: Option<&serde_json::Value>) -> Option { + value + .and_then(serde_json::Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) +} +fn json_bool_flag(value: Option<&serde_json::Value>) -> Option { + value + .and_then(serde_json::Value::as_i64) + .and_then(|value| match value { + 0 => Some(false), + 1 => Some(true), + _ => None, + }) +} +fn optional_safe_text(value: Option<&String>) -> Result, ProbeError> { + value + .filter(|value| !value.is_empty()) + .map(|value| SafeProbeText::parse(Some(value), "unknown")) + .transpose() +} +fn parse_u64_text(value: Option<&str>) -> Option { + value?.parse().ok() +} +fn parse_u32_text(value: Option<&str>) -> Option { + value?.parse().ok() +} +fn parse_u8_text(value: Option<&str>) -> Option { + value?.parse().ok() +} +fn parse_signed_micros(value: Option<&str>) -> Option { + let seconds = value?.parse::().ok()?; + if !seconds.is_finite() { + return None; + } + let micros = seconds * 1_000_000.0; + (micros >= i64::MIN as f64 && micros <= i64::MAX as f64).then(|| micros.round() as i64) +} +fn parse_unsigned_micros(value: Option<&str>) -> Option { + let seconds = value?.parse::().ok()?; + if !seconds.is_finite() || seconds < 0.0 { + return None; + } + let micros = seconds * 1_000_000.0; + (micros <= u64::MAX as f64).then(|| micros.round() as u64) +} +fn parse_rate(value: Option<&str>) -> Option { + let ratio = parse_ratio(value)?; + let numerator = u32::try_from(ratio.numerator).ok()?; + let denominator = u32::try_from(ratio.denominator) + .ok() + .and_then(NonZeroU32::new)?; + RationalRate::new(numerator, denominator).ok() +} +fn parse_ratio(value: Option<&str>) -> Option { + let value = value?; + let (numerator, denominator) = value.split_once('/').or_else(|| value.split_once(':'))?; + let mut numerator = numerator.parse::().ok()?; + let mut denominator = denominator.parse::().ok()?; + if denominator == 0 || numerator == 0 { + return None; + } + if denominator < 0 { + numerator = numerator.checked_neg()?; + denominator = denominator.checked_neg()?; + } + let denominator = u64::try_from(denominator).ok()?; + let divisor = gcd_u64(numerator.unsigned_abs(), denominator); + Some(ProbeRational { + numerator: numerator / i64::try_from(divisor).ok()?, + denominator: denominator / divisor, + }) +} +fn gcd_u64(mut left: u64, mut right: u64) -> u64 { + while right != 0 { + let remainder = left % right; + left = right; + right = remainder; + } + left.max(1) +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct ProbeCacheKey { + source_policy: super::source::SourceProtocolPolicy, + source_version: String, +} + +struct CacheEntry { + value: Arc, + expires_at: Instant, + weight: usize, +} + +struct ProbeFlight { + notify: Notify, + result: Mutex, ProbeError>>>, +} + +#[derive(Default)] +struct ProbeCacheState { + entries: HashMap, + lru: VecDeque, + in_flight: HashMap>, + weight: usize, +} + +pub(crate) struct ProbeCache { + state: Mutex, + ttl: Duration, + max_entries: usize, + max_weight: usize, + max_in_flight: usize, +} + +struct FlightLeaderGuard<'a> { + cache: &'a ProbeCache, + key: ProbeCacheKey, + flight: Arc, + completed: bool, +} + +impl Drop for FlightLeaderGuard<'_> { + fn drop(&mut self) { + if self.completed { + return; + } + let cancelled = Err(ProbeError::new(ProbeErrorCode::Cancelled)); + let mut result = self + .flight + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if result.is_none() { + *result = Some(cancelled); + } + drop(result); + let mut state = self + .cache + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if state + .in_flight + .get(&self.key) + .is_some_and(|current| Arc::ptr_eq(current, &self.flight)) + { + state.in_flight.remove(&self.key); + } + drop(state); + self.flight.notify.notify_waiters(); + } +} + +impl Default for ProbeCache { + fn default() -> Self { + Self { + state: Mutex::new(ProbeCacheState::default()), + ttl: CACHE_TTL, + max_entries: CACHE_MAX_ENTRIES, + max_weight: CACHE_MAX_WEIGHT, + max_in_flight: CACHE_MAX_IN_FLIGHT, + } + } +} + +impl ProbeCache { + #[cfg(test)] + fn with_limits( + ttl: Duration, + max_entries: usize, + max_weight: usize, + max_in_flight: usize, + ) -> Self { + Self { + state: Mutex::new(ProbeCacheState::default()), + ttl, + max_entries, + max_weight, + max_in_flight, + } + } + + async fn get_or_probe( + &self, + key: ProbeCacheKey, + probe: F, + ) -> Result, ProbeError> + where + F: FnOnce() -> Fut, + Fut: std::future::Future, ProbeError>>, + { + let (flight, leader) = { + let now = Instant::now(); + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(value) = state + .entries + .get(&key) + .filter(|entry| entry.expires_at > now) + .map(|entry| entry.value.clone()) + { + state.lru.retain(|candidate| candidate != &key); + state.lru.push_back(key.clone()); + return Ok(value); + } + remove_cache_entry(&mut state, &key); + if let Some(flight) = state.in_flight.get(&key) { + (flight.clone(), false) + } else { + if state.in_flight.len() >= self.max_in_flight { + return Err(ProbeError::new(ProbeErrorCode::CapacityExceeded)); + } + let flight = Arc::new(ProbeFlight { + notify: Notify::new(), + result: Mutex::new(None), + }); + state.in_flight.insert(key.clone(), flight.clone()); + (flight, true) + } + }; + if !leader { + loop { + let notified = flight.notify.notified(); + if let Some(result) = flight + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + { + return result; + } + notified.await; + } + } + + let mut leader_guard = FlightLeaderGuard { + cache: self, + key: key.clone(), + flight: flight.clone(), + completed: false, + }; + let result = probe().await; + { + let mut slot = flight + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *slot = Some(result.clone()); + } + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.in_flight.remove(&key); + if let Ok(value) = &result { + let weight = value.estimated_weight().min(self.max_weight); + state.weight = state.weight.saturating_add(weight); + state.lru.push_back(key.clone()); + state.entries.insert( + key, + CacheEntry { + value: value.clone(), + expires_at: Instant::now() + self.ttl, + weight, + }, + ); + while state.entries.len() > self.max_entries || state.weight > self.max_weight { + let Some(oldest) = state.lru.pop_front() else { + break; + }; + remove_cache_entry(&mut state, &oldest); + } + } + drop(state); + flight.notify.notify_waiters(); + leader_guard.completed = true; + result + } +} + +fn remove_cache_entry(state: &mut ProbeCacheState, key: &ProbeCacheKey) { + if let Some(entry) = state.entries.remove(key) { + state.weight = state.weight.saturating_sub(entry.weight); + } + state.lru.retain(|candidate| candidate != key); +} + +pub async fn probe_media( + service: &TranscodingService, + source: &ValidatedMediaSource, +) -> Result, ProbeError> { + let key = ProbeCacheKey { + source_policy: source.protocol_policy(), + source_version: source.version().to_owned(), + }; + let document = service + .probe_cache() + .get_or_probe(key, || async { + run_probe(service, source).await.map(Arc::new) + }) + .await?; + Ok(Arc::new(MediaDescriptor::from_probe( + source.clone(), + document.as_ref().clone(), + ))) +} + +async fn run_probe( + service: &TranscodingService, + source: &ValidatedMediaSource, +) -> Result { + let session = service + .runtime_for_session() + .await + .map_err(|_| ProbeError::new(ProbeErrorCode::RuntimeUnavailable))?; + let input = source.input_argument().map_err(map_source_error)?; + let command = probe_command(source, input); + let cancellation = CancellationToken::new(); + let process = + session.run_bounded_with_cancellation(RuntimeExecutable::Ffprobe, command, &cancellation); + tokio::pin!(process); + let watchdog = watch_probe_activity(source.subscribe_activity()); + tokio::pin!(watchdog); + let output = tokio::select! { + output = &mut process => output.map_err(map_runtime_error)?, + reason = &mut watchdog => { + cancellation.cancel(); + let _ = process.await; + return Err(reason); + } + }; + if !output.status.success() { + return Err(ProbeError::new(ProbeErrorCode::NonZeroExit)); + } + parse_probe_document(&output.stdout) +} + +fn probe_command(source: &ValidatedMediaSource, input: OsString) -> RuntimeCommand { + const ENTRIES: &str = "format=format_name,duration,start_time,bit_rate:stream=index,codec_type,codec_name,codec_tag_string,profile,level,start_time,duration,width,height,sample_aspect_ratio,display_aspect_ratio,pix_fmt,bits_per_raw_sample,bits_per_sample,r_frame_rate,avg_frame_rate,time_base,codec_time_base,field_order,color_range,color_space,color_transfer,color_primaries,bit_rate,sample_rate,channels,channel_layout:stream_tags=language,rotate:stream_disposition=default,forced,hearing_impaired,visual_impaired,attached_pic:side_data=side_data_type,rotation,red_x,red_y,green_x,green_y,blue_x,blue_y,white_point_x,white_point_y,min_luminance,max_luminance,max_content,max_average,dv_profile,dv_level,rpu_present_flag,el_present_flag,bl_present_flag:chapter=id,start_time,end_time:chapter_tags=title"; + RuntimeCommand::new( + vec![ + "-v".into(), + "error".into(), + "-protocol_whitelist".into(), + source.ffmpeg_protocol_allowlist().into(), + "-of".into(), + "json".into(), + "-show_format".into(), + "-show_streams".into(), + "-show_chapters".into(), + "-show_entries".into(), + ENTRIES.into(), + input, + ], + StdoutPolicy::Capture { + byte_limit: MAX_PROBE_STDOUT_BYTES, + }, + MAX_PROBE_STDERR_BYTES, + PROBE_HARD_DEADLINE, + ) +} + +async fn watch_probe_activity( + mut activity: tokio::sync::watch::Receiver, +) -> ProbeError { + let started = Instant::now(); + let hard_deadline = started + PROBE_HARD_DEADLINE; + let mut last = activity.borrow_and_update().clone(); + let mut starved = is_confirmed_starvation(&last); + let mut inactivity_remaining = PROBE_INACTIVITY; + let mut inactivity_started = started; + let mut starvation_remaining = PROBE_STARVATION_DEFAULT; + let mut starvation_started = started; + loop { + let now = Instant::now(); + if now >= hard_deadline { + return ProbeError::new(ProbeErrorCode::OverallDeadline); + } + let deadline = if starved { + (starvation_started + starvation_remaining).min(hard_deadline) + } else { + (inactivity_started + inactivity_remaining).min(hard_deadline) + }; + tokio::select! { + _ = tokio::time::sleep_until(deadline) => { + return ProbeError::new(if deadline == hard_deadline { ProbeErrorCode::OverallDeadline } else if starved { ProbeErrorCode::SourceStarvation } else { ProbeErrorCode::Inactivity }); + } + changed = activity.changed() => { + if changed.is_err() { + tokio::time::sleep_until(deadline).await; + return ProbeError::new(if deadline == hard_deadline { + ProbeErrorCode::OverallDeadline + } else if starved { + ProbeErrorCode::SourceStarvation + } else { + ProbeErrorCode::Inactivity + }); + } + let now = Instant::now(); + let next = activity.borrow_and_update().clone(); + let next_starved = is_confirmed_starvation(&next); + if starved { + starvation_remaining = starvation_remaining.saturating_sub(now.saturating_duration_since(starvation_started)); + } else { + inactivity_remaining = inactivity_remaining.saturating_sub(now.saturating_duration_since(inactivity_started)); + } + if next.delivered_bytes_total > last.delivered_bytes_total { + inactivity_remaining = PROBE_INACTIVITY; + } + starved = next_starved; + if starved { starvation_started = now; } else { inactivity_started = now; } + last = next; + } + } + } +} + +fn is_confirmed_starvation(snapshot: &SourceActivitySnapshot) -> bool { + snapshot.active_requests > 0 && snapshot.all_active_requests_piece_blocked +} + +fn map_source_error(_: SourceError) -> ProbeError { + ProbeError::new(ProbeErrorCode::SourceInvalid) +} +fn map_runtime_error(error: RuntimeCommandError) -> ProbeError { + match error { + RuntimeCommandError::Runtime(_) => ProbeError::new(ProbeErrorCode::RuntimeUnavailable), + RuntimeCommandError::Process(error) => { + ProbeError::new(map_process_error_code(error.code())) + } + } +} + +fn map_process_error_code(code: ProcessErrorCode) -> ProbeErrorCode { + match code { + ProcessErrorCode::Cancelled => ProbeErrorCode::Cancelled, + ProcessErrorCode::DeadlineExceeded => ProbeErrorCode::OverallDeadline, + ProcessErrorCode::StdoutLimitExceeded => ProbeErrorCode::OutputTooLarge, + _ => ProbeErrorCode::ProcessFailure, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[test] + fn supervisor_deadline_is_the_probe_hard_overall_deadline() { + assert_eq!( + map_process_error_code(ProcessErrorCode::DeadlineExceeded), + ProbeErrorCode::OverallDeadline + ); + } + + #[tokio::test(start_paused = true)] + async fn watchdog_ignores_non_source_output_and_pauses_only_for_all_blocked_source_requests() { + let (sender, receiver) = tokio::sync::watch::channel(SourceActivitySnapshot::default()); + let watchdog = tokio::spawn(watch_probe_activity(receiver)); + tokio::time::advance(Duration::from_secs(29)).await; + assert!(!watchdog.is_finished()); + sender.send_modify(|snapshot| { + snapshot.sequence += 1; + snapshot.active_requests = 1; + snapshot.all_active_requests_piece_blocked = true; + }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(9 * 60)).await; + assert!(!watchdog.is_finished()); + sender.send_modify(|snapshot| { + snapshot.sequence += 1; + snapshot.delivered_bytes_total += 1; + snapshot.all_active_requests_piece_blocked = false; + }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(29)).await; + assert!(!watchdog.is_finished()); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(watchdog.await.unwrap().code(), ProbeErrorCode::Inactivity); + } + + #[tokio::test(start_paused = true)] + async fn watchdog_enforces_default_starvation_and_hard_overall_deadlines() { + let (sender, receiver) = tokio::sync::watch::channel(SourceActivitySnapshot::default()); + sender.send_modify(|snapshot| { + snapshot.active_requests = 1; + snapshot.all_active_requests_piece_blocked = true; + }); + let watchdog = tokio::spawn(watch_probe_activity(receiver)); + tokio::time::advance(PROBE_STARVATION_DEFAULT).await; + assert_eq!( + watchdog.await.unwrap().code(), + ProbeErrorCode::SourceStarvation + ); + + let (sender, receiver) = tokio::sync::watch::channel(SourceActivitySnapshot::default()); + let hard_watchdog = tokio::spawn(watch_probe_activity(receiver)); + tokio::task::yield_now().await; + for delivered in 1..=62 { + tokio::time::advance(Duration::from_secs(29)).await; + sender.send_modify(|snapshot| { + snapshot.sequence += 1; + snapshot.active_requests = 1; + snapshot.delivered_bytes_total = delivered; + snapshot.all_active_requests_piece_blocked = false; + }); + tokio::task::yield_now().await; + assert!(!hard_watchdog.is_finished()); + } + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!( + hard_watchdog.await.unwrap().code(), + ProbeErrorCode::OverallDeadline + ); + } + + #[tokio::test] + async fn cache_single_flights_and_versions_invalidate() { + let cache = Arc::new(ProbeCache::default()); + let calls = Arc::new(AtomicUsize::new(0)); + let document = parse_probe_document(br#"{"format":{},"streams":[]}"#).unwrap(); + let value = Arc::new(document); + let key = ProbeCacheKey { + source_policy: super::super::source::SourceProtocolPolicy::CompletedFile, + source_version: "v1".into(), + }; + let mut tasks = Vec::new(); + for _ in 0..8 { + let cache = cache.clone(); + let calls = calls.clone(); + let value = value.clone(); + let key = key.clone(); + tasks.push(tokio::spawn(async move { + cache + .get_or_probe(key, || async move { + calls.fetch_add(1, Ordering::SeqCst); + tokio::task::yield_now().await; + Ok(value) + }) + .await + .unwrap() + })); + } + for task in tasks { + task.await.unwrap(); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let version_two = ProbeCacheKey { + source_version: "v2".into(), + ..key + }; + cache + .get_or_probe(version_two, || async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(value) + }) + .await + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn cancelled_cache_leader_does_not_strand_followers() { + let cache = Arc::new(ProbeCache::default()); + let key = ProbeCacheKey { + source_policy: super::super::source::SourceProtocolPolicy::CompletedFile, + source_version: "v1".into(), + }; + let started = Arc::new(Notify::new()); + let leader = { + let cache = cache.clone(); + let key = key.clone(); + let started = started.clone(); + tokio::spawn(async move { + cache + .get_or_probe(key, || async move { + started.notify_one(); + std::future::pending().await + }) + .await + }) + }; + started.notified().await; + leader.abort(); + let _ = leader.await; + + let follower = cache.get_or_probe(key, || async { + Err(ProbeError::new(ProbeErrorCode::ProcessFailure)) + }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), follower) + .await + .expect("follower must not wait on an abandoned flight") + .unwrap_err() + .code(), + ProbeErrorCode::ProcessFailure + ); + } + + #[tokio::test(start_paused = true)] + async fn cache_ttl_expiry_reprobes() { + let cache = ProbeCache::default(); + let calls = AtomicUsize::new(0); + let document = parse_probe_document(br#"{"format":{},"streams":[]}"#).unwrap(); + let value = Arc::new(document); + let key = ProbeCacheKey { + source_policy: super::super::source::SourceProtocolPolicy::CompletedFile, + source_version: "v1".into(), + }; + + for _ in 0..2 { + cache + .get_or_probe(key.clone(), || async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(value.clone()) + }) + .await + .unwrap(); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + tokio::time::advance(CACHE_TTL + Duration::from_secs(1)).await; + cache + .get_or_probe(key, || async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(value) + }) + .await + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn cache_lru_enforces_count_and_weight_bounds() { + let document = Arc::new( + parse_probe_document(include_bytes!( + "../../tests/fixtures/ffprobe/compatibility.json" + )) + .unwrap(), + ); + let weight = document.estimated_weight(); + let cache = ProbeCache::with_limits(CACHE_TTL, 2, usize::MAX, CACHE_MAX_IN_FLIGHT); + let key = |version: &str| ProbeCacheKey { + source_policy: super::super::source::SourceProtocolPolicy::CompletedFile, + source_version: version.to_owned(), + }; + for version in ["a", "b"] { + cache + .get_or_probe(key(version), || async { Ok(document.clone()) }) + .await + .unwrap(); + } + cache + .get_or_probe(key("a"), || async { unreachable!("cache hit") }) + .await + .unwrap(); + cache + .get_or_probe(key("c"), || async { Ok(document.clone()) }) + .await + .unwrap(); + { + let state = cache + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(state.entries.contains_key(&key("a"))); + assert!(!state.entries.contains_key(&key("b"))); + assert!(state.entries.contains_key(&key("c"))); + } + + let weighted = ProbeCache::with_limits(CACHE_TTL, 10, weight * 2 - 1, CACHE_MAX_IN_FLIGHT); + for version in ["a", "b"] { + weighted + .get_or_probe(key(version), || async { Ok(document.clone()) }) + .await + .unwrap(); + } + let state = weighted + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(state.entries.len(), 1); + assert!(state.weight < weight * 2); + } + + #[test] + fn ffprobe_recipe_is_closed_and_uses_only_the_source_protocol_policy() { + let local = ValidatedMediaSource::completed_file("local-source").unwrap(); + let local_command = probe_command(&local, OsString::from(r"C:\media\fixture.mkv")); + let local_args = local_command + .args() + .iter() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + local_args + .windows(2) + .find(|pair| pair[0] == "-protocol_whitelist") + .map(|pair| pair[1].as_str()), + Some("file,pipe") + ); + assert_eq!(local_args.last().unwrap(), r"C:\media\fixture.mkv"); + assert!( + local_args + .iter() + .any(|argument| argument == "-show_entries") + ); + assert!(!local_args.iter().any(|argument| argument.contains("https"))); + + let engine = ValidatedMediaSource::engine_loopback("engine-source").unwrap(); + let engine_command = probe_command(&engine, OsString::from("http://127.0.0.1/source")); + let engine_args = engine_command + .args() + .iter() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + engine_args + .windows(2) + .find(|pair| pair[0] == "-protocol_whitelist") + .map(|pair| pair[1].as_str()), + Some("http,tcp") + ); + } + + #[tokio::test] + async fn cache_bounds_distinct_in_flight_probes_without_blocking_same_key_followers() { + let cache = Arc::new(ProbeCache::with_limits(CACHE_TTL, 8, usize::MAX, 1)); + let started = Arc::new(Notify::new()); + let first_key = ProbeCacheKey { + source_policy: super::super::source::SourceProtocolPolicy::CompletedFile, + source_version: "first".into(), + }; + let leader = { + let cache = cache.clone(); + let started = started.clone(); + tokio::spawn(async move { + cache + .get_or_probe(first_key, || async move { + started.notify_one(); + std::future::pending().await + }) + .await + }) + }; + started.notified().await; + let second_key = ProbeCacheKey { + source_policy: super::super::source::SourceProtocolPolicy::CompletedFile, + source_version: "second".into(), + }; + let error = cache + .get_or_probe(second_key, || async { + unreachable!("must reject before work") + }) + .await + .unwrap_err(); + assert_eq!(error.code(), ProbeErrorCode::CapacityExceeded); + leader.abort(); + let _ = leader.await; + } +} diff --git a/server/src/transcoding/process.rs b/server/src/transcoding/process.rs index 78e6de8f..e9b2fd54 100644 --- a/server/src/transcoding/process.rs +++ b/server/src/transcoding/process.rs @@ -912,6 +912,10 @@ impl ProcessSupervisor { } } + #[allow( + dead_code, + reason = "the uncancelled wrapper remains the closed command boundary for later transcode sessions and is exercised by process lifecycle tests" + )] pub async fn run_bounded(&self, spec: ProcessSpec) -> Result { self.run_bounded_with_cancellation(spec, &CancellationToken::new()) .await diff --git a/server/src/transcoding/runtime.rs b/server/src/transcoding/runtime.rs index 870b5637..08d45228 100644 --- a/server/src/transcoding/runtime.rs +++ b/server/src/transcoding/runtime.rs @@ -1,4 +1,5 @@ use super::{ + probe::ProbeCache, process::{ BoundedOutput, ProcessError, ProcessErrorCode, ProcessSpec, ProcessSupervisor, StdinPolicy, StdoutPolicy, @@ -152,7 +153,7 @@ pub(crate) enum RuntimeExecutable { /// }; /// ``` #[allow(dead_code)] -#[derive(Clone, Debug)] +#[derive(Clone)] pub(crate) struct RuntimeCommand { args: Vec, stdout: StdoutPolicy, @@ -160,6 +161,27 @@ pub(crate) struct RuntimeCommand { wall_deadline: Duration, } +impl RuntimeCommand { + pub(super) fn new( + args: Vec, + stdout: StdoutPolicy, + stderr_byte_limit: usize, + wall_deadline: Duration, + ) -> Self { + Self { + args, + stdout, + stderr_byte_limit, + wall_deadline, + } + } + + #[cfg(test)] + pub(super) fn args(&self) -> &[OsString] { + &self.args + } +} + #[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum RuntimeCommandError { @@ -221,6 +243,17 @@ impl VerifiedRuntimeSession { &self, executable: RuntimeExecutable, command: RuntimeCommand, + ) -> Result { + let cancellation = tokio_util::sync::CancellationToken::new(); + self.run_bounded_with_cancellation(executable, command, &cancellation) + .await + } + + pub(super) async fn run_bounded_with_cancellation( + &self, + executable: RuntimeExecutable, + command: RuntimeCommand, + cancellation: &tokio_util::sync::CancellationToken, ) -> Result { let execution_lease = open_pair_lease( self.runtime.lease.root.clone(), @@ -253,16 +286,19 @@ impl VerifiedRuntimeSession { bound_execution_path(&execution_lease.lease._root_file, &execution_lease.root) .map_err(|_| RuntimeCommandError::Runtime(RuntimeError::RuntimeChanged))?; self.supervisor - .run_bounded(ProcessSpec { - executable: executable_path, - args: command.args, - environment: minimal_runtime_environment(¤t_dir), - current_dir, - stdin: StdinPolicy::Null, - stdout: command.stdout, - stderr_byte_limit: command.stderr_byte_limit, - wall_deadline: command.wall_deadline, - }) + .run_bounded_with_cancellation( + ProcessSpec { + executable: executable_path, + args: command.args, + environment: minimal_runtime_environment(¤t_dir), + current_dir, + stdin: StdinPolicy::Null, + stdout: command.stdout, + stderr_byte_limit: command.stderr_byte_limit, + wall_deadline: command.wall_deadline, + }, + cancellation, + ) .await .map_err(RuntimeCommandError::Process) } @@ -308,6 +344,7 @@ impl RuntimeKind { pub struct TranscodingService { supervisor: Arc, state: tokio::sync::RwLock, + probe_cache: ProbeCache, } enum ServiceState { @@ -323,6 +360,7 @@ impl TranscodingService { Self { supervisor, state: tokio::sync::RwLock::new(ServiceState::Unavailable), + probe_cache: ProbeCache::default(), } } @@ -334,6 +372,7 @@ impl TranscodingService { Self { supervisor, state: tokio::sync::RwLock::new(ServiceState::Resolved { config, runtime }), + probe_cache: ProbeCache::default(), } } @@ -347,6 +386,10 @@ impl TranscodingService { } } + pub(super) fn probe_cache(&self) -> &ProbeCache { + &self.probe_cache + } + pub async fn status(&self) -> RuntimeStatus { match &*self.state.read().await { ServiceState::Unavailable => RuntimeStatus::Unavailable, diff --git a/server/src/transcoding/source.rs b/server/src/transcoding/source.rs new file mode 100644 index 00000000..cfed6b23 --- /dev/null +++ b/server/src/transcoding/source.rs @@ -0,0 +1,2373 @@ +use axum::{ + body::Body, + extract::{ConnectInfo, Extension, RawQuery, State}, + http::{HeaderMap, HeaderValue, Method, Response, StatusCode, header}, +}; +use bytes::Bytes; +use enginefs::backend::{TorrentHandle, priorities::PlaybackIntent}; +use futures_util::{Stream, StreamExt, stream}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + ffi::OsString, + fmt, + hash::{Hash, Hasher}, + ops::Range, + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; +use subtle::ConstantTimeEq; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncSeekExt}, + sync::{OwnedSemaphorePermit, Semaphore, watch}, +}; +use tokio_util::sync::CancellationToken; + +const CAPABILITY_BYTES: usize = 32; +const CAPABILITY_HEX_BYTES: usize = CAPABILITY_BYTES * 2; +const MAX_LIVE_CAPABILITIES: usize = 256; +const MAX_CONCURRENT_REQUESTS_PER_CAPABILITY: usize = 8; +const MAX_CAPABILITY_LIFETIME: Duration = Duration::from_secs(30 * 60); + +/// Monotonic, source-owned evidence used to distinguish a stalled probe from +/// a torrent request that is demonstrably waiting for unavailable pieces. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct SourceActivitySnapshot { + pub sequence: u64, + pub delivered_bytes_total: u64, + pub active_requests: u32, + pub waiting_for_pieces: Vec>, + pub all_active_requests_piece_blocked: bool, +} + +/// Closed protocol policy derived from the trusted source variant. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum SourceProtocolPolicy { + CompletedFile, + EngineLoopback, + ApprovedRemote, + SyntheticFixture, +} + +impl SourceProtocolPolicy { + pub const fn ffmpeg_allowlist(self) -> &'static str { + match self { + Self::CompletedFile | Self::SyntheticFixture => "file,pipe", + Self::EngineLoopback | Self::ApprovedRemote => "http,tcp", + } + } +} + +/// A media source issued by the server-owned source broker. The variants are +/// public for exhaustive planning, but all fields and production constructors +/// remain sealed. +/// +/// Route text cannot be deserialized into a trusted source: +/// +/// ```compile_fail +/// use stream_server::transcoding::ValidatedMediaSource; +/// let _: ValidatedMediaSource = serde_json::from_str( +/// r#"{"approvedRemote":{"id":"route-text"}}"#, +/// ).unwrap(); +/// ``` +/// +/// Variant fields are private, so callers cannot manufacture capabilities: +/// +/// ```compile_fail +/// use stream_server::transcoding::{EngineSource, ValidatedMediaSource}; +/// let source = EngineSource { id: "route-text".into() }; +/// let _ = ValidatedMediaSource::EngineLoopback(source); +/// ``` +#[derive(Clone, Eq, PartialEq, Hash, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ValidatedMediaSource { + CompletedFile(CompletedFileSource), + EngineLoopback(EngineSource), + ApprovedRemote(RemoteSourceHandle), + SyntheticFixture(FixtureSource), +} + +impl ValidatedMediaSource { + pub fn id(&self) -> &str { + match self { + Self::CompletedFile(source) => source.id(), + Self::EngineLoopback(source) => source.id(), + Self::ApprovedRemote(source) => source.id(), + Self::SyntheticFixture(source) => source.id(), + } + } + + pub fn version(&self) -> &str { + match self { + Self::CompletedFile(source) => source.version(), + Self::EngineLoopback(source) => source.version(), + Self::ApprovedRemote(source) => source.version(), + Self::SyntheticFixture(source) => source.version(), + } + } + + pub const fn protocol_policy(&self) -> SourceProtocolPolicy { + match self { + Self::CompletedFile(_) => SourceProtocolPolicy::CompletedFile, + Self::EngineLoopback(_) => SourceProtocolPolicy::EngineLoopback, + Self::ApprovedRemote(_) => SourceProtocolPolicy::ApprovedRemote, + Self::SyntheticFixture(_) => SourceProtocolPolicy::SyntheticFixture, + } + } + + pub fn ffmpeg_protocol_allowlist(&self) -> &'static str { + self.protocol_policy().ffmpeg_allowlist() + } + + pub(crate) fn input_argument(&self) -> Result { + match self { + Self::CompletedFile(source) => source.input_argument(), + Self::EngineLoopback(source) => source.input_argument(), + Self::ApprovedRemote(source) => source.input_argument(), + Self::SyntheticFixture(source) => source.input_argument(), + } + } + + pub fn subscribe_activity(&self) -> watch::Receiver { + match self { + Self::EngineLoopback(source) => source.subscribe_activity(), + Self::ApprovedRemote(source) => source.subscribe_activity(), + Self::CompletedFile(_) | Self::SyntheticFixture(_) => { + let (_, receiver) = watch::channel(SourceActivitySnapshot::default()); + receiver + } + } + } + + #[cfg(test)] + pub(super) fn completed_file(id: impl Into) -> Result { + Ok(Self::CompletedFile(CompletedFileSource::stub(id)?)) + } + + #[cfg(test)] + pub(super) fn engine_loopback(id: impl Into) -> Result { + Ok(Self::EngineLoopback(EngineSource::stub(id)?)) + } + + #[cfg(test)] + pub(super) fn approved_remote(id: impl Into) -> Result { + Ok(Self::ApprovedRemote(RemoteSourceHandle::stub(id)?)) + } + + #[cfg(test)] + pub(super) fn synthetic_fixture(id: impl Into) -> Result { + Ok(Self::SyntheticFixture(FixtureSource::stub(id)?)) + } + + #[cfg(test)] + pub(super) fn synthetic_fixture_path( + id: impl Into, + version: impl Into, + path: PathBuf, + ) -> Result { + Ok(Self::SyntheticFixture(FixtureSource { + id: SourceId::new(id)?, + version: version.into(), + path: Some(path), + })) + } +} + +impl fmt::Debug for ValidatedMediaSource { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CompletedFile(source) => formatter + .debug_tuple("CompletedFile") + .field(source) + .finish(), + Self::EngineLoopback(source) => formatter + .debug_tuple("EngineLoopback") + .field(source) + .finish(), + Self::ApprovedRemote(source) => formatter + .debug_tuple("ApprovedRemote") + .field(source) + .finish(), + Self::SyntheticFixture(source) => formatter + .debug_tuple("SyntheticFixture") + .field(source) + .finish(), + } + } +} + +#[derive(Clone, Eq, PartialEq, Hash, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CompletedFileSource { + id: SourceId, + #[serde(skip)] + version: String, + #[serde(skip)] + canonical_path: Option, + #[serde(skip)] + identity: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct CompletedFileIdentity { + length: u64, + modified_nanos: Option, + #[cfg(windows)] + volume: u64, + #[cfg(windows)] + file: u64, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, +} + +impl CompletedFileSource { + pub fn id(&self) -> &str { + self.id.as_str() + } + + pub fn version(&self) -> &str { + &self.version + } + + fn input_argument(&self) -> Result { + let path = self + .canonical_path + .as_ref() + .ok_or(SourceError::InvalidSource)?; + let expected = self.identity.as_ref().ok_or(SourceError::InvalidSource)?; + let actual = CompletedFileIdentity::from_path(path)?; + if &actual != expected { + return Err(SourceError::InvalidSource); + } + Ok(path.as_os_str().to_owned()) + } + + #[cfg(test)] + fn stub(id: impl Into) -> Result { + Ok(Self { + id: SourceId::new(id)?, + version: "test-version".to_owned(), + canonical_path: None, + identity: None, + }) + } +} + +impl fmt::Debug for CompletedFileSource { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CompletedFileSource") + .field("id", &self.id) + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +impl CompletedFileIdentity { + fn from_path(path: &std::path::Path) -> Result { + let file = std::fs::File::open(path).map_err(|_| SourceError::NotFound)?; + let metadata = file.metadata().map_err(|_| SourceError::Io)?; + if !metadata.is_file() { + return Err(SourceError::InvalidSource); + } + let modified_nanos = metadata + .modified() + .ok() + .and_then(|modified| modified.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_nanos()); + #[cfg(windows)] + let (volume, file_identity) = windows_file_identity(&file)?; + + Ok(Self { + length: metadata.len(), + modified_nanos, + #[cfg(windows)] + volume, + #[cfg(windows)] + file: file_identity, + #[cfg(unix)] + device: std::os::unix::fs::MetadataExt::dev(&metadata), + #[cfg(unix)] + inode: std::os::unix::fs::MetadataExt::ino(&metadata), + }) + } +} + +#[cfg(windows)] +fn windows_file_identity(file: &std::fs::File) -> Result<(u64, u64), SourceError> { + use std::os::windows::io::AsRawHandle; + use windows::Win32::{ + Foundation::HANDLE, + Storage::FileSystem::{BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle}, + }; + + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + unsafe { + GetFileInformationByHandle(HANDLE(file.as_raw_handle()), &raw mut information) + .map_err(|_| SourceError::Io)?; + } + Ok(( + information.dwVolumeSerialNumber as u64, + ((information.nFileIndexHigh as u64) << 32) | information.nFileIndexLow as u64, + )) +} + +#[derive(Clone, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct EngineSource { + id: SourceId, + #[serde(skip)] + version: String, + #[serde(skip)] + lease: Option>, + #[serde(skip)] + activity: Arc, +} + +impl EngineSource { + pub fn id(&self) -> &str { + self.id.as_str() + } + + pub fn version(&self) -> &str { + &self.version + } + + pub fn subscribe_activity(&self) -> watch::Receiver { + self.activity.subscribe() + } + + pub fn revoke(&self) { + if let Some(lease) = &self.lease { + lease.revoke(); + } + } + + fn input_argument(&self) -> Result { + self.lease + .as_ref() + .map(|lease| OsString::from(lease.input_url.as_str())) + .ok_or(SourceError::InvalidSource) + } + + #[cfg(test)] + fn stub(id: impl Into) -> Result { + Ok(Self { + id: SourceId::new(id)?, + version: "test-version".to_owned(), + lease: None, + activity: Arc::new(ActivityTracker::default()), + }) + } +} + +impl fmt::Debug for EngineSource { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("EngineSource") + .field("id", &self.id) + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +impl PartialEq for EngineSource { + fn eq(&self, other: &Self) -> bool { + self.id == other.id && self.version == other.version + } +} + +impl Eq for EngineSource {} + +impl Hash for EngineSource { + fn hash(&self, state: &mut H) { + self.id.hash(state); + self.version.hash(state); + } +} + +#[derive(Clone, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RemoteSourceHandle { + id: SourceId, + #[serde(skip)] + version: String, + #[serde(skip)] + lease: Option>, + #[serde(skip)] + activity: Arc, +} + +impl RemoteSourceHandle { + pub fn id(&self) -> &str { + self.id.as_str() + } + + pub fn version(&self) -> &str { + &self.version + } + + pub fn subscribe_activity(&self) -> watch::Receiver { + self.activity.subscribe() + } + + pub fn revoke(&self) { + if let Some(lease) = &self.lease { + lease.revoke(); + } + } + + fn input_argument(&self) -> Result { + self.lease + .as_ref() + .map(|lease| OsString::from(lease.input_url.as_str())) + .ok_or(SourceError::InvalidSource) + } + + #[cfg(test)] + fn stub(id: impl Into) -> Result { + Ok(Self { + id: SourceId::new(id)?, + version: "test-version".to_owned(), + lease: None, + activity: Arc::new(ActivityTracker::default()), + }) + } +} + +impl fmt::Debug for RemoteSourceHandle { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RemoteSourceHandle") + .field("id", &self.id) + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +impl PartialEq for RemoteSourceHandle { + fn eq(&self, other: &Self) -> bool { + self.id == other.id && self.version == other.version + } +} + +impl Eq for RemoteSourceHandle {} + +impl Hash for RemoteSourceHandle { + fn hash(&self, state: &mut H) { + self.id.hash(state); + self.version.hash(state); + } +} + +#[derive(Clone, Eq, PartialEq, Hash, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FixtureSource { + id: SourceId, + #[serde(skip)] + version: String, + #[serde(skip)] + path: Option, +} + +impl FixtureSource { + pub fn id(&self) -> &str { + self.id.as_str() + } + + pub fn version(&self) -> &str { + &self.version + } + + fn input_argument(&self) -> Result { + self.path + .as_ref() + .map(|path| path.as_os_str().to_owned()) + .ok_or(SourceError::InvalidSource) + } + + #[cfg(test)] + fn stub(id: impl Into) -> Result { + Ok(Self { + id: SourceId::new(id)?, + version: "test-version".to_owned(), + path: None, + }) + } +} + +impl fmt::Debug for FixtureSource { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FixtureSource") + .field("id", &self.id) + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, serde::Serialize)] +#[serde(transparent)] +struct SourceId(String); + +impl SourceId { + #[cfg(test)] + fn new(value: impl Into) -> Result { + let value = value.into(); + if !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + Ok(Self(value)) + } else { + Err(SourceError::InvalidSource) + } + } + + fn random() -> Result { + let mut bytes = [0_u8; CAPABILITY_BYTES]; + getrandom::fill(&mut bytes).map_err(|_| SourceError::Io)?; + Ok(Self(hex::encode(bytes))) + } + + fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum SourceError { + #[error("invalid source")] + InvalidSource, + #[error("source not found")] + NotFound, + #[error("invalid source capability")] + InvalidCapability, + #[error("source capability expired")] + Expired, + #[error("source capability revoked")] + Revoked, + #[error("source request capacity exceeded")] + RateLimited, + #[error("invalid byte range")] + InvalidRange, + #[error("source broker capacity exceeded")] + Capacity, + #[error("source I/O failed")] + Io, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct CapabilityScope { + source_id: String, + source_version: String, +} + +impl CapabilityScope { + #[cfg(test)] + fn fixture(source_id: &str, source_version: &str) -> Self { + Self { + source_id: source_id.to_owned(), + source_version: source_version.to_owned(), + } + } +} + +struct SecretCredential { + bytes: [u8; CAPABILITY_BYTES], + encoded: String, +} + +impl SecretCredential { + fn generate() -> Result { + let mut bytes = [0_u8; CAPABILITY_BYTES]; + getrandom::fill(&mut bytes).map_err(|_| SourceError::Io)?; + let encoded = hex::encode(bytes); + Ok(Self { bytes, encoded }) + } + + fn encoded(&self) -> &str { + &self.encoded + } +} + +impl fmt::Debug for SecretCredential { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretCredential([REDACTED])") + } +} + +struct CapabilityEntry { + credential: SecretCredential, + scope: CapabilityScope, + expires_at: tokio::time::Instant, + revoked: AtomicBool, + revocation: CancellationToken, + requests: Arc, + payload: CapabilityPayload, +} + +impl CapabilityEntry { + fn try_acquire(self: &Arc) -> Result { + self.requests + .clone() + .try_acquire_owned() + .map_err(|_| SourceError::RateLimited) + } + + fn revoke(&self) { + self.revoked.store(true, Ordering::Release); + self.revocation.cancel(); + } +} + +impl fmt::Debug for CapabilityEntry { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CapabilityEntry") + .field("source_id", &self.scope.source_id) + .field("source_version", &self.scope.source_version) + .field("revoked", &self.revoked.load(Ordering::Acquire)) + .finish_non_exhaustive() + } +} + +struct IssuedCapability { + credential: SecretCredential, + entry: Arc, +} + +impl IssuedCapability { + #[cfg(test)] + fn revoke(&self) { + self.entry.revoke(); + } + + fn into_lease( + self, + listener: std::net::SocketAddr, + ) -> Result, SourceError> { + let host = if listener.is_ipv6() { + "[::1]".to_owned() + } else { + "127.0.0.1".to_owned() + }; + let input_url = url::Url::parse(&format!( + "http://{host}:{}/_transcoding/source?cap={}", + listener.port(), + self.credential.encoded() + )) + .map_err(|_| SourceError::Io)?; + Ok(Arc::new(CapabilityLease { + input_url, + entry: self.entry, + })) + } +} + +struct CapabilityLease { + input_url: url::Url, + entry: Arc, +} + +impl CapabilityLease { + fn revoke(&self) { + self.entry.revoke(); + } +} + +impl fmt::Debug for CapabilityLease { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("CapabilityLease([REDACTED])") + } +} + +impl Drop for CapabilityLease { + fn drop(&mut self) { + self.revoke(); + } +} + +#[derive(Default)] +struct CapabilityRegistry { + entries: Mutex>>, +} + +impl CapabilityRegistry { + #[cfg(test)] + fn issue( + &self, + scope: CapabilityScope, + lifetime: Duration, + ) -> Result { + self.issue_with_payload(scope, lifetime, CapabilityPayload::Fixture) + } + + fn issue_with_payload( + &self, + scope: CapabilityScope, + lifetime: Duration, + payload: CapabilityPayload, + ) -> Result { + if lifetime.is_zero() || lifetime > MAX_CAPABILITY_LIFETIME { + return Err(SourceError::InvalidSource); + } + let credential = SecretCredential::generate()?; + let lookup_key = Sha256::digest(credential.bytes).into(); + let entry = Arc::new(CapabilityEntry { + credential: SecretCredential { + bytes: credential.bytes, + encoded: credential.encoded.clone(), + }, + scope, + expires_at: tokio::time::Instant::now() + lifetime, + revoked: AtomicBool::new(false), + revocation: CancellationToken::new(), + requests: Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS_PER_CAPABILITY)), + payload, + }); + let mut entries = self + .entries + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + entries.retain(|_, existing| { + !existing.revoked.load(Ordering::Acquire) + && tokio::time::Instant::now() < existing.expires_at + }); + if entries.len() >= MAX_LIVE_CAPABILITIES { + return Err(SourceError::Capacity); + } + if entries.contains_key(&lookup_key) { + return Err(SourceError::Io); + } + entries.insert(lookup_key, entry.clone()); + Ok(IssuedCapability { credential, entry }) + } + + fn lookup(&self, encoded: &str) -> Result, SourceError> { + if encoded.len() != CAPABILITY_HEX_BYTES { + return Err(SourceError::InvalidCapability); + } + let mut candidate = [0_u8; CAPABILITY_BYTES]; + hex::decode_to_slice(encoded, &mut candidate) + .map_err(|_| SourceError::InvalidCapability)?; + let lookup_key: [u8; CAPABILITY_BYTES] = Sha256::digest(candidate).into(); + let entry = self + .entries + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&lookup_key) + .cloned() + .ok_or(SourceError::InvalidCapability)?; + if !bool::from(candidate.ct_eq(&entry.credential.bytes)) { + return Err(SourceError::InvalidCapability); + } + if entry.revoked.load(Ordering::Acquire) { + return Err(SourceError::Revoked); + } + if tokio::time::Instant::now() >= entry.expires_at { + return Err(SourceError::Expired); + } + Ok(entry) + } +} + +#[derive(Clone)] +enum CapabilityPayload { + Engine(Arc), + Remote(Arc), + #[cfg(test)] + Fixture, +} + +struct EngineCapability { + info_hash: String, + file_index: usize, + intent: PlaybackIntent, + metadata: EngineFileMetadata, + provider: Arc, + activity: Arc, +} + +struct RemoteCapability { + target: url::Url, + runtime: Arc, + activity: Arc, +} + +impl fmt::Debug for RemoteCapability { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("RemoteCapability([REDACTED])") + } +} + +impl fmt::Debug for EngineCapability { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("EngineCapability") + .field("file_index", &self.file_index) + .field("intent", &self.intent) + .field("metadata", &self.metadata) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +struct EngineFileMetadata { + length: u64, + version: String, + content_type: &'static str, +} + +struct CompletedFileCandidate { + path: PathBuf, + allowed_roots: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PieceAvailability { + Available, + Unavailable, + Unknown, +} + +struct OpenedEngineSource { + reader: std::pin::Pin>, + _lifecycle: Box, +} + +#[async_trait::async_trait] +trait EngineSourceProvider: Send + Sync { + async fn describe( + &self, + info_hash: &str, + file_index: usize, + ) -> Result; + + async fn open( + &self, + info_hash: &str, + file_index: usize, + range: ByteRange, + intent: PlaybackIntent, + ) -> Result; + + async fn piece_availability( + &self, + info_hash: &str, + file_index: usize, + offset: u64, + intent: PlaybackIntent, + ) -> Result; + + async fn refresh(&self, info_hash: &str, file_index: usize) -> Result<(), SourceError>; + + async fn completed_file( + &self, + _info_hash: &str, + _file_index: usize, + ) -> Result, SourceError> { + Ok(None) + } +} + +pub struct SourceBroker { + registry: CapabilityRegistry, + provider: Arc, + remote_runtime: Option>, + listener: Mutex, +} + +impl fmt::Debug for SourceBroker { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SourceBroker") + .field( + "listener", + &*self + .listener + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ) + .finish_non_exhaustive() + } +} + +impl SourceBroker { + pub(crate) fn new( + engine: Arc, + listener: std::net::SocketAddr, + remote_runtime: Arc, + ) -> Self { + Self { + registry: CapabilityRegistry::default(), + provider: Arc::new(EngineFsSourceProvider { engine }), + remote_runtime: Some(remote_runtime), + listener: Mutex::new(listener), + } + } + + #[cfg(test)] + fn with_provider( + provider: Arc, + listener: std::net::SocketAddr, + ) -> Self { + Self { + registry: CapabilityRegistry::default(), + provider, + remote_runtime: None, + listener: Mutex::new(listener), + } + } + + pub async fn issue_engine_source( + &self, + info_hash: &str, + file_index: usize, + intent: PlaybackIntent, + lifetime: Duration, + ) -> Result { + let info_hash = normalize_info_hash(info_hash)?; + let metadata = self.provider.describe(&info_hash, file_index).await?; + let id = SourceId::random()?; + let activity = Arc::new(ActivityTracker::default()); + let scope = CapabilityScope { + source_id: id.as_str().to_owned(), + source_version: metadata.version.clone(), + }; + let payload = CapabilityPayload::Engine(Arc::new(EngineCapability { + info_hash, + file_index, + intent, + metadata: metadata.clone(), + provider: self.provider.clone(), + activity: activity.clone(), + })); + let issued = self.registry.issue_with_payload(scope, lifetime, payload)?; + let listener = *self + .listener + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let lease = Some(issued.into_lease(listener)?); + Ok(ValidatedMediaSource::EngineLoopback(EngineSource { + id, + version: metadata.version, + lease, + activity, + })) + } + + pub async fn issue_completed_file( + &self, + info_hash: &str, + file_index: usize, + ) -> Result { + let info_hash = normalize_info_hash(info_hash)?; + let described = self.provider.describe(&info_hash, file_index).await?; + let candidate = self + .provider + .completed_file(&info_hash, file_index) + .await? + .ok_or(SourceError::NotFound)?; + let canonical_path = tokio::fs::canonicalize(&candidate.path) + .await + .map_err(|_| SourceError::NotFound)?; + let mut allowed = false; + for root in candidate.allowed_roots { + let Ok(root) = tokio::fs::canonicalize(root).await else { + continue; + }; + if canonical_path.starts_with(&root) && canonical_path != root { + allowed = true; + break; + } + } + if !allowed { + return Err(SourceError::InvalidSource); + } + let identity_path = canonical_path.clone(); + let identity = + tokio::task::spawn_blocking(move || CompletedFileIdentity::from_path(&identity_path)) + .await + .map_err(|_| SourceError::Io)??; + if identity.length != described.length { + return Err(SourceError::InvalidSource); + } + let version = hex::encode(Sha256::digest( + format!( + "completed-source-v1\0{}\0{}\0{:?}", + described.version, identity.length, identity.modified_nanos + ) + .as_bytes(), + )); + Ok(ValidatedMediaSource::CompletedFile(CompletedFileSource { + id: SourceId::random()?, + version, + canonical_path: Some(canonical_path), + identity: Some(identity), + })) + } + + #[allow( + dead_code, + reason = "casting remains on its legacy remote path until the shared casting adapter gate" + )] + pub(crate) async fn issue_remote_source( + &self, + mut target: url::Url, + lifetime: Duration, + ) -> Result { + let runtime = self + .remote_runtime + .as_ref() + .cloned() + .ok_or(SourceError::InvalidSource)?; + target.set_fragment(None); + let context = runtime + .try_request_for_peer(None) + .map_err(|_| SourceError::RateLimited)?; + runtime + .validate(&context, &target) + .await + .map_err(|_| SourceError::InvalidSource)?; + drop(context); + + let id = SourceId::random()?; + let version = hex::encode(Sha256::digest( + format!("approved-remote-v1\0{}", target.as_str()).as_bytes(), + )); + let activity = Arc::new(ActivityTracker::default()); + let scope = CapabilityScope { + source_id: id.as_str().to_owned(), + source_version: version.clone(), + }; + let payload = CapabilityPayload::Remote(Arc::new(RemoteCapability { + target, + runtime, + activity: activity.clone(), + })); + let issued = self.registry.issue_with_payload(scope, lifetime, payload)?; + let listener = *self + .listener + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + Ok(ValidatedMediaSource::ApprovedRemote(RemoteSourceHandle { + id, + version, + lease: Some(issued.into_lease(listener)?), + activity, + })) + } + + async fn serve( + &self, + capability: &str, + method: Method, + range_header: Option<&str>, + peer: Option, + ) -> Response { + if !peer.is_some_and(|peer| peer.is_loopback()) { + return empty_response(StatusCode::FORBIDDEN); + } + if !matches!(method, Method::GET | Method::HEAD) { + return empty_response(StatusCode::METHOD_NOT_ALLOWED); + } + let entry = match self.registry.lookup(capability) { + Ok(entry) => entry, + Err(error) => return empty_response(source_error_status(error)), + }; + let _permit = match entry.try_acquire() { + Ok(permit) => permit, + Err(error) => return empty_response(source_error_status(error)), + }; + let source = match entry.payload.clone() { + CapabilityPayload::Engine(source) => source, + CapabilityPayload::Remote(source) => { + return serve_remote_source(entry, _permit, source, method, range_header).await; + } + #[cfg(test)] + CapabilityPayload::Fixture => return empty_response(StatusCode::NOT_FOUND), + }; + let range = match parse_single_range(range_header, source.metadata.length) { + Ok(range) => range, + Err(error) => return empty_response(source_error_status(error)), + }; + let status = if range.partial { + StatusCode::PARTIAL_CONTENT + } else { + StatusCode::OK + }; + + if method == Method::HEAD { + return build_source_response(status, &source, range, Body::empty()); + } + + let opened = match source + .provider + .open(&source.info_hash, source.file_index, range, source.intent) + .await + { + Ok(opened) => opened, + Err(error) => return empty_response(source_error_status(error)), + }; + if source + .provider + .refresh(&source.info_hash, source.file_index) + .await + .is_err() + { + return empty_response(StatusCode::INTERNAL_SERVER_ERROR); + } + let request_id = source.activity.start_request(range.half_open()); + let body_state = EngineBodyState { + opened, + source: source.clone(), + entry, + _permit, + remaining: range.len(), + offset: range.start, + activity_lease: ActivityRequestLease { + activity: source.activity.clone(), + request_id, + }, + }; + let body = Body::from_stream(stream::unfold(Some(body_state), |state| async move { + let mut state = state?; + if state.remaining == 0 { + return None; + } + let availability = state + .source + .provider + .piece_availability( + &state.source.info_hash, + state.source.file_index, + state.offset, + state.source.intent, + ) + .await + .unwrap_or(PieceAvailability::Unknown); + state.activity_lease.activity.mark_piece_blocked( + state.activity_lease.request_id, + matches!(availability, PieceAvailability::Unavailable), + ); + if state + .source + .provider + .refresh(&state.source.info_hash, state.source.file_index) + .await + .is_err() + { + return Some((Err(std::io::Error::other("source refresh failed")), None)); + } + let buffer_len = usize::try_from(state.remaining.min(64 * 1024)).unwrap_or(64 * 1024); + let mut buffer = vec![0_u8; buffer_len]; + let expiry = tokio::time::sleep_until(state.entry.expires_at); + tokio::pin!(expiry); + let read = tokio::select! { + biased; + _ = state.entry.revocation.cancelled() => { + return Some((Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "source revoked")), None)); + } + _ = &mut expiry => { + return Some((Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "source expired")), None)); + } + read = state.opened.reader.read(&mut buffer) => read, + }; + match read { + Ok(0) => Some(( + Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "source ended before the requested range", + )), + None, + )), + Ok(read) => { + buffer.truncate(read); + let delivered = read as u64; + state.remaining = state.remaining.saturating_sub(delivered); + state.offset = state.offset.saturating_add(delivered); + state + .activity_lease + .activity + .record_delivery(state.activity_lease.request_id, delivered); + Some((Ok(Bytes::from(buffer)), Some(state))) + } + Err(error) => Some((Err(error), None)), + } + })); + build_source_response(status, &source, range, body) + } +} + +struct EngineFsSourceProvider { + engine: Arc, +} + +struct EngineFsRequestLease { + engine: Arc, + info_hash: String, + file_index: usize, +} + +impl Drop for EngineFsRequestLease { + fn drop(&mut self) { + let engine = self.engine.clone(); + let info_hash = self.info_hash.clone(); + let file_index = self.file_index; + tokio::spawn(async move { + engine.on_stream_end(&info_hash, file_index).await; + }); + } +} + +#[async_trait::async_trait] +impl EngineSourceProvider for EngineFsSourceProvider { + async fn describe( + &self, + info_hash: &str, + file_index: usize, + ) -> Result { + let engine = self + .engine + .get_engine(info_hash) + .await + .ok_or(SourceError::NotFound)?; + let files = engine.handle.get_files().await; + let file = files.get(file_index).ok_or(SourceError::NotFound)?; + let version = hex::encode(Sha256::digest( + format!( + "engine-source-v1\0{info_hash}\0{file_index}\0{}", + file.length + ) + .as_bytes(), + )); + Ok(EngineFileMetadata { + length: file.length, + version, + content_type: media_content_type(&file.name), + }) + } + + async fn open( + &self, + info_hash: &str, + file_index: usize, + range: ByteRange, + intent: PlaybackIntent, + ) -> Result { + use enginefs::backend::{HotFilePriorityPlan, TorrentHandle}; + + let engine = self + .engine + .get_engine(info_hash) + .await + .ok_or(SourceError::NotFound)?; + let priority = match intent { + PlaybackIntent::InternalProbe => 255, + PlaybackIntent::Background => 0, + _ => 1, + }; + self.engine.on_stream_start(info_hash, file_index).await; + let lifecycle = Box::new(EngineFsRequestLease { + engine: self.engine.clone(), + info_hash: info_hash.to_owned(), + file_index, + }); + self.engine + .refresh_hls_playback(info_hash, file_index, "transcoding-source") + .await; + if !engine.handle.manages_playback_lifecycle() { + self.engine + .activate_multifile_file_for_playback( + info_hash, + file_index, + Some(HotFilePriorityPlan { + file_idx: file_index, + start_offset: range.start, + priority, + intent, + bitrate_bytes_per_sec: None, + }), + "transcoding-source", + ) + .await; + } + let mut file = engine + .get_file_with_intent(file_index, range.start, priority, intent) + .await + .ok_or(SourceError::Io)?; + file.seek(std::io::SeekFrom::Start(range.start)) + .await + .map_err(|_| SourceError::Io)?; + Ok(OpenedEngineSource { + reader: Box::pin(file), + _lifecycle: lifecycle, + }) + } + + async fn piece_availability( + &self, + info_hash: &str, + file_index: usize, + offset: u64, + intent: PlaybackIntent, + ) -> Result { + use enginefs::backend::TorrentHandle; + + let engine = self + .engine + .get_engine(info_hash) + .await + .ok_or(SourceError::NotFound)?; + let readiness = engine + .handle + .wait_for_piece_ready(file_index, offset, Duration::from_millis(1), intent) + .await + .map_err(|_| SourceError::Io)?; + if readiness.reason == "librqbit-reader" { + return Ok(PieceAvailability::Unknown); + } + Ok(if readiness.ready { + PieceAvailability::Available + } else { + PieceAvailability::Unavailable + }) + } + + async fn refresh(&self, info_hash: &str, file_index: usize) -> Result<(), SourceError> { + self.engine + .refresh_hls_playback(info_hash, file_index, "transcoding-source") + .await; + Ok(()) + } + + async fn completed_file( + &self, + info_hash: &str, + file_index: usize, + ) -> Result, SourceError> { + let engine = self + .engine + .get_engine(info_hash) + .await + .ok_or(SourceError::NotFound)?; + if !engine.handle.is_file_complete(file_index).await { + return Ok(None); + } + let Some(path) = engine.handle.get_file_path(file_index).await else { + return Ok(None); + }; + Ok(Some(CompletedFileCandidate { + path: PathBuf::from(path), + allowed_roots: vec![ + self.engine.download_dir.clone(), + self.engine.cache_dir.clone(), + ], + })) + } +} + +fn media_content_type(name: &str) -> &'static str { + let lower = name.to_ascii_lowercase(); + if lower.ends_with(".mp4") || lower.ends_with(".m4v") { + "video/mp4" + } else if lower.ends_with(".mkv") { + "video/x-matroska" + } else if lower.ends_with(".webm") { + "video/webm" + } else if lower.ends_with(".ts") || lower.ends_with(".m2ts") { + "video/mp2t" + } else if lower.ends_with(".avi") { + "video/x-msvideo" + } else if lower.ends_with(".mov") { + "video/quicktime" + } else if lower.ends_with(".mp3") { + "audio/mpeg" + } else if lower.ends_with(".flac") { + "audio/flac" + } else { + "application/octet-stream" + } +} + +pub(crate) async fn route_source( + State(state): State, + peer: Option>>, + method: Method, + headers: HeaderMap, + RawQuery(raw_query): RawQuery, +) -> Response { + let Some(capability) = raw_query.as_deref().and_then(parse_capability_query) else { + return empty_response(StatusCode::UNAUTHORIZED); + }; + let range = headers + .get(header::RANGE) + .and_then(|value| value.to_str().ok()); + let peer = peer.map(|Extension(ConnectInfo(address))| address.ip()); + state + .source_broker + .serve(capability, method, range, peer) + .await +} + +fn parse_capability_query(query: &str) -> Option<&str> { + let capability = query.strip_prefix("cap=")?; + (capability.len() == CAPABILITY_HEX_BYTES + && capability.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then_some(capability) +} + +pub async fn issue_engine_source( + broker: &SourceBroker, + info_hash: &str, + file_index: usize, + intent: PlaybackIntent, + lifetime: Duration, +) -> Result { + broker + .issue_engine_source(info_hash, file_index, intent, lifetime) + .await +} + +struct EngineBodyState { + opened: OpenedEngineSource, + source: Arc, + entry: Arc, + _permit: OwnedSemaphorePermit, + remaining: u64, + offset: u64, + activity_lease: ActivityRequestLease, +} + +struct RemoteBodyState { + upstream: std::pin::Pin> + Send + 'static>>, + entry: Arc, + _source_lease: crate::network_security::ProxyProducerLease, + _permit: OwnedSemaphorePermit, + activity_lease: ActivityRequestLease, +} + +async fn serve_remote_source( + entry: Arc, + permit: OwnedSemaphorePermit, + source: Arc, + method: Method, + range_header: Option<&str>, +) -> Response { + let range_value = match range_header { + Some(value) if remote_range_is_valid(value) => match HeaderValue::from_str(value) { + Ok(value) => Some(value), + Err(_) => return empty_response(StatusCode::RANGE_NOT_SATISFIABLE), + }, + Some(_) => return empty_response(StatusCode::RANGE_NOT_SATISFIABLE), + None => None, + }; + let fetched = match crate::routes::proxy::fetch_media_source( + &source.runtime, + source.target.clone(), + method.clone(), + range_value.as_ref(), + ) + .await + { + Ok(fetched) => fetched, + Err(_) => return empty_response(StatusCode::BAD_GATEWAY), + }; + let crate::routes::proxy::FetchedMediaSource { + response: upstream, + _lease: source_lease, + } = fetched; + let status = upstream.status(); + if !status.is_success() { + return empty_response(status); + } + let upstream_headers = upstream.headers().clone(); + let mut builder = Response::builder() + .status(status) + .header(header::CACHE_CONTROL, "no-store"); + for name in [ + header::CONTENT_TYPE, + header::CONTENT_LENGTH, + header::CONTENT_RANGE, + header::ACCEPT_RANGES, + ] { + if let Some(value) = upstream_headers.get(&name) { + builder = builder.header(name, value); + } + } + if method == Method::HEAD { + return builder + .body(Body::empty()) + .expect("validated remote source response is valid"); + } + + let activity_range = remote_activity_range(&upstream_headers); + let request_id = source.activity.start_request(activity_range); + let body_state = RemoteBodyState { + upstream: Box::pin(upstream.bytes_stream()), + entry, + _source_lease: source_lease, + _permit: permit, + activity_lease: ActivityRequestLease { + activity: source.activity.clone(), + request_id, + }, + }; + let body = Body::from_stream(stream::unfold(Some(body_state), |state| async move { + let mut state = state?; + let expiry = tokio::time::sleep_until(state.entry.expires_at); + tokio::pin!(expiry); + let item = tokio::select! { + biased; + _ = state.entry.revocation.cancelled() => { + return Some((Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "source revoked")), None)); + } + _ = &mut expiry => { + return Some((Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "source expired")), None)); + } + item = state.upstream.next() => item, + }; + match item { + Some(Ok(bytes)) => { + state + .activity_lease + .activity + .record_delivery(state.activity_lease.request_id, bytes.len() as u64); + Some((Ok(bytes), Some(state))) + } + Some(Err(_)) => Some((Err(std::io::Error::other("remote source failed")), None)), + None => None, + } + })); + builder + .body(body) + .expect("validated remote source response is valid") +} + +fn remote_range_is_valid(value: &str) -> bool { + let Some(spec) = value.strip_prefix("bytes=") else { + return false; + }; + if spec.is_empty() || spec.contains(',') { + return false; + } + let Some((left, right)) = spec.split_once('-') else { + return false; + }; + match (left.is_empty(), right.is_empty()) { + (true, false) => right.parse::().is_ok_and(|suffix| suffix > 0), + (false, true) => left.parse::().is_ok(), + (false, false) => match (left.parse::(), right.parse::()) { + (Ok(start), Ok(end)) => start <= end, + _ => false, + }, + (true, true) => false, + } +} + +fn remote_activity_range(headers: &HeaderMap) -> Range { + if let Some(value) = headers + .get(header::CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("bytes ")) + .and_then(|value| value.split_once('/').map(|(range, _)| range)) + .and_then(|range| range.split_once('-')) + && let (Ok(start), Ok(end)) = (value.0.parse::(), value.1.parse::()) + && start <= end + { + return start..end.saturating_add(1); + } + let length = headers + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + 0..length +} + +struct ActivityRequestLease { + activity: Arc, + request_id: u64, +} + +impl Drop for ActivityRequestLease { + fn drop(&mut self) { + self.activity.finish_request(self.request_id); + } +} + +fn normalize_info_hash(info_hash: &str) -> Result { + if matches!(info_hash.len(), 40 | 64) && info_hash.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + Ok(info_hash.to_ascii_lowercase()) + } else { + Err(SourceError::InvalidSource) + } +} + +fn source_error_status(error: SourceError) -> StatusCode { + match error { + SourceError::InvalidCapability => StatusCode::UNAUTHORIZED, + SourceError::Expired | SourceError::Revoked => StatusCode::GONE, + SourceError::RateLimited | SourceError::Capacity => StatusCode::TOO_MANY_REQUESTS, + SourceError::InvalidRange => StatusCode::RANGE_NOT_SATISFIABLE, + SourceError::NotFound => StatusCode::NOT_FOUND, + SourceError::InvalidSource | SourceError::Io => StatusCode::INTERNAL_SERVER_ERROR, + } +} + +fn empty_response(status: StatusCode) -> Response { + Response::builder() + .status(status) + .header(header::CACHE_CONTROL, "no-store") + .body(Body::empty()) + .expect("static source response is valid") +} + +fn build_source_response( + status: StatusCode, + source: &EngineCapability, + range: ByteRange, + body: Body, +) -> Response { + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, source.metadata.content_type) + .header(header::CONTENT_LENGTH, range.len()) + .header(header::ACCEPT_RANGES, "bytes") + .header(header::CACHE_CONTROL, "no-store"); + if range.partial { + builder = builder.header( + header::CONTENT_RANGE, + format!( + "bytes {}-{}/{}", + range.start, range.end_inclusive, range.full_size + ), + ); + } + builder + .body(body) + .expect("validated source response headers are valid") +} + +#[derive(Clone, Debug)] +struct RequestActivity { + range: Range, + piece_blocked: bool, +} + +struct ActivityState { + next_request_id: u64, + snapshot: SourceActivitySnapshot, + requests: HashMap, +} + +struct ActivityTracker { + state: Mutex, + sender: watch::Sender, +} + +impl Default for ActivityTracker { + fn default() -> Self { + let snapshot = SourceActivitySnapshot::default(); + let (sender, _) = watch::channel(snapshot.clone()); + Self { + state: Mutex::new(ActivityState { + next_request_id: 1, + snapshot, + requests: HashMap::new(), + }), + sender, + } + } +} + +impl ActivityTracker { + fn subscribe(&self) -> watch::Receiver { + self.sender.subscribe() + } + + fn start_request(&self, range: Range) -> u64 { + let mut state = self.lock(); + let request_id = state.next_request_id; + state.next_request_id = state.next_request_id.saturating_add(1); + state.requests.insert( + request_id, + RequestActivity { + range, + piece_blocked: false, + }, + ); + self.publish(&mut state); + request_id + } + + fn mark_piece_blocked(&self, request_id: u64, blocked: bool) { + let mut state = self.lock(); + if let Some(request) = state.requests.get_mut(&request_id) + && request.piece_blocked != blocked + { + request.piece_blocked = blocked; + self.publish(&mut state); + } + } + + fn record_delivery(&self, request_id: u64, bytes: u64) { + if bytes == 0 { + return; + } + let mut state = self.lock(); + let Some(request) = state.requests.get_mut(&request_id) else { + return; + }; + request.piece_blocked = false; + request.range.start = request + .range + .start + .saturating_add(bytes) + .min(request.range.end); + state.snapshot.delivered_bytes_total = + state.snapshot.delivered_bytes_total.saturating_add(bytes); + self.publish(&mut state); + } + + fn finish_request(&self, request_id: u64) { + let mut state = self.lock(); + if state.requests.remove(&request_id).is_some() { + self.publish(&mut state); + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, ActivityState> { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn publish(&self, state: &mut ActivityState) { + state.snapshot.sequence = state.snapshot.sequence.saturating_add(1); + state.snapshot.active_requests = u32::try_from(state.requests.len()).unwrap_or(u32::MAX); + let mut waiting = state + .requests + .values() + .filter(|request| request.piece_blocked) + .map(|request| request.range.clone()) + .collect::>(); + waiting.sort_by_key(|range| (range.start, range.end)); + waiting.truncate(MAX_CONCURRENT_REQUESTS_PER_CAPABILITY); + state.snapshot.waiting_for_pieces = waiting; + state.snapshot.all_active_requests_piece_blocked = !state.requests.is_empty() + && state.requests.values().all(|request| request.piece_blocked); + self.sender.send_replace(state.snapshot.clone()); + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ByteRange { + start: u64, + end_inclusive: u64, + full_size: u64, + partial: bool, +} + +impl ByteRange { + fn full(full_size: u64) -> Self { + Self { + start: 0, + end_inclusive: full_size.saturating_sub(1), + full_size, + partial: false, + } + } + + fn partial(start: u64, end_inclusive: u64, full_size: u64) -> Self { + Self { + start, + end_inclusive, + full_size, + partial: true, + } + } + + fn len(self) -> u64 { + if self.full_size == 0 { + 0 + } else { + self.end_inclusive.saturating_sub(self.start) + 1 + } + } + + fn half_open(self) -> Range { + self.start..self.start.saturating_add(self.len()) + } +} + +fn parse_single_range(value: Option<&str>, size: u64) -> Result { + let Some(value) = value else { + return Ok(ByteRange::full(size)); + }; + let spec = value + .strip_prefix("bytes=") + .ok_or(SourceError::InvalidRange)?; + if spec.is_empty() || spec.contains(',') || size == 0 { + return Err(SourceError::InvalidRange); + } + let (left, right) = spec.split_once('-').ok_or(SourceError::InvalidRange)?; + let (start, end) = match (left.is_empty(), right.is_empty()) { + (true, false) => { + let suffix = right + .parse::() + .map_err(|_| SourceError::InvalidRange)?; + if suffix == 0 { + return Err(SourceError::InvalidRange); + } + let length = suffix.min(size); + (size - length, size - 1) + } + (false, true) => { + let start = left.parse::().map_err(|_| SourceError::InvalidRange)?; + if start >= size { + return Err(SourceError::InvalidRange); + } + (start, size - 1) + } + (false, false) => { + let start = left.parse::().map_err(|_| SourceError::InvalidRange)?; + let end = right + .parse::() + .map_err(|_| SourceError::InvalidRange)?; + if start > end || start >= size { + return Err(SourceError::InvalidRange); + } + (start, end.min(size - 1)) + } + (true, true) => return Err(SourceError::InvalidRange), + }; + Ok(ByteRange::partial(start, end, size)) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::to_bytes, http::Method}; + use enginefs::backend::priorities::PlaybackIntent; + use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + pin::Pin, + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll}, + time::Duration, + }; + use tokio::io::{AsyncRead, ReadBuf}; + + #[derive(Default)] + struct MockProvider { + data: Vec, + completed_path: Option, + opens: AtomicUsize, + reads: Arc, + refreshes: AtomicUsize, + finishes: Arc, + intent: Mutex>, + } + + impl MockProvider { + fn with_data(data: &[u8]) -> Self { + Self { + data: data.to_vec(), + ..Self::default() + } + } + } + + struct CountingReader { + inner: std::io::Cursor>, + reads: Arc, + } + + impl AsyncRead for CountingReader { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + self.reads.fetch_add(1, Ordering::SeqCst); + Pin::new(&mut self.inner).poll_read(context, buffer) + } + } + + struct FinishCounter(Arc); + + impl Drop for FinishCounter { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + #[async_trait::async_trait] + impl EngineSourceProvider for MockProvider { + async fn describe( + &self, + _info_hash: &str, + file_index: usize, + ) -> Result { + if file_index != 0 { + return Err(SourceError::NotFound); + } + Ok(EngineFileMetadata { + length: self.data.len() as u64, + version: "immutable-v1".to_owned(), + content_type: "video/x-matroska", + }) + } + + async fn open( + &self, + _info_hash: &str, + _file_index: usize, + range: ByteRange, + intent: PlaybackIntent, + ) -> Result { + self.opens.fetch_add(1, Ordering::SeqCst); + *self + .intent + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(intent); + let start = usize::try_from(range.start).map_err(|_| SourceError::InvalidRange)?; + let mut reader = CountingReader { + inner: std::io::Cursor::new(self.data.clone()), + reads: self.reads.clone(), + }; + reader.inner.set_position(start as u64); + Ok(OpenedEngineSource { + reader: Box::pin(reader), + _lifecycle: Box::new(FinishCounter(self.finishes.clone())), + }) + } + + async fn piece_availability( + &self, + _info_hash: &str, + _file_index: usize, + _offset: u64, + _intent: PlaybackIntent, + ) -> Result { + Ok(PieceAvailability::Available) + } + + async fn refresh(&self, _info_hash: &str, _file_index: usize) -> Result<(), SourceError> { + self.refreshes.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + async fn completed_file( + &self, + _info_hash: &str, + file_index: usize, + ) -> Result, SourceError> { + if file_index != 0 { + return Err(SourceError::NotFound); + } + Ok(self + .completed_path + .as_ref() + .map(|path| CompletedFileCandidate { + path: path.clone(), + allowed_roots: vec![path.parent().unwrap().to_path_buf()], + })) + } + } + + fn broker_fixture(data: &[u8]) -> (Arc, SourceBroker) { + let provider = Arc::new(MockProvider::with_data(data)); + let broker = + SourceBroker::with_provider(provider.clone(), SocketAddr::from(([0, 0, 0, 0], 43123))); + (provider, broker) + } + + fn capability_from(source: &ValidatedMediaSource) -> String { + let input = source.input_argument().unwrap(); + let input = input.to_str().unwrap(); + url::Url::parse(input) + .unwrap() + .query_pairs() + .find_map(|(name, value)| (name == "cap").then(|| value.into_owned())) + .unwrap() + } + + #[tokio::test] + async fn issued_engine_source_is_opaque_numeric_loopback_and_exact_intent_scoped() { + let (provider, broker) = broker_fixture(b"0123456789"); + let info_hash = "0123456789abcdef0123456789abcdef01234567"; + let source = broker + .issue_engine_source( + info_hash, + 0, + PlaybackIntent::InternalProbe, + Duration::from_secs(60), + ) + .await + .unwrap(); + let input = source.input_argument().unwrap(); + let input = input.to_str().unwrap(); + let parsed = url::Url::parse(input).unwrap(); + + assert_eq!(parsed.host_str(), Some("127.0.0.1")); + assert_eq!(parsed.port(), Some(43123)); + assert_eq!(source.ffmpeg_protocol_allowlist(), "http,tcp"); + assert!(!source.id().contains(info_hash)); + assert!(!format!("{source:?}").contains(&capability_from(&source))); + assert!( + !serde_json::to_string(&source) + .unwrap() + .contains(&capability_from(&source)) + ); + + let capability = capability_from(&source); + let response = broker + .serve( + &capability, + Method::GET, + Some("bytes=2-5"), + Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ) + .await; + assert_eq!(response.status(), axum::http::StatusCode::PARTIAL_CONTENT); + assert_eq!( + to_bytes(response.into_body(), 16).await.unwrap().as_ref(), + b"2345" + ); + assert_eq!( + *provider + .intent + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + Some(PlaybackIntent::InternalProbe) + ); + assert!(provider.refreshes.load(Ordering::SeqCst) > 0); + assert_eq!(provider.finishes.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn head_does_not_open_and_get_is_lazy_backpressured_and_cancel_safe() { + let (provider, broker) = broker_fixture(b"0123456789"); + let source = broker + .issue_engine_source( + "0123456789abcdef0123456789abcdef01234567", + 0, + PlaybackIntent::HlsInitial, + Duration::from_secs(60), + ) + .await + .unwrap(); + let capability = capability_from(&source); + + let head = broker + .serve( + &capability, + Method::HEAD, + Some("bytes=1-3"), + Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ) + .await; + assert_eq!(head.status(), axum::http::StatusCode::PARTIAL_CONTENT); + assert_eq!(provider.opens.load(Ordering::SeqCst), 0); + + let get = broker + .serve( + &capability, + Method::GET, + None, + Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ) + .await; + assert_eq!(provider.opens.load(Ordering::SeqCst), 1); + assert_eq!(provider.reads.load(Ordering::SeqCst), 0); + drop(get); + tokio::task::yield_now().await; + assert_eq!(provider.finishes.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn source_route_rejects_non_loopback_multiple_ranges_and_revocation() { + let (_provider, broker) = broker_fixture(b"0123456789"); + let source = broker + .issue_engine_source( + "0123456789abcdef0123456789abcdef01234567", + 0, + PlaybackIntent::HlsSeek, + Duration::from_secs(60), + ) + .await + .unwrap(); + let capability = capability_from(&source); + + let remote = broker + .serve( + &capability, + Method::GET, + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))), + ) + .await; + assert_eq!(remote.status(), axum::http::StatusCode::FORBIDDEN); + + let multiple = broker + .serve( + &capability, + Method::GET, + Some("bytes=0-1,3-4"), + Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ) + .await; + assert_eq!( + multiple.status(), + axum::http::StatusCode::RANGE_NOT_SATISFIABLE + ); + + if let ValidatedMediaSource::EngineLoopback(engine) = &source { + engine.revoke(); + } + let revoked = broker + .serve( + &capability, + Method::GET, + None, + Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ) + .await; + assert_eq!(revoked.status(), axum::http::StatusCode::GONE); + } + + #[tokio::test] + async fn completed_file_is_canonical_allowlisted_and_change_detected() { + let temp = tempfile::tempdir().unwrap(); + let media = temp.path().join("movie.mkv"); + tokio::fs::write(&media, b"immutable-media").await.unwrap(); + let provider = Arc::new(MockProvider { + data: b"immutable-media".to_vec(), + completed_path: Some(media.clone()), + ..MockProvider::default() + }); + let broker = + SourceBroker::with_provider(provider, SocketAddr::from(([127, 0, 0, 1], 43123))); + let source = broker + .issue_completed_file("0123456789abcdef0123456789abcdef01234567", 0) + .await + .unwrap(); + + assert!(matches!(source, ValidatedMediaSource::CompletedFile(_))); + let rendered = format!("{source:?}"); + let canonical = media.canonicalize().unwrap(); + assert!(!rendered.contains(&canonical.to_string_lossy().to_string())); + assert!(!rendered.contains("movie.mkv")); + assert!(!rendered.contains("immutable-media")); + assert_eq!(source.ffmpeg_protocol_allowlist(), "file,pipe"); + assert_eq!(source.input_argument().unwrap(), canonical); + + let replacement = temp.path().join("replacement.mkv"); + tokio::fs::write(&replacement, vec![b'x'; b"immutable-media".len()]) + .await + .unwrap(); + tokio::fs::remove_file(&media).await.unwrap(); + tokio::fs::rename(&replacement, &media).await.unwrap(); + assert_eq!(source.input_argument(), Err(SourceError::InvalidSource)); + } + + #[tokio::test] + async fn approved_remote_uses_shared_ssrf_policy_and_keeps_target_credentials_opaque() { + let provider = Arc::new(MockProvider::with_data(b"remote-placeholder")); + let validator = Arc::new(crate::network_security::DestinationValidator::new( + Arc::new(crate::network_security::SystemDnsResolver), + Arc::new(crate::network_security::SystemLocalNetworkProvider), + Arc::new(crate::network_security::SystemClock), + vec![crate::network_security::ListenerBinding { + socket: SocketAddr::from(([0, 0, 0, 0], 43123)), + }], + )); + let runtime = Arc::new(crate::network_security::ProxyRuntime::new( + crate::network_security::ProxyPolicySettings::default(), + validator, + )); + let mut broker = + SourceBroker::with_provider(provider, SocketAddr::from(([0, 0, 0, 0], 43123))); + broker.remote_runtime = Some(runtime); + + let blocked = broker + .issue_remote_source( + url::Url::parse("http://169.254.169.254/latest/meta-data").unwrap(), + Duration::from_secs(60), + ) + .await; + assert!(matches!(blocked, Err(SourceError::InvalidSource))); + + let target = "http://user:secret@93.184.216.34/media.mkv?token=private"; + let source = broker + .issue_remote_source(url::Url::parse(target).unwrap(), Duration::from_secs(60)) + .await + .unwrap(); + let capability = capability_from(&source); + let debug = format!("{source:?}"); + let serialized = serde_json::to_string(&source).unwrap(); + + assert!(matches!(source, ValidatedMediaSource::ApprovedRemote(_))); + assert_eq!(source.ffmpeg_protocol_allowlist(), "http,tcp"); + assert!(!debug.contains("secret")); + assert!(!debug.contains("93.184.216.34")); + assert!(!debug.contains(&capability)); + assert!(!serialized.contains("secret")); + assert!(!serialized.contains("93.184.216.34")); + assert!(!serialized.contains(&capability)); + } + + #[test] + fn capabilities_are_256_bit_random_redacted_and_constant_time_verified() { + let registry = CapabilityRegistry::default(); + let first = registry + .issue( + CapabilityScope::fixture("source-a", "version-a"), + Duration::from_secs(60), + ) + .unwrap(); + let second = registry + .issue( + CapabilityScope::fixture("source-b", "version-b"), + Duration::from_secs(60), + ) + .unwrap(); + + assert_eq!(first.credential.encoded().len(), 64); + assert_eq!(second.credential.encoded().len(), 64); + assert_ne!(first.credential.encoded(), second.credential.encoded()); + assert!(!format!("{:?}", first.credential).contains(first.credential.encoded())); + + let resolved = registry + .lookup(first.credential.encoded()) + .expect("issued capability resolves"); + assert_eq!(resolved.scope.source_id, "source-a"); + + let mut altered = first.credential.encoded().as_bytes().to_vec(); + altered[0] = if altered[0] == b'a' { b'b' } else { b'a' }; + let altered = std::str::from_utf8(&altered).unwrap(); + assert!(matches!( + registry.lookup(altered), + Err(SourceError::InvalidCapability) + )); + } + + #[tokio::test(start_paused = true)] + async fn capabilities_expire_revoke_and_enforce_per_source_request_capacity() { + let registry = CapabilityRegistry::default(); + let issued = registry + .issue( + CapabilityScope::fixture("source-a", "version-a"), + Duration::from_secs(5), + ) + .unwrap(); + let entry = registry.lookup(issued.credential.encoded()).unwrap(); + + let permits = (0..MAX_CONCURRENT_REQUESTS_PER_CAPABILITY) + .map(|_| entry.try_acquire().unwrap()) + .collect::>(); + assert!(matches!(entry.try_acquire(), Err(SourceError::RateLimited))); + drop(permits); + assert!(entry.try_acquire().is_ok()); + + issued.revoke(); + assert!(matches!( + registry.lookup(issued.credential.encoded()), + Err(SourceError::Revoked) + )); + + let expiring = registry + .issue( + CapabilityScope::fixture("source-b", "version-b"), + Duration::from_secs(5), + ) + .unwrap(); + tokio::time::advance(Duration::from_secs(6)).await; + assert!(matches!( + registry.lookup(expiring.credential.encoded()), + Err(SourceError::Expired) + )); + } + + #[test] + fn activity_is_monotonic_bounded_and_requires_every_active_request_to_be_blocked() { + let activity = ActivityTracker::default(); + let mut observer = activity.subscribe(); + let first = activity.start_request(0..64); + let second = activity.start_request(128..192); + + activity.mark_piece_blocked(first, true); + let partial = observer.borrow_and_update().clone(); + assert_eq!(partial.active_requests, 2); + assert_eq!(partial.waiting_for_pieces, vec![0..64]); + assert!(!partial.all_active_requests_piece_blocked); + + activity.mark_piece_blocked(second, true); + let all_blocked = observer.borrow_and_update().clone(); + assert!(all_blocked.sequence > partial.sequence); + assert_eq!(all_blocked.waiting_for_pieces, vec![0..64, 128..192]); + assert!(all_blocked.all_active_requests_piece_blocked); + + activity.record_delivery(first, 16); + let progressed = observer.borrow_and_update().clone(); + assert!(progressed.sequence > all_blocked.sequence); + assert_eq!(progressed.delivered_bytes_total, 16); + assert_eq!(progressed.waiting_for_pieces, vec![128..192]); + assert!(!progressed.all_active_requests_piece_blocked); + + activity.mark_piece_blocked(first, true); + let resumed_wait = observer.borrow_and_update().clone(); + assert_eq!(resumed_wait.waiting_for_pieces, vec![16..64, 128..192]); + assert!(resumed_wait.all_active_requests_piece_blocked); + + activity.finish_request(first); + activity.finish_request(second); + let finished = observer.borrow_and_update().clone(); + assert_eq!(finished.active_requests, 0); + assert!(!finished.all_active_requests_piece_blocked); + } + + #[test] + fn range_parser_accepts_one_bounded_range_and_rejects_multiple_or_invalid_ranges() { + assert_eq!(parse_single_range(None, 100).unwrap(), ByteRange::full(100)); + assert_eq!( + parse_single_range(Some("bytes=10-19"), 100).unwrap(), + ByteRange::partial(10, 19, 100) + ); + assert_eq!( + parse_single_range(Some("bytes=90-"), 100).unwrap(), + ByteRange::partial(90, 99, 100) + ); + assert_eq!( + parse_single_range(Some("bytes=-10"), 100).unwrap(), + ByteRange::partial(90, 99, 100) + ); + + for invalid in [ + "bytes=0-1,4-5", + "items=0-1", + "bytes=100-100", + "bytes=20-10", + "bytes=-0", + "bytes=", + ] { + assert_eq!( + parse_single_range(Some(invalid), 100), + Err(SourceError::InvalidRange), + "{invalid}" + ); + } + } +} diff --git a/server/tests/fixtures/ffprobe/codec_matrix.json b/server/tests/fixtures/ffprobe/codec_matrix.json new file mode 100644 index 00000000..acb569da --- /dev/null +++ b/server/tests/fixtures/ffprobe/codec_matrix.json @@ -0,0 +1,64 @@ +{ + "format": { + "format_name": "matroska,webm", + "start_time": "0.125000", + "duration": "120.500000", + "bit_rate": "8500000" + }, + "streams": [ + { + "index": 2, + "codec_type": "video", + "codec_name": "h264", + "codec_tag_string": "avc1", + "profile": "High", + "level": 41, + "width": 1920, + "height": 1080, + "sample_aspect_ratio": "1:1", + "display_aspect_ratio": "16:9", + "pix_fmt": "yuv420p", + "bits_per_raw_sample": "8", + "r_frame_rate": "24000/1001", + "avg_frame_rate": "24000/1001", + "time_base": "1/1000", + "codec_time_base": "1001/48000", + "field_order": "progressive", + "color_range": "tv", + "color_space": "bt709", + "color_transfer": "bt709", + "color_primaries": "bt709", + "bit_rate": "8000000", + "tags": { "language": "eng" }, + "disposition": { "default": 1, "forced": 0, "attached_pic": 0 }, + "side_data_list": [ + { "side_data_type": "Display Matrix", "rotation": -90 }, + { + "side_data_type": "Mastering display metadata", + "red_x": "17/25", "red_y": "8/25", + "green_x": "53/200", "green_y": "69/100", + "blue_x": "3/20", "blue_y": "3/50", + "white_point_x": "3127/10000", "white_point_y": "329/1000", + "min_luminance": "1/10000", "max_luminance": "1000/1" + }, + { "side_data_type": "Content light level metadata", "max_content": 1000, "max_average": 400 }, + { "side_data_type": "DOVI configuration record", "dv_profile": 8, "dv_level": 6, "rpu_present_flag": 1, "el_present_flag": 0, "bl_present_flag": 1 } + ] + }, + { "index": 3, "codec_type": "video", "codec_name": "h264", "profile": "High 10", "pix_fmt": "yuv420p10le", "bits_per_raw_sample": "10", "r_frame_rate": "30000/1001", "avg_frame_rate": "24000/1001", "disposition": { "default": 0 } }, + { "index": 4, "codec_type": "video", "codec_name": "hevc", "profile": "Main", "pix_fmt": "yuv420p", "bits_per_raw_sample": "8" }, + { "index": 5, "codec_type": "video", "codec_name": "hevc", "profile": "Main 10", "pix_fmt": "yuv420p10le", "bits_per_raw_sample": "10" }, + { "index": 6, "codec_type": "video", "codec_name": "av1", "profile": "Main", "pix_fmt": "yuv420p", "bits_per_raw_sample": "8" }, + { "index": 7, "codec_type": "video", "codec_name": "av1", "profile": "Main", "pix_fmt": "yuv420p10le", "bits_per_raw_sample": "10" }, + { "index": 8, "codec_type": "video", "codec_name": "vp9", "profile": "Profile 0", "pix_fmt": "yuv420p", "bits_per_raw_sample": "8" }, + { "index": 9, "codec_type": "video", "codec_name": "vp9", "profile": "Profile 2", "pix_fmt": "yuv420p10le", "bits_per_raw_sample": "10" }, + { "index": 10, "codec_type": "video", "codec_name": "mpeg2video", "profile": "Main", "pix_fmt": "yuv420p" }, + { "index": 11, "codec_type": "video", "codec_name": "vc1", "profile": "Advanced", "pix_fmt": "yuv420p" }, + { "index": 12, "codec_type": "video", "codec_name": "future_codec", "profile": "Future", "pix_fmt": "future_fmt" }, + { "index": 20, "codec_type": "audio", "codec_name": "aac", "profile": "LC", "sample_rate": "48000", "channels": 6, "channel_layout": "5.1", "bit_rate": "384000", "tags": { "language": "jpn" }, "disposition": { "default": 1 } }, + { "index": 30, "codec_type": "subtitle", "codec_name": "subrip", "tags": { "language": "spa" }, "disposition": { "forced": 1 } } + ], + "chapters": [ + { "id": 0, "start_time": "0.000000", "end_time": "60.000000", "tags": { "title": "Part 1" } } + ] +} diff --git a/server/tests/fixtures/ffprobe/compatibility.json b/server/tests/fixtures/ffprobe/compatibility.json new file mode 100644 index 00000000..fe54a609 --- /dev/null +++ b/server/tests/fixtures/ffprobe/compatibility.json @@ -0,0 +1,15 @@ +{ + "format": { "format_name": "matroska,webm", "duration": "12.500000" }, + "streams": [ + { + "index": 0, "codec_type": "video", "codec_name": "h264", "profile": "High", + "width": 1280, "height": 720, "r_frame_rate": "30/1", "avg_frame_rate": "30/1", + "bit_rate": "3000000", "tags": { "language": "eng" }, "disposition": { "default": 1 } + }, + { + "index": 1, "codec_type": "audio", "codec_name": "aac", "profile": "LC", + "channels": 2, "bit_rate": "128000", "tags": { "language": "eng" }, + "disposition": { "default": 1 } + } + ] +} diff --git a/server/tests/fixtures/ffprobe/missing_unknown.json b/server/tests/fixtures/ffprobe/missing_unknown.json new file mode 100644 index 00000000..e2faac17 --- /dev/null +++ b/server/tests/fixtures/ffprobe/missing_unknown.json @@ -0,0 +1,10 @@ +{ + "format": {}, + "streams": [ + { "index": 0, "codec_type": "video" }, + { "index": 1, "codec_type": "audio" }, + { "index": 2, "codec_type": "subtitle" }, + { "index": 3, "codec_type": "data", "codec_name": "bin_data" } + ], + "chapters": [] +} diff --git a/server/tests/transcoding_probe.rs b/server/tests/transcoding_probe.rs new file mode 100644 index 00000000..b1d0aa1b --- /dev/null +++ b/server/tests/transcoding_probe.rs @@ -0,0 +1,232 @@ +use stream_server::transcoding::{ + ChromaSubsampling, ColorMatrix, ColorPrimaries, ColorRange, ColorTransfer, FieldOrder, + FrameRateClass, InputVideoCodec, PixelFormat, VideoProfile, parse_probe_document, +}; + +#[test] +fn complete_fixture_parses_all_supported_input_families_and_selects_default_stream() { + let parsed = parse_probe_document(include_bytes!("fixtures/ffprobe/codec_matrix.json")) + .expect("parse complete ffprobe fixture"); + let videos = parsed.video_streams().collect::>(); + + assert_eq!(parsed.container_display(), "matroska,webm"); + assert_eq!(parsed.duration_micros(), Some(120_500_000)); + assert_eq!(parsed.start_micros(), Some(125_000)); + assert_eq!(parsed.selected_video_stream(), Some(2)); + assert_eq!(parsed.selected_audio_stream(), Some(20)); + assert_eq!(videos.len(), 11); + assert_eq!(videos[0].codec(), InputVideoCodec::H264); + assert_eq!(videos[0].profile(), VideoProfile::H264High); + assert_eq!(videos[0].pixel_format(), PixelFormat::Yuv420p); + assert_eq!(videos[0].bit_depth(), Some(8)); + assert_eq!(videos[0].chroma(), ChromaSubsampling::Cs420); + assert_eq!(videos[0].frame_rate_class(), FrameRateClass::Constant); + assert_eq!(videos[1].profile(), VideoProfile::H264High10); + assert_eq!(videos[1].frame_rate_class(), FrameRateClass::Variable); + assert_eq!(videos[2].codec(), InputVideoCodec::Hevc); + assert_eq!(videos[3].profile(), VideoProfile::HevcMain10); + assert_eq!(videos[4].codec(), InputVideoCodec::Av1); + assert_eq!(videos[5].bit_depth(), Some(10)); + assert_eq!(videos[6].codec(), InputVideoCodec::Vp9); + assert_eq!(videos[7].profile(), VideoProfile::Vp9Profile2); + assert_eq!(videos[8].codec(), InputVideoCodec::Mpeg2); + assert_eq!(videos[9].codec(), InputVideoCodec::Vc1); + assert_eq!(videos[10].codec(), InputVideoCodec::OtherProbed); + assert_eq!(videos[10].codec_display(), "future_codec"); + assert_eq!(videos[10].pixel_format(), PixelFormat::OtherProbed); + + let selected = parsed.selected_video().expect("selected video"); + assert_eq!(selected.codec_tag(), Some("avc1")); + assert_eq!(selected.level(), Some(41)); + assert_eq!(selected.sample_aspect_ratio().unwrap().numerator(), 1); + assert_eq!(selected.display_aspect_ratio().unwrap().numerator(), 16); + assert_eq!(selected.stream_time_base().unwrap().denominator(), 1000); + assert_eq!(selected.codec_time_base().unwrap().numerator(), 1001); + assert_eq!(selected.field_order(), FieldOrder::Progressive); + assert_eq!(selected.color().primaries, ColorPrimaries::Bt709); + assert_eq!(selected.color().transfer, ColorTransfer::Bt709); + assert_eq!(selected.color().matrix, ColorMatrix::Bt709); + assert_eq!(selected.color().range, ColorRange::Limited); + assert_eq!(selected.rotation_degrees(), Some(-90)); + assert!(selected.hdr().mastering_display().is_some()); + assert!(selected.hdr().content_light().is_some()); + assert!(selected.hdr().dolby_vision().is_some()); + let audio = parsed.selected_audio().expect("selected audio"); + assert_eq!(audio.codec_display(), "aac"); + assert_eq!(audio.profile_display(), Some("LC")); + assert_eq!(audio.sample_rate(), Some(48_000)); + assert_eq!(audio.channels(), Some(6)); + assert_eq!(audio.channel_layout(), Some("5.1")); + assert_eq!(parsed.subtitle_streams().count(), 1); + assert_eq!(parsed.chapters()[0].title(), Some("Part 1")); +} + +#[test] +fn missing_values_remain_typed_unknown_and_average_rate_alone_is_never_constant() { + let parsed = parse_probe_document(include_bytes!("fixtures/ffprobe/missing_unknown.json")) + .expect("parse missing-value fixture"); + let video = parsed.selected_video().expect("fallback first video"); + + assert_eq!(video.codec(), InputVideoCodec::OtherProbed); + assert_eq!(video.codec_display(), "unknown"); + assert_eq!(video.profile(), VideoProfile::Unknown); + assert_eq!(video.pixel_format(), PixelFormat::Unknown); + assert_eq!(video.chroma(), ChromaSubsampling::Unknown); + assert_eq!(video.frame_rate_class(), FrameRateClass::Unknown); + + let average_only = parse_probe_document( + br#"{"streams":[{"index":0,"codec_type":"video","avg_frame_rate":"30/1"}]}"#, + ) + .unwrap(); + assert_eq!( + average_only.selected_video().unwrap().frame_rate_class(), + FrameRateClass::Unknown + ); +} + +#[test] +fn monochrome_pixel_format_infers_the_exact_known_bit_depth() { + for (pixel_format, expected_format, expected_depth) in [ + ("gray", PixelFormat::Gray8, 8), + ("gray10le", PixelFormat::Gray10le, 10), + ] { + let json = format!( + r#"{{"streams":[{{"index":0,"codec_type":"video","codec_name":"h264","pix_fmt":"{pixel_format}"}}]}}"# + ); + let document = parse_probe_document(json.as_bytes()).unwrap(); + let video = document.selected_video().unwrap(); + assert_eq!(video.pixel_format(), expected_format); + assert_eq!(video.bit_depth(), Some(expected_depth)); + assert_eq!(video.chroma(), ChromaSubsampling::Monochrome); + } +} + +#[test] +fn malformed_deep_and_oversized_documents_fail_closed() { + assert!(parse_probe_document(br#"{"streams":[}"#).is_err()); + let deep = format!("{}0{}", "[".repeat(40), "]".repeat(40)); + assert!(parse_probe_document(deep.as_bytes()).is_err()); + assert!(parse_probe_document(&vec![b' '; 8 * 1024 * 1024 + 1]).is_err()); + assert!( + parse_probe_document( + br#"{"streams":[{"index":0,"codec_type":"video","side_data_list":[ + {"side_data_type":"Content light level metadata","max_content":1000}, + {"side_data_type":"Content light level metadata","max_content":2000} + ]}]}"# + ) + .is_err() + ); +} + +#[test] +fn typed_media_signature_changes_with_authorizing_video_fields() { + let original = include_bytes!("fixtures/ffprobe/codec_matrix.json"); + let first = parse_probe_document(original).unwrap(); + let original_text = String::from_utf8(original.to_vec()).unwrap(); + for (name, from, to) in [ + ( + "codec", + "\"codec_name\": \"h264\"", + "\"codec_name\": \"hevc\"", + ), + ( + "sample entry", + "\"codec_tag_string\": \"avc1\"", + "\"codec_tag_string\": \"avc3\"", + ), + ("profile", "\"profile\": \"High\"", "\"profile\": \"Main\""), + ( + "bit depth", + "\"bits_per_raw_sample\": \"8\"", + "\"bits_per_raw_sample\": \"10\"", + ), + ( + "pixel format", + "\"pix_fmt\": \"yuv420p\"", + "\"pix_fmt\": \"nv12\"", + ), + ( + "chroma", + "\"pix_fmt\": \"yuv420p\"", + "\"pix_fmt\": \"yuv422p\"", + ), + ( + "color", + "\"color_primaries\": \"bt709\"", + "\"color_primaries\": \"bt2020\"", + ), + ( + "frame rate", + "\"r_frame_rate\": \"24000/1001\"", + "\"r_frame_rate\": \"25/1\"", + ), + ( + "container", + "\"format_name\": \"matroska,webm\"", + "\"format_name\": \"mov,mp4,m4a,3gp,3g2,mj2\"", + ), + ] { + let changed = original_text.replacen(from, to, 1); + assert_ne!(changed, original_text, "fixture mutation exists: {name}"); + let second = parse_probe_document(changed.as_bytes()).unwrap(); + assert_ne!( + first.media_signature(), + second.media_signature(), + "signature field: {name}" + ); + } +} + +#[test] +fn unknown_probe_text_does_not_fan_out_typed_media_signatures() { + let first = parse_probe_document( + br#"{ + "format":{"format_name":"future_container_a"}, + "streams":[{ + "index":4,"codec_type":"video","codec_name":"future_codec_a", + "codec_tag_string":"future_tag_a","profile":"future_profile_a", + "pix_fmt":"future_pixel_a","width":1920,"height":1080, + "r_frame_rate":"24/1","avg_frame_rate":"24/1", + "tags":{"language":"language_a"},"disposition":{"default":1} + }], + "chapters":[{"id":0,"tags":{"title":"title_a"}}] + }"#, + ) + .unwrap(); + let second = parse_probe_document( + br#"{ + "format":{"format_name":"future_container_b"}, + "streams":[{ + "index":4,"codec_type":"video","codec_name":"future_codec_b", + "codec_tag_string":"future_tag_b","profile":"future_profile_b", + "pix_fmt":"future_pixel_b","width":1920,"height":1080, + "r_frame_rate":"24/1","avg_frame_rate":"24/1", + "tags":{"language":"language_b"},"disposition":{"default":1} + }], + "chapters":[{"id":0,"tags":{"title":"title_b"}}] + }"#, + ) + .unwrap(); + + assert_eq!(first.media_signature(), second.media_signature()); +} + +#[test] +fn duplicate_stream_ids_fail_and_attached_pictures_are_not_selected_as_video() { + assert!( + parse_probe_document( + br#"{"streams":[{"index":1,"codec_type":"video"},{"index":1,"codec_type":"audio"}]}"# + ) + .is_err() + ); + let document = r#"{"streams":[ + {"index":0,"codec_type":"video","codec_name":"mjpeg","disposition":{"default":1,"attached_pic":1}}, + {"index":2,"codec_type":"video","codec_name":"h264","tags":{"rotate":"180"},"disposition":{"default":0}} + ],"chapters":[{"id":0,"tags":{"title":"日本語の章"}}]}"#; + let parsed = parse_probe_document(document.as_bytes()).unwrap(); + assert_eq!(parsed.selected_video_stream(), Some(2)); + assert_eq!( + parsed.selected_video().unwrap().rotation_degrees(), + Some(180) + ); +} diff --git a/server/tests/transcoding_runtime.rs b/server/tests/transcoding_runtime.rs index f5a572db..d10b85f9 100644 --- a/server/tests/transcoding_runtime.rs +++ b/server/tests/transcoding_runtime.rs @@ -4108,8 +4108,10 @@ fn workflow_reader_cleanup_helper_command(ready: &Path) -> Command { fn wait_for_workflow_reader_helper(ready: &Path) -> std::net::SocketAddr { let deadline = Instant::now() + Duration::from_secs(5); loop { - if let Ok(address) = fs::read_to_string(ready) { - return address.parse().expect("helper listener address"); + if let Ok(address) = fs::read_to_string(ready) + && let Ok(address) = address.parse() + { + return address; } assert!( Instant::now() < deadline, @@ -4119,6 +4121,27 @@ fn wait_for_workflow_reader_helper(ready: &Path) -> std::net::SocketAddr { } } +#[test] +fn workflow_reader_wait_ignores_partial_address_publication() { + let temporary = tempfile::tempdir().expect("partial readiness fixture"); + let ready = temporary.path().join("ready"); + fs::write(&ready, "127.0.0.").expect("publish partial helper address"); + + let completed_ready = ready.clone(); + let writer = thread::spawn(move || { + thread::sleep(Duration::from_millis(20)); + fs::write(completed_ready, "127.0.0.1:31415").expect("publish complete helper address"); + }); + + assert_eq!( + wait_for_workflow_reader_helper(&ready), + "127.0.0.1:31415" + .parse() + .expect("expected helper listener address") + ); + writer.join().expect("readiness writer must finish"); +} + #[test] fn reader_spawn_failure_kills_and_waits_for_the_owned_git_child() { let temporary = tempfile::tempdir().expect("reader spawn failure fixture"); diff --git a/server/tests/transcoding_source.rs b/server/tests/transcoding_source.rs new file mode 100644 index 00000000..8a2a4ff8 --- /dev/null +++ b/server/tests/transcoding_source.rs @@ -0,0 +1,48 @@ +use std::ops::Range; + +use stream_server::transcoding::{SourceActivitySnapshot, SourceProtocolPolicy}; + +#[test] +fn source_activity_snapshot_starts_monotonic_and_unblocked() { + let snapshot = SourceActivitySnapshot::default(); + + assert_eq!(snapshot.sequence, 0); + assert_eq!(snapshot.delivered_bytes_total, 0); + assert_eq!(snapshot.active_requests, 0); + assert!(snapshot.waiting_for_pieces.is_empty()); + assert!(!snapshot.all_active_requests_piece_blocked); +} + +#[test] +fn source_protocol_policies_are_closed_and_source_specific() { + assert_eq!( + SourceProtocolPolicy::CompletedFile.ffmpeg_allowlist(), + "file,pipe" + ); + assert_eq!( + SourceProtocolPolicy::SyntheticFixture.ffmpeg_allowlist(), + "file,pipe" + ); + assert_eq!( + SourceProtocolPolicy::EngineLoopback.ffmpeg_allowlist(), + "http,tcp" + ); + assert_eq!( + SourceProtocolPolicy::ApprovedRemote.ffmpeg_allowlist(), + "http,tcp" + ); +} + +#[test] +fn activity_snapshot_keeps_exact_half_open_ranges() { + let range: Range = 16..32; + let snapshot = SourceActivitySnapshot { + sequence: 7, + delivered_bytes_total: 16, + active_requests: 1, + waiting_for_pieces: vec![range.clone()], + all_active_requests_piece_blocked: true, + }; + + assert_eq!(snapshot.waiting_for_pieces, vec![range]); +}