diff --git a/docs/core-protocol.md b/docs/core-protocol.md index 36aa4816a..32fd89570 100644 --- a/docs/core-protocol.md +++ b/docs/core-protocol.md @@ -796,20 +796,31 @@ reuse that discovery to stop a seat. CloudMatch instead of treating the first empty regional response as authoritative. Discovery deduplicates session IDs and uses at most four concurrent requests, a three-second per-request timeout, a 12-second total budget (including region -discovery), and at most 32 regional endpoints. If no sessions were found but a -region failed, returned invalid data, or could not be checked within these bounds, -the request fails with `session_discovery_failed` rather than returning an empty -list. HTTP 401 remains `http_unauthorized` after at most one safe-read renewal; -HTTP 403 remains `authentication_required`. A found session -can be returned even when another region fails; an empty successful list means -all discovered regions were checked successfully. Reconnect, reconciliation, and -known-conflict discovery remain strict: callers must not treat a discovery failure -as proof that an existing session has ended. For a current new-launch intent with -no active or known conflicting session, Qt may revalidate the selected game and -attempt `session.create` after `session_discovery_failed`. CloudMatch's -`session_conflict` response still redirects that attempt to resume/end handling; -discovery failures after a known conflict never trigger another create. Authentication -errors, cancellation, and request timeouts do not use this fallback. +discovery), and at most 32 regional endpoints. While no session has been found and +no region rejected the credentials, a region or region-discovery request that failed +transiently is retried once with a 1.5-second timeout. A fast failure is retried 750 +ms later, and a timeout, which has already waited three seconds, at once. The retry +runs only if more than the delay plus 1.5 seconds of the budget remain, so a region +that keeps timing out fails the sweep after about 4.5 seconds. Transient means HTTP +408, 425, 429, 500, 502, 503, or 504 (the statuses session polling retries) or a +transport failure such as a timeout or a closed connection; other HTTP statuses, +invalid bodies, and rejected `requestStatus` values are final. If no sessions were +found but a region failed, returned invalid data, or could not be checked within +these bounds, even after that retry, the request fails with +`session_discovery_failed` rather than returning an empty list. HTTP 401 remains +`http_unauthorized` after at most one safe-read renewal; HTTP 403 remains +`authentication_required`. A found session can be returned even when another region +fails; an empty successful list means all discovered regions were checked +successfully. Reconnect, reconciliation, and known-conflict discovery remain strict: +callers must not treat a discovery failure as proof that an existing session has +ended. For a current new-launch intent with no active or known conflicting session, +Qt may revalidate the selected game and attempt `session.create` after +`session_discovery_failed`. CloudMatch's `session_conflict` response still redirects +that attempt to resume/end handling; discovery failures after a known conflict never +trigger another create. Authentication errors, cancellation, and request timeouts do +not use this fallback. Each sweep records one bounded `[session] discovery:` +diagnostics line with each failed request's kind and attempt number, naming regions +by CloudMatch metadata key, never by URL. ### Provider routing and account scope diff --git a/native/opennow-core/src/cloudmatch.rs b/native/opennow-core/src/cloudmatch.rs index bcb170c7e..b648a54f4 100644 --- a/native/opennow-core/src/cloudmatch.rs +++ b/native/opennow-core/src/cloudmatch.rs @@ -23,6 +23,16 @@ const NETWORK_TEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(8); const MAXIMUM_NETWORK_TEST_RESPONSE_BYTES: u64 = 4 * 1024 * 1024; const MAX_DISCOVERY_REGIONS: usize = 32; const DISCOVERY_CONCURRENCY: usize = 4; +/// Sweep summaries kept until an RPC records them; bounds memory if nothing drains them. +const DISCOVERY_SUMMARY_BACKLOG: usize = 4; +/// Pause before retrying a fast failure, such as a refused or reset connection, so a transient +/// condition can clear. A timed-out request has already waited out its timeout. +const DISCOVERY_RETRY_DELAY: Duration = Duration::from_millis(750); +/// Per-request timeout of the retry, and the budget that must remain after the delay for the +/// retry to run. A region that answers normally replies well within it, and it bounds what a +/// region that keeps timing out adds to a failing sweep. +const DISCOVERY_RETRY_REQUEST_TIMEOUT: Duration = Duration::from_millis(1500); +const MAX_DISCOVERY_LOG_FAILURES: usize = 6; const MAX_CLEANUP_RECORD_BYTES: usize = 16 * 1024; fn webrtc_compatibility_enabled(settings: &Value, auth: &AuthSession) -> bool { @@ -205,6 +215,7 @@ pub struct CloudMatchService { client: Client, active: Mutex>, discovered: Mutex>, + discovery_summaries: Mutex>, conflict: Mutex>, allocation_admission: Mutex<()>, fresh: Mutex>, @@ -269,6 +280,7 @@ impl CloudMatchService { client, active: Mutex::new(None), discovered: Mutex::new(HashMap::new()), + discovery_summaries: Mutex::new(Vec::new()), conflict: Mutex::new(None), allocation_admission: Mutex::new(()), fresh: Mutex::new(None), @@ -1251,81 +1263,35 @@ impl CloudMatchService { .expect("CloudMatch unresolved state poisoned") .clone() .filter(|record| record.matches(auth, device_id)); - let mut bases = vec![requested.clone()]; + // The region of an unresolved allocation is checked even when serverInfo omits it. + let mut regions = vec![("selected".to_owned(), requested.clone())]; if let Some(base) = unresolved .as_ref() .and_then(|record| record.verified_base(auth)) - && base != requested + .filter(|base| *base != requested) { - bases.push(base); + regions.push(("unresolved".to_owned(), base)); } let server_info_url = requested .join("v2/serverInfo") .map_err(|_| invalid("Invalid server-info URL"))?; #[cfg(test)] let server_info_url = self.fixture_url(server_info_url); - let server_info = client - .get(server_info_url) - .headers(headers.clone()) - .timeout(DISCOVERY_REQUEST_TIMEOUT) - .send() - .map_err(|error| network("Region discovery failed", error)) - .and_then(|response| { - if !response.status().is_success() { - return Err(response_error("Region discovery failed", response)); - } - let payload = response - .json::() - .map_err(|error| network("Invalid region response", error))?; - if payload["metaData"].as_array().is_none() - || (payload.get("requestStatus").is_some() - && value_i64(&payload["requestStatus"]["statusCode"]) != Some(1)) - { - return Err(upstream("Invalid region response")); - } - Ok(payload) - }); - if let Err(error) = &server_info - && matches!(error.code, "authentication_required" | "http_unauthorized") - { - return Err(error.clone()); - } - if let Ok(payload) = &server_info { - for base in regional_bases(payload) { - if !bases.contains(&base) { - bases.push(base); - } - } - } - let incomplete = server_info.is_err() || bases.len() > MAX_DISCOVERY_REGIONS; - bases.truncate(MAX_DISCOVERY_REGIONS); - let (mut sessions, complete) = - discover_sessions_with_status(&bases, deadline, incomplete, |base, timeout| { - let url = base - .join("v2/session") - .map_err(|_| invalid("Invalid active-session URL"))?; - #[cfg(test)] - let url = self.fixture_url(url); - let response = client - .get(url) - .headers(headers.clone()) - .timeout(timeout) - .send() - .map_err(|error| network("Active-session discovery failed", error))?; - let payload = - read_cloudmatch_response("Active-session discovery failed", response, false)?; - let sessions = payload["sessions"] - .as_array() - .ok_or_else(|| upstream("Invalid active-session response"))?; - sessions - .iter() - .filter(|session| value_i64(&session["status"]) != Some(7)) - .map(|session| { - remote_session_info(session, base) - .ok_or_else(|| upstream("Invalid active-session entry")) - }) - .collect() - })?; + let mut log = DiscoveryLog::default(); + let sweep = self.discover_remote_sessions( + &client, + &headers, + regions, + server_info_url, + deadline, + &mut log, + ); + self.push_discovery_summary(log.summary()); + let DiscoverySweep { + mut sessions, + complete, + bases, + } = sweep?; let owner = (auth.provider.idp_id.clone(), auth.user.user_id.clone()); if let Some(record) = unresolved && complete @@ -1395,6 +1361,100 @@ impl CloudMatchService { } } + /// Queues one sweep's diagnostics summary. Every sweep keeps its own line, including the + /// first sweep of a token-renewal retry; only the oldest lines beyond the bound are dropped. + fn push_discovery_summary(&self, summary: String) { + let mut summaries = self + .discovery_summaries + .lock() + .expect("CloudMatch discovery state poisoned"); + if summaries.len() >= DISCOVERY_SUMMARY_BACKLOG { + summaries.remove(0); + } + summaries.push(summary); + } + + /// Returns the diagnostics summaries of the discovery sweeps since the last call, oldest + /// first. + pub(crate) fn take_discovery_summaries(&self) -> Vec { + std::mem::take( + &mut *self + .discovery_summaries + .lock() + .expect("CloudMatch discovery state poisoned"), + ) + } + + fn discover_remote_sessions( + &self, + client: &Client, + headers: &HeaderMap, + mut regions: Vec<(String, Url)>, + server_info_url: Url, + deadline: Instant, + log: &mut DiscoveryLog, + ) -> Result { + let mut server_info = fetch_server_info( + client, + server_info_url.clone(), + headers, + DISCOVERY_REQUEST_TIMEOUT, + ); + log.server_info.push(attempt_outcome(&server_info)); + if let Err((_, kind)) = &server_info + && kind.is_transient() + && has_retry_budget(deadline, retry_delay([*kind])) + { + crate::requests::current().sleep(retry_delay([*kind]))?; + let timeout = deadline + .saturating_duration_since(Instant::now()) + .min(DISCOVERY_RETRY_REQUEST_TIMEOUT); + server_info = fetch_server_info(client, server_info_url, headers, timeout); + log.server_info.push(attempt_outcome(&server_info)); + } + let server_info = server_info.map_err(|(error, _)| error); + if let Err(error) = &server_info + && is_auth_failure(error) + { + return Err(error.clone()); + } + if let Ok(payload) = &server_info { + for (name, base) in regional_bases(payload) { + match regions.iter_mut().find(|(_, known)| known == &base) { + Some(region) => region.0 = name, + None => regions.push((name, base)), + } + } + } + let incomplete = server_info.is_err() || regions.len() > MAX_DISCOVERY_REGIONS; + regions.truncate(MAX_DISCOVERY_REGIONS); + let (names, bases): (Vec<_>, Vec<_>) = regions.into_iter().unzip(); + log.attempts = Mutex::new(vec![Vec::new(); names.len()]); + log.regions = names; + let log = &*log; + let (sessions, complete) = + discover_sessions_with_status(&bases, deadline, incomplete, |base, timeout| { + let url = base.join("v2/session").map_err(|_| { + ( + invalid("Invalid active-session URL"), + FailureKind::Unchecked, + ) + })?; + #[cfg(test)] + let url = self.fixture_url(url); + let result = fetch_region_sessions(client, url, headers, base, timeout); + if let Some(index) = bases.iter().position(|known| known == base) { + log.record(index, attempt_outcome(&result)); + } + result + })?; + Ok(DiscoverySweep { + sessions, + complete, + bases, + }) + } + pub fn claim( &self, params: &Value, @@ -1738,13 +1798,7 @@ impl CloudMatchService { let mut last_error = None; for attempt in 0..=2 { match client.get(url.clone()).headers(headers.clone()).send() { - Ok(response) - if attempt < 2 - && matches!( - response.status().as_u16(), - 408 | 425 | 429 | 500 | 502 | 503 | 504 - ) => - { + Ok(response) if attempt < 2 && is_transient_status(response.status().as_u16()) => { thread::sleep(Duration::from_millis(if attempt == 0 { 250 } else { 750 })); } Ok(response) => { @@ -1812,6 +1866,7 @@ impl CloudMatchService { }; Ok(regional_bases(&payload) .into_iter() + .map(|(_, base)| base) .find(|base| { !prefer_regional || !base.host_str().unwrap_or_default().starts_with("np-") }) @@ -2959,7 +3014,7 @@ fn discover_sessions( bases: &[Url], deadline: Instant, incomplete: bool, - fetch: impl Fn(&Url, Duration) -> Result, ServiceError> + Sync, + fetch: impl Fn(&Url, Duration) -> Result, RegionFailure> + Sync, ) -> Result, ServiceError> { discover_sessions_with_status(bases, deadline, incomplete, fetch).map(|(sessions, _)| sessions) } @@ -2968,30 +3023,115 @@ fn discover_sessions_with_status( bases: &[Url], deadline: Instant, mut incomplete: bool, - fetch: impl Fn(&Url, Duration) -> Result, ServiceError> + Sync, + fetch: impl Fn(&Url, Duration) -> Result, RegionFailure> + Sync, ) -> Result<(Vec, bool), ServiceError> { - let fetch = &fetch; let cancellation = crate::requests::current(); + let all = (0..bases.len()).collect::>(); + let mut results = fetch_regions( + bases, + &all, + deadline, + DISCOVERY_REQUEST_TIMEOUT, + &cancellation, + &fetch, + ); + cancellation.check()?; + // A single transient regional failure must not fail discovery on its own. While nothing was + // found and no region rejected the credentials, retry the transiently failed regions once, + // if the budget still leaves room for a useful attempt. Fast failures wait a short delay + // first; timeouts are retried at once. The retry has a shorter timeout, so a region that + // keeps timing out adds at most DISCOVERY_RETRY_REQUEST_TIMEOUT. Other failures stay final, + // so an incomplete sweep still fails closed below. + let transient = all + .into_iter() + .filter(|index| { + results[*index] + .as_ref() + .is_err_and(|(_, kind)| kind.is_transient()) + }) + .collect::>(); + let delay = retry_delay( + transient + .iter() + .filter_map(|index| results[*index].as_ref().err().map(|(_, kind)| *kind)), + ); + if !transient.is_empty() + && !results + .iter() + .any(|result| result.as_ref().is_ok_and(|found| !found.is_empty())) + && !results.iter().any(|result| { + result + .as_ref() + .is_err_and(|(error, _)| is_auth_failure(error)) + }) + && has_retry_budget(deadline, delay) + { + cancellation.sleep(delay)?; + let retried = fetch_regions( + bases, + &transient, + deadline, + DISCOVERY_RETRY_REQUEST_TIMEOUT, + &cancellation, + &fetch, + ); + cancellation.check()?; + for (index, result) in transient.into_iter().zip(retried) { + results[index] = result; + } + } + let mut sessions = Vec::new(); + for result in results { + match result { + Ok(found) => { + for session in found { + if !sessions + .iter() + .any(|known: &Value| known["sessionId"] == session["sessionId"]) + { + sessions.push(session); + } + } + } + Err((error, _)) if is_auth_failure(&error) => return Err(error), + Err(_) => incomplete = true, + } + } + if sessions.is_empty() && (incomplete || bases.is_empty()) { + return Err(discovery_failed()); + } + Ok((sessions, !incomplete && !bases.is_empty())) +} + +/// Fetches `bases[index]` for each of `indices` with bounded concurrency, each request limited +/// to `request_timeout` and the deadline. Results are returned in the order of `indices`. +fn fetch_regions( + bases: &[Url], + indices: &[usize], + deadline: Instant, + request_timeout: Duration, + cancellation: &crate::requests::Cancellation, + fetch: &(impl Fn(&Url, Duration) -> Result, RegionFailure> + Sync), +) -> Vec, RegionFailure>> { let mut results = thread::scope(|scope| { - let workers = (0..DISCOVERY_CONCURRENCY.min(bases.len())) + let workers = (0..DISCOVERY_CONCURRENCY.min(indices.len())) .map(|worker| { - let cancellation = cancellation.clone(); scope.spawn(move || { - bases + indices .iter() .enumerate() .skip(worker) .step_by(DISCOVERY_CONCURRENCY) - .map(|(index, base)| { + .map(|(position, index)| { let remaining = deadline.saturating_duration_since(Instant::now()); - let result = cancellation.check().and_then(|()| { - if remaining.is_zero() { - Err(discovery_failed()) - } else { - fetch(base, remaining.min(DISCOVERY_REQUEST_TIMEOUT)) + let result = match cancellation.check() { + Err(error) => Err((error, FailureKind::Unchecked)), + Ok(()) if remaining.is_zero() => { + Err((discovery_failed(), FailureKind::Unchecked)) } - }); - (index, result) + Ok(()) => fetch(&bases[*index], remaining.min(request_timeout)), + }; + (position, result) }) .collect::>() }) @@ -3002,31 +3142,312 @@ fn discover_sessions_with_status( .flat_map(|worker| worker.join().expect("CloudMatch discovery worker panicked")) .collect::>() }); - cancellation.check()?; - results.sort_by_key(|(index, _)| *index); - let mut sessions = Vec::new(); - for (_, result) in results { - match result { - Ok(found) => { - for session in found { - if !sessions - .iter() - .any(|known: &Value| known["sessionId"] == session["sessionId"]) - { - sessions.push(session); - } - } + results.sort_by_key(|(position, _)| *position); + results.into_iter().map(|(_, result)| result).collect() +} + +/// One discovery sweep: the sessions found, whether every region was checked, and the regional +/// endpoints it covered. +struct DiscoverySweep { + sessions: Vec, + complete: bool, + bases: Vec, +} + +/// A failed regional request: the error for the caller and its URL-free classification. +type RegionFailure = (ServiceError, FailureKind); + +fn is_auth_failure(error: &ServiceError) -> bool { + matches!(error.code, "authentication_required" | "http_unauthorized") +} + +fn has_retry_budget(deadline: Instant, delay: Duration) -> bool { + deadline.saturating_duration_since(Instant::now()) > delay + DISCOVERY_RETRY_REQUEST_TIMEOUT +} + +/// The pause before a retry: none when every failure being retried was a timeout, which already +/// waited out its request, otherwise `DISCOVERY_RETRY_DELAY`. +fn retry_delay(kinds: impl IntoIterator) -> Duration { + if kinds.into_iter().all(|kind| kind == FailureKind::Timeout) { + Duration::ZERO + } else { + DISCOVERY_RETRY_DELAY + } +} + +/// HTTP statuses after which one more CloudMatch read can succeed. Session polling and +/// discovery retry the same set. +fn is_transient_status(status: u16) -> bool { + matches!(status, 408 | 425 | 429 | 500 | 502 | 503 | 504) +} + +fn fetch_server_info( + client: &Client, + url: Url, + headers: &HeaderMap, + timeout: Duration, +) -> Result { + let response = client + .get(url) + .headers(headers.clone()) + .timeout(timeout) + .send() + .map_err(|error| request_failure("Region discovery failed", error))?; + let status = response.status(); + if !status.is_success() { + return Err(( + response_error("Region discovery failed", response), + FailureKind::Http(status.as_u16()), + )); + } + let payload = response + .json::() + .map_err(|error| request_failure("Invalid region response", error))?; + if payload["metaData"].as_array().is_none() { + return Err((upstream("Invalid region response"), FailureKind::Body)); + } + if payload.get("requestStatus").is_some() + && value_i64(&payload["requestStatus"]["statusCode"]) != Some(1) + { + return Err((upstream("Invalid region response"), FailureKind::Rejected)); + } + Ok(payload) +} + +fn fetch_region_sessions( + client: &Client, + url: Url, + headers: &HeaderMap, + base: &Url, + timeout: Duration, +) -> Result, (ServiceError, FailureKind)> { + const CONTEXT: &str = "Active-session discovery failed"; + let response = client + .get(url) + .headers(headers.clone()) + .timeout(timeout) + .send() + .map_err(|error| request_failure(CONTEXT, error))?; + let status = response.status(); + let payload = response.json::(); + let kind = match &payload { + _ if !status.is_success() => FailureKind::Http(status.as_u16()), + Err(error) => FailureKind::of(error), + Ok(_) => FailureKind::Rejected, + }; + let payload = validate_cloudmatch_response(CONTEXT, status, payload, false) + .map_err(|error| (error, kind))?; + let sessions = payload["sessions"].as_array().ok_or_else(|| { + ( + upstream("Invalid active-session response"), + FailureKind::Body, + ) + })?; + sessions + .iter() + .filter(|session| value_i64(&session["status"]) != Some(7)) + .map(|session| { + remote_session_info(session, base) + .ok_or_else(|| (upstream("Invalid active-session entry"), FailureKind::Body)) + }) + .collect() +} + +fn request_failure(context: &str, error: reqwest::Error) -> (ServiceError, FailureKind) { + (network(context, &error), FailureKind::of(&error)) +} + +fn attempt_outcome(result: &Result) -> Result<(), FailureKind> { + result.as_ref().map(|_| ()).map_err(|(_, kind)| *kind) +} + +/// URL-free classification of one failed discovery request, for diagnostics. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FailureKind { + Http(u16), + Timeout, + Dns, + Tls, + Connect, + /// The connection was closed or reset under the request, as with a stale keep-alive + /// connection. + Closed, + /// Invalid JSON or a payload without the expected fields. + Body, + /// CloudMatch answered with a non-success `requestStatus`. + Rejected, + /// Another transport failure. + Network, + /// Never sent: the deadline or the request cancellation ended the sweep first. + Unchecked, +} + +impl FailureKind { + /// Whether one more attempt can plausibly succeed. As in session polling, that is the + /// transient HTTP statuses and any transport failure. Other statuses, malformed bodies, + /// rejected requests and unsent requests are final. + fn is_transient(self) -> bool { + match self { + Self::Http(status) => is_transient_status(status), + Self::Timeout + | Self::Dns + | Self::Tls + | Self::Connect + | Self::Closed + | Self::Network => true, + Self::Body | Self::Rejected | Self::Unchecked => false, + } + } + + /// Classifies a failed send or body read. The `timeout`, `dns`, `tls` and `connect` stages + /// come from the shared GFN transport classifier; discovery adds closed connections and + /// invalid bodies. + fn of(error: &reqwest::Error) -> Self { + match crate::gfn::transport_failure(error) { + Some("timeout") => Self::Timeout, + Some("dns") => Self::Dns, + Some("tls") => Self::Tls, + Some(_) => Self::Connect, + None if connection_closed(error) => Self::Closed, + None if error.is_decode() => Self::Body, + None => Self::Network, + } + } +} + +/// Whether the connection was closed or reset under the request. hyper reports a pooled +/// keep-alive connection that the server closed first as "connection closed before message +/// completed". +fn connection_closed(error: &reqwest::Error) -> bool { + let mut source = std::error::Error::source(error); + while let Some(cause) = source { + if cause.to_string().starts_with("connection closed") { + return true; + } + source = match cause.downcast_ref::() { + Some(io) + if matches!( + io.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + ) => + { + return true; + } + Some(io) => io.get_ref().map(|inner| inner as _), + None => cause.source(), + }; + } + false +} + +impl std::fmt::Display for FailureKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Http(status) => write!(formatter, "http={status}"), + Self::Timeout => formatter.write_str("timeout"), + Self::Dns => formatter.write_str("dns"), + Self::Tls => formatter.write_str("tls"), + Self::Connect => formatter.write_str("connect"), + Self::Closed => formatter.write_str("closed"), + Self::Body => formatter.write_str("body"), + Self::Rejected => formatter.write_str("rejected"), + Self::Network => formatter.write_str("network"), + Self::Unchecked => formatter.write_str("unchecked"), + } + } +} + +type Attempts = Vec>; + +/// Bounded record of one discovery sweep for `current.log`. It names regions by their +/// CloudMatch metadata key and never holds URLs, tokens or error messages. +#[derive(Default)] +struct DiscoveryLog { + server_info: Attempts, + regions: Vec, + attempts: Mutex>, +} + +impl DiscoveryLog { + fn record(&self, region: usize, outcome: Result<(), FailureKind>) { + if let Some(attempts) = self + .attempts + .lock() + .expect("CloudMatch discovery log poisoned") + .get_mut(region) + { + attempts.push(outcome); + } + } + + /// For example `serverInfo=ok regions=17 failed=1 retried=1 recovered=1 + /// failures=NP-AMS-03:closed#1`. Attempts are numbered from 1; a region that was never + /// requested (deadline or cancellation) is reported as `unchecked`. + fn summary(self) -> String { + let attempts = self + .attempts + .into_inner() + .expect("CloudMatch discovery log poisoned"); + let failures_of = |name: &str, attempts: &Attempts| { + attempts + .iter() + .enumerate() + .filter_map(|(attempt, outcome)| { + outcome + .err() + .map(|kind| format!("{name}:{kind}#{}", attempt + 1)) + }) + .collect::>() + }; + let mut failures = failures_of("serverInfo", &self.server_info); + let (mut failed, mut retried, mut recovered) = (0, 0, 0); + for (index, (name, attempts)) in self.regions.iter().zip(&attempts).enumerate() { + let name = region_label(index, name); + failed += usize::from(attempts.first().is_none_or(Result::is_err)); + retried += usize::from(attempts.len() > 1); + recovered += + usize::from(attempts.len() > 1 && attempts.last().is_some_and(Result::is_ok)); + if attempts.is_empty() { + failures.push(format!("{name}:unchecked#1")); } - Err(error) if matches!(error.code, "authentication_required" | "http_unauthorized") => { - return Err(error); + failures.extend(failures_of(&name, attempts)); + } + let server_info = match self.server_info.as_slice() { + [Ok(())] => "ok", + [Err(_), Ok(())] => "recovered", + _ => "failed", + }; + let mut summary = format!( + "serverInfo={server_info} regions={} failed={failed} retried={retried} recovered={recovered}", + self.regions.len() + ); + if !failures.is_empty() { + let omitted = failures.len().saturating_sub(MAX_DISCOVERY_LOG_FAILURES); + failures.truncate(MAX_DISCOVERY_LOG_FAILURES); + if omitted > 0 { + failures.push(format!("+{omitted}")); } - Err(_) => incomplete = true, + summary.push_str(" failures="); + summary.push_str(&failures.join(",")); } + summary } - if sessions.is_empty() && (incomplete || bases.is_empty()) { - return Err(discovery_failed()); +} + +/// Logs a region by its metadata key only when that key looks like a region identifier. +fn region_label(index: usize, name: &str) -> String { + if (1..=24).contains(&name.len()) + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + name.to_owned() + } else { + format!("region{index}") } - Ok((sessions, !incomplete && !bases.is_empty())) } fn discovery_failed() -> ServiceError { @@ -3037,7 +3458,8 @@ fn discovery_failed() -> ServiceError { } } -fn regional_bases(payload: &Value) -> Vec { +/// Returns the advertised regional endpoints with their metadata names, local region first. +fn regional_bases(payload: &Value) -> Vec<(String, Url)> { let metadata = payload["metaData"].as_array().cloned().unwrap_or_default(); let value_for = |key: &str| { metadata.iter().find_map(|entry| { @@ -3062,9 +3484,11 @@ fn regional_bases(payload: &Value) -> Vec { for name in names { if let Some(raw) = value_for(&name) && let Ok(base) = trusted_cloudmatch_base(&raw) - && !result.iter().any(|existing: &Url| existing == &base) + && !result + .iter() + .any(|(_, existing): &(String, Url)| existing == &base) { - result.push(base); + result.push((name, base)); } } result @@ -5997,7 +6421,7 @@ mod tests { false, |base, _| { if base == &bases[0] { - Err(upstream("failed region")) + Err(failure(FailureKind::Body)) } else { Ok(if found { vec![json!({"sessionId":"seat"})] @@ -6033,7 +6457,7 @@ mod tests { ); assert_eq!( discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { - Err(upstream("failed")) + Err(failure(FailureKind::Body)) }) .unwrap_err() .code, @@ -6081,10 +6505,13 @@ mod tests { fn discovery_preserves_authentication_failures_and_cancellation() { let bases = [trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap()]; let error = discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { - Err(ServiceError { - code: "authentication_required", - message: "Expired credentials".to_owned(), - }) + Err(( + ServiceError { + code: "authentication_required", + message: "Expired credentials".to_owned(), + }, + FailureKind::Http(403), + )) }) .unwrap_err(); assert_eq!(error.code, "authentication_required"); @@ -6101,6 +6528,479 @@ mod tests { assert_eq!(error.code, "cancelled"); } + fn retry_bases() -> [Url; 3] { + [ + "https://first.nvidiagrid.net", + "https://second.nvidiagrid.net", + "https://third.nvidiagrid.net", + ] + .map(|url| trusted_cloudmatch_base(url).unwrap()) + } + + /// A failed regional request of the given kind, with the error the caller would see. + fn failure(kind: FailureKind) -> RegionFailure { + (network("Active-session discovery failed", kind), kind) + } + + /// Runs discovery over `bases` and returns the result with the number of calls per base. + fn discover_counting( + bases: &[Url], + deadline: Instant, + fetch: impl Fn(usize, usize) -> Result, RegionFailure> + Sync, + ) -> (Result, ServiceError>, Vec) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let calls = bases + .iter() + .map(|_| AtomicUsize::new(0)) + .collect::>(); + let result = discover_sessions(bases, deadline, false, |base, timeout| { + assert!(timeout <= DISCOVERY_REQUEST_TIMEOUT); + let index = bases.iter().position(|known| known == base).unwrap(); + fetch(index, calls[index].fetch_add(1, Ordering::SeqCst)) + }); + let calls = calls.iter().map(|calls| calls.load(Ordering::SeqCst)); + (result, calls.collect()) + } + + #[test] + fn discovery_retries_a_transient_region_failure_once() { + let started = Instant::now(); + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, call| { + if index == 0 && call == 0 { + Err(failure(FailureKind::Closed)) + } else { + Ok(vec![]) + } + }, + ); + assert!(result.unwrap().is_empty()); + assert_eq!(calls, [2, 1, 1]); + assert!(started.elapsed() >= DISCOVERY_RETRY_DELAY); + } + + #[test] + fn a_region_that_keeps_timing_out_fails_after_one_shorter_retry() { + // A timed-out request has already waited out its timeout, so its retry skips the delay + // and gets the shorter retry timeout: about 4.5 s to fail instead of 3 + 0.75 + 3 s. + let bases = retry_bases(); + let timeouts = std::sync::Mutex::new(Vec::new()); + let started = Instant::now(); + let result = discover_sessions( + &bases, + Instant::now() + DISCOVERY_TIMEOUT, + false, + |base, timeout| { + if base != &bases[0] { + return Ok(vec![]); + } + timeouts.lock().unwrap().push(timeout); + thread::sleep(timeout); + Err(failure(FailureKind::Timeout)) + }, + ); + let elapsed = started.elapsed(); + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + assert_eq!( + *timeouts.lock().unwrap(), + [DISCOVERY_REQUEST_TIMEOUT, DISCOVERY_RETRY_REQUEST_TIMEOUT] + ); + assert!(elapsed >= DISCOVERY_REQUEST_TIMEOUT + DISCOVERY_RETRY_REQUEST_TIMEOUT); + // Below the delayed retry this replaces, with the delay's 750 ms as scheduling slack. + assert!( + elapsed + < DISCOVERY_REQUEST_TIMEOUT + + DISCOVERY_RETRY_DELAY + + DISCOVERY_RETRY_REQUEST_TIMEOUT + ); + } + + #[test] + fn discovery_still_fails_closed_when_the_retry_fails() { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| { + if index == 0 { + Err(failure(FailureKind::Closed)) + } else { + Ok(vec![]) + } + }, + ); + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + assert_eq!(calls, [2, 1, 1]); + } + + #[test] + fn discovery_does_not_retry_authentication_failures() { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| match index { + 0 => Err(( + ServiceError { + code: "http_unauthorized", + message: "Expired credentials".to_owned(), + }, + FailureKind::Http(401), + )), + 1 => Err(failure(FailureKind::Timeout)), + _ => Ok(vec![]), + }, + ); + assert_eq!(result.unwrap_err().code, "http_unauthorized"); + assert_eq!(calls, [1, 1, 1]); + } + + #[test] + fn discovery_does_not_retry_non_transient_regional_failures() { + for kind in [ + FailureKind::Http(400), + FailureKind::Http(404), + FailureKind::Http(501), + FailureKind::Body, + FailureKind::Rejected, + ] { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| { + if index == 0 { + Err(failure(kind)) + } else { + Ok(vec![]) + } + }, + ); + // The retry delay only runs before a retry, so one call per region also proves + // that no delay was spent. + assert_eq!( + result.unwrap_err().code, + "session_discovery_failed", + "{kind}" + ); + assert_eq!(calls, [1, 1, 1], "{kind}"); + } + } + + #[test] + fn failure_kinds_separate_transient_from_final_failures() { + // The HTTP statuses are the ones session polling (`get_session`) retries. + for kind in [ + FailureKind::Http(408), + FailureKind::Http(425), + FailureKind::Http(429), + FailureKind::Http(500), + FailureKind::Http(502), + FailureKind::Http(503), + FailureKind::Http(504), + FailureKind::Timeout, + FailureKind::Dns, + FailureKind::Tls, + FailureKind::Connect, + FailureKind::Closed, + FailureKind::Network, + ] { + assert!(kind.is_transient(), "{kind}"); + } + for kind in [ + FailureKind::Http(302), + FailureKind::Http(400), + FailureKind::Http(401), + FailureKind::Http(403), + FailureKind::Http(404), + FailureKind::Http(501), + FailureKind::Http(505), + FailureKind::Body, + FailureKind::Rejected, + FailureKind::Unchecked, + ] { + assert!(!kind.is_transient(), "{kind}"); + } + } + + #[test] + fn discovery_skips_the_retry_once_a_session_is_found() { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| match index { + 0 => Err(failure(FailureKind::Timeout)), + 1 => Ok(vec![json!({"sessionId":"seat"})]), + _ => Ok(vec![]), + }, + ); + assert_eq!(result.unwrap().len(), 1); + assert_eq!(calls, [1, 1, 1]); + } + + #[test] + fn discovery_skips_the_retry_without_budget() { + // Exactly the minimum budget at the start, so less than it remains after the first + // pass, while the first pass itself has ample time even on a loaded machine. + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_RETRY_DELAY + DISCOVERY_RETRY_REQUEST_TIMEOUT, + |index, _| { + if index == 0 { + Err(failure(FailureKind::Closed)) + } else { + Ok(vec![]) + } + }, + ); + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + assert_eq!(calls, [1, 1, 1]); + } + + #[test] + fn discovery_retry_delay_honours_cancellation() { + let bases = retry_bases(); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("discovery", "session.remote.list").unwrap(); + let regions = bases.len(); + let (fetched_tx, fetched_rx) = std::sync::mpsc::channel(); + let started = Instant::now(); + let (result, calls) = thread::scope(|scope| { + let requests = &requests; + scope.spawn(move || { + // Cancel once every region has been requested. This usually lands inside the + // retry delay; landing just before it must end discovery the same way, so a + // slow machine only loses timing margin, never the outcome. + for _ in 0..regions { + fetched_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + thread::sleep(Duration::from_millis(50)); + requests.cancel("discovery"); + }); + crate::requests::scope(permit.token.clone(), || { + discover_counting(&bases, Instant::now() + DISCOVERY_TIMEOUT, |index, _| { + let _ = fetched_tx.send(()); + if index == 0 { + Err(failure(FailureKind::Closed)) + } else { + Ok(vec![]) + } + }) + }) + }); + assert_eq!(result.unwrap_err().code, "cancelled"); + assert_eq!(calls, [1, 1, 1]); + assert!(started.elapsed() < DISCOVERY_RETRY_DELAY); + } + + #[test] + fn remote_sessions_retries_server_info_and_records_the_sweep() { + let session_list = json!({"requestStatus":{"statusCode":1},"sessions":[]}); + let (base, server) = session_server( + vec![ + (503, json!({})), + ( + 200, + json!({"metaData":[ + {"key":"local-region","value":"NP-TEST-01"}, + {"key":"NP-TEST-01","value":"https://np-test-01.nvidiagrid.net/"} + ]}), + ), + (200, session_list.clone()), + (200, session_list), + ], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.set_test_control_base(base); + let response = service + .remote_sessions(&json!({}), &json!({}), &conflict_auth(), "device") + .unwrap(); + assert_eq!(response, json!({"sessions":[]})); + assert_eq!( + server.join().unwrap(), + [ + "GET /v2/serverInfo HTTP/1.1", + "GET /v2/serverInfo HTTP/1.1", + "GET /v2/session HTTP/1.1", + "GET /v2/session HTTP/1.1", + ] + ); + assert_eq!( + service.take_discovery_summaries(), + [ + "serverInfo=recovered regions=2 failed=0 retried=0 recovered=0 \ + failures=serverInfo:http=503#1" + ] + ); + assert!(service.take_discovery_summaries().is_empty()); + } + + #[test] + fn remote_sessions_retries_a_transient_regional_failure() { + // The selected endpoint is also the only advertised region, so requests are sequential. + let (base, server) = session_server( + vec![ + ( + 200, + json!({"metaData":[ + {"key":"local-region","value":"NP-TEST-01"}, + {"key":"NP-TEST-01","value":DEFAULT_STREAMING_BASE} + ]}), + ), + (503, json!({})), + (200, json!({"requestStatus":{"statusCode":1},"sessions":[]})), + ], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.set_test_control_base(base); + let response = service + .remote_sessions(&json!({}), &json!({}), &conflict_auth(), "device") + .unwrap(); + assert_eq!(response, json!({"sessions":[]})); + assert_eq!( + server.join().unwrap(), + [ + "GET /v2/serverInfo HTTP/1.1", + "GET /v2/session HTTP/1.1", + "GET /v2/session HTTP/1.1", + ] + ); + assert_eq!( + service.take_discovery_summaries(), + ["serverInfo=ok regions=1 failed=1 retried=1 recovered=1 \ + failures=NP-TEST-01:http=503#1"] + ); + } + + #[test] + fn remote_sessions_does_not_retry_final_failures() { + // A 404 from region discovery and a rejected regional answer are each requested once, + // and the incomplete sweep still fails closed. + let (base, server) = session_server( + vec![ + (404, json!({})), + (200, json!({"requestStatus":{"statusCode":4}})), + ], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.set_test_control_base(base); + let error = service + .remote_sessions(&json!({}), &json!({}), &conflict_auth(), "device") + .unwrap_err(); + assert_eq!(error.code, "session_discovery_failed"); + assert_eq!( + server.join().unwrap(), + ["GET /v2/serverInfo HTTP/1.1", "GET /v2/session HTTP/1.1"] + ); + assert_eq!( + service.take_discovery_summaries(), + [ + "serverInfo=failed regions=1 failed=1 retried=0 recovered=0 \ + failures=serverInfo:http=404#1,selected:rejected#1" + ] + ); + } + + #[test] + fn discovery_summaries_keep_every_sweep_until_recorded() { + // A token-renewal retry runs a second sweep before the RPC records anything; both + // sweeps must still reach the log, and an undrained backlog stays bounded. + let service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.push_discovery_summary("first".to_owned()); + service.push_discovery_summary("second".to_owned()); + assert_eq!(service.take_discovery_summaries(), ["first", "second"]); + assert!(service.take_discovery_summaries().is_empty()); + for index in 0..DISCOVERY_SUMMARY_BACKLOG + 2 { + service.push_discovery_summary(format!("sweep{index}")); + } + let kept = service.take_discovery_summaries(); + assert_eq!(kept.len(), DISCOVERY_SUMMARY_BACKLOG); + assert_eq!(kept.first().map(String::as_str), Some("sweep2")); + assert_eq!(kept.last().map(String::as_str), Some("sweep5")); + } + + #[test] + fn discovery_summary_names_regions_and_stays_bounded() { + let log = DiscoveryLog { + server_info: vec![Ok(())], + regions: vec![ + "selected".into(), + "NP-AMS-03".into(), + "https://np-ams-04.nvidiagrid.net/".into(), + "NP-FRK-01".into(), + ], + attempts: Mutex::new(vec![ + vec![Ok(())], + vec![Err(FailureKind::Closed), Ok(())], + vec![Err(FailureKind::Http(503)), Err(FailureKind::Timeout)], + vec![], + ]), + }; + assert_eq!( + log.summary(), + "serverInfo=ok regions=4 failed=3 retried=2 recovered=1 \ + failures=NP-AMS-03:closed#1,region2:http=503#1,region2:timeout#2,NP-FRK-01:unchecked#1" + ); + + let log = DiscoveryLog { + server_info: vec![Err(FailureKind::Connect), Err(FailureKind::Connect)], + regions: (0..MAX_DISCOVERY_REGIONS) + .map(|index| format!("NP-REGION-{index:02}")) + .collect(), + attempts: Mutex::new(vec![ + vec![Err(FailureKind::Closed), Err(FailureKind::Closed)]; + MAX_DISCOVERY_REGIONS + ]), + }; + let summary = log.summary(); + assert!(summary.starts_with( + "serverInfo=failed regions=32 failed=32 retried=32 recovered=0 \ + failures=serverInfo:connect#1,serverInfo:connect#2,NP-REGION-00:closed#1," + )); + assert!(summary.ends_with(",+60"), "{summary}"); + // Diagnostics redact whitespace-separated words containing '@' or URLs. + assert!(summary.len() < 300 && !summary.contains('@') && !summary.contains("://")); + } + + #[test] + fn failure_kind_tells_closed_connections_from_timeouts() { + use std::io::{BufRead, BufReader}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/v2/session", listener.local_addr().unwrap()); + let (timed_out_tx, timed_out_rx) = std::sync::mpsc::channel::<()>(); + let server = thread::spawn(move || { + // Read the request, then close without answering, like a stale keep-alive peer. + let (stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 0 && line != "\r\n" { + line.clear(); + } + drop(stream); + // Never answer, and keep the connection open until the client has given up. + let (stream, _) = listener.accept().unwrap(); + let _ = timed_out_rx.recv_timeout(Duration::from_secs(30)); + drop(stream); + }); + let client = Client::builder().no_proxy().build().unwrap(); + let closed = client.get(&url).send().unwrap_err(); + assert_eq!(FailureKind::of(&closed), FailureKind::Closed, "{closed:?}"); + let timeout = client + .get(&url) + .timeout(Duration::from_millis(100)) + .send() + .unwrap_err(); + let _ = timed_out_tx.send(()); + assert_eq!( + FailureKind::of(&timeout), + FailureKind::Timeout, + "{timeout:?}" + ); + server.join().unwrap(); + } + fn cloudmatch_response(status: u16, body: &str) -> Response { use std::io::{BufRead, BufReader, Write}; use std::net::TcpListener; diff --git a/native/opennow-core/src/gfn.rs b/native/opennow-core/src/gfn.rs index 6d6923850..381f4c307 100644 --- a/native/opennow-core/src/gfn.rs +++ b/native/opennow-core/src/gfn.rs @@ -391,7 +391,8 @@ impl ServiceError { } } -fn transport_failure(error: &reqwest::Error) -> Option<&'static str> { +/// The transport stage a request failed at, when the HTTP client identifies it. +pub(crate) fn transport_failure(error: &reqwest::Error) -> Option<&'static str> { if error.is_timeout() { return Some("timeout"); } @@ -2159,6 +2160,12 @@ impl GfnService { Ok(result) } + /// The diagnostics summaries of the session-discovery sweeps since the last call, oldest + /// first. + pub fn take_discovery_summaries(&self) -> Vec { + self.cloudmatch.take_discovery_summaries() + } + pub fn claim_session(&self, params: &Value, settings: &Value) -> Result { let mut routing = crate::store_requests::lock(&self.session_routing)?; let _operation = crate::store_requests::lock(&self.auth_operation)?; diff --git a/native/opennow-core/src/main.rs b/native/opennow-core/src/main.rs index 90d254c0a..4cd8697ce 100644 --- a/native/opennow-core/src/main.rs +++ b/native/opennow-core/src/main.rs @@ -839,17 +839,15 @@ fn dispatch( .map_err(gfn_error), "session.active.get" => { let settings = core.settings.lock().expect("settings poisoned").all(); - core.gfn - .reconcile_active_session(params, &settings) - .map(|value| (value, None)) - .map_err(gfn_error) + let result = core.gfn.reconcile_active_session(params, &settings); + record_discovery(core); + result.map(|value| (value, None)).map_err(gfn_error) } "session.remote.list" => { let settings = core.settings.lock().expect("settings poisoned").all(); - core.gfn - .remote_sessions(params, &settings) - .map(|value| (value, None)) - .map_err(gfn_error) + let result = core.gfn.remote_sessions(params, &settings); + record_discovery(core); + result.map(|value| (value, None)).map_err(gfn_error) } "session.claim" => { let settings = core.settings.lock().expect("settings poisoned").all(); @@ -1568,6 +1566,12 @@ fn ensure_install_id(core: &AppCore) -> Result { Ok(install_id) } +fn record_discovery(core: &AppCore) { + for summary in core.gfn.take_discovery_summaries() { + core.diagnostics.record("session", "discovery", summary); + } +} + fn upstream_http_status(message: &str) -> Option { let (_, detail) = message.split_once(" (")?; let status = detail.get(..3)?.parse::().ok()?; diff --git a/native/opennow-core/src/requests.rs b/native/opennow-core/src/requests.rs index 774c65b69..a26a1975e 100644 --- a/native/opennow-core/src/requests.rs +++ b/native/opennow-core/src/requests.rs @@ -75,6 +75,22 @@ impl Cancellation { .expect("request receipt poisoned"); receipt.accepted && receipt.cancellation != CancellationState::Cancelled } + + /// Waits for `duration`, returning early with `cancelled` if the request is cancelled. + /// Must not be called from inside `commit`, which holds the same lock. + pub fn sleep(&self, duration: Duration) -> Result<(), ServiceError> { + let receipt = self.0.receipt.lock().expect("request receipt poisoned"); + drop( + self.0 + .wake + .wait_timeout_while(receipt, duration, |receipt| { + receipt.cancellation != CancellationState::Cancelled + }) + .expect("request receipt poisoned"), + ); + self.check() + } + pub fn check(&self) -> Result<(), ServiceError> { self.0 .receipt @@ -277,6 +293,28 @@ mod tests { }); } #[test] + fn cancellation_interrupts_a_request_sleep() { + let started = std::time::Instant::now(); + assert!( + Cancellation::default() + .sleep(Duration::from_millis(20)) + .is_ok() + ); + assert!(started.elapsed() >= Duration::from_millis(20)); + + let requests = Arc::new(Requests::default()); + let permit = requests.admit("discovery", "session.remote.list").unwrap(); + let started = std::time::Instant::now(); + std::thread::scope(|scope| { + let token = permit.token.clone(); + let worker = scope.spawn(move || token.sleep(Duration::from_secs(10))); + std::thread::sleep(Duration::from_millis(50)); + requests.cancel("discovery"); + assert_eq!(worker.join().unwrap().unwrap_err().code, "cancelled"); + }); + assert!(started.elapsed() < Duration::from_secs(5)); + } + #[test] fn background_load_reserves_control_capacity_until_workers_really_exit() { let requests = Arc::new(Requests::default()); let mut permits = Vec::new();