From b12b800d91ef5fd42dfcc1c93aeb9270b51a0716 Mon Sep 17 00:00:00 2001 From: Evan Brown <153393779+etgbrown@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:27:31 -0500 Subject: [PATCH 1/2] fix(core): retry failed discovery regions once before failing a launch Problem: a direct launch (--launch-app-id, opennow://launch, Playnite) runs its own session.remote.list about a second after the startup background sweep. When one regional request of that sweep failed and no session was found, discovery returned session_discovery_failed and the launch stopped, although the next click worked. Cause: discover_sessions had no retry, so a single transient regional failure, such as a timeout or a dropped keep-alive connection, decided the whole sweep. The per-region error was also discarded, so current.log could not say which region failed or why. Change: - While no session has been found and no region rejected the credentials, regions whose failure is transient are retried once, 750 ms later, inside the existing 12 s discovery budget. The retry runs only if more than 2.25 s of the budget remain (the delay plus 1.5 s for a useful attempt), and retried requests keep the min(remaining, 3 s) timeout. v2/serverInfo gets the same single retry. - Transient means HTTP 408, 425, 429, 500, 502, 503 or 504, the set get_session already retries (now shared as is_transient_status), or any transport failure. Other statuses, invalid bodies and rejected requestStatus values are final, and authentication errors still short-circuit. - The delay uses a new Cancellation::sleep that waits on the request's condvar, so cancelling the RPC ends the wait and no retry is sent. - The fail-closed contract in docs/core-protocol.md (#898) is unchanged: if a region still failed or was never checked and no session was found, discovery returns session_discovery_failed, and callers must not create a session. - Each sweep queues one diagnostics line (at most four queued, oldest dropped), and the session.remote.list and session.active.get handlers write every queued line to current.log: [session] discovery: serverInfo= regions=N failed=N retried=N recovered=N failures=:#,... Kinds are http=, timeout, dns, tls, connect, closed, body, rejected, network and unchecked, and the list stops after six entries with +N. The timeout, dns, tls and connect stages come from the shared transport_failure classifier (#1109); closed marks a connection closed or reset under the request. Regions are named by CloudMatch metadata key (region when the key is not a plain identifier). Lines never contain URLs, tokens or error messages. - gfn.rs remote_sessions, like hinted session reconciliation, holds the session_routing and auth_operation locks for the whole sweep, so a sweep that retries holds them longer: the 750 ms delay plus the retried requests, still within the 12 s budget. Concurrent routing calls wait for it. Verification (native/opennow-core, debug, as in CI): - cargo fmt --check and cargo clippy --locked --all-targets -- -D warnings: clean. - cargo test --locked: the opennow-core binary passes 713/713, and the other binaries and integration tests pass. New tests cover a transient failure recovering on the retry, fail-closed after a failed retry, no retry for 4xx, 501, body, rejected or authentication failures, no retry once a session is found or without budget, cancellation during the delay, the serverInfo retry, the summary queue and line format, and closed versus timeout classification. Four of them run over loopback HTTP. - The library target's update_apply process tests fail intermittently in this environment, with the same assertion on unmodified dev. That target builds only push/ and update_apply/, which this change does not touch. Co-Authored-By: Claude Opus 5.5 --- docs/core-protocol.md | 24 +- native/opennow-core/src/cloudmatch.rs | 1027 ++++++++++++++++++++++--- native/opennow-core/src/gfn.rs | 9 +- native/opennow-core/src/main.rs | 20 +- native/opennow-core/src/requests.rs | 36 + 5 files changed, 992 insertions(+), 124 deletions(-) diff --git a/docs/core-protocol.md b/docs/core-protocol.md index e2eba350f..a66b2f37a 100644 --- a/docs/core-protocol.md +++ b/docs/core-protocol.md @@ -699,14 +699,22 @@ reuse that discovery to stop a seat. CloudMatch instead of treating the first empty regional response as authoritative. Discovery deduplicates session IDs and uses at most four concurrent requests, a three-second per-request timeout, a 12-second total budget (including region -discovery), and at most 32 regional endpoints. If no sessions were found but a -region failed, returned invalid data, or could not be checked within these bounds, -the request fails with `session_discovery_failed` rather than returning an empty -list. HTTP 401 remains `http_unauthorized` after at most one safe-read renewal; -HTTP 403 remains `authentication_required`. A found session -can be returned even when another region fails; an empty successful list means -all discovered regions were checked successfully. Callers must not create a new -session after a discovery failure. +discovery), and at most 32 regional endpoints. While no session has been found and +no region rejected the credentials, a region or region-discovery request that failed +transiently is retried once, 750 ms later, if more than 2.25 seconds of the budget +remain. Transient means HTTP 408, 425, 429, 500, 502, 503, or 504 (the statuses +session polling retries) or a transport failure such as a timeout or a closed +connection; other HTTP statuses, invalid bodies, and rejected `requestStatus` values +are final. If no sessions were found but a region failed, returned invalid data, or +could not be checked within these bounds, even after that retry, the request fails +with `session_discovery_failed` rather than returning an empty list. HTTP 401 +remains `http_unauthorized` after at most one safe-read renewal; HTTP 403 remains +`authentication_required`. A found session can be returned even when another region +fails; an empty successful list means all discovered regions were checked +successfully. Callers must not create a new session after a discovery failure. Each +sweep records one bounded `[session] discovery:` diagnostics line with each failed +request's kind and attempt number, naming regions by CloudMatch metadata key, never +by URL. ### Provider routing and account scope diff --git a/native/opennow-core/src/cloudmatch.rs b/native/opennow-core/src/cloudmatch.rs index 50bfecbb5..6815c507b 100644 --- a/native/opennow-core/src/cloudmatch.rs +++ b/native/opennow-core/src/cloudmatch.rs @@ -23,6 +23,12 @@ const NETWORK_TEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(8); const MAXIMUM_NETWORK_TEST_RESPONSE_BYTES: u64 = 4 * 1024 * 1024; const MAX_DISCOVERY_REGIONS: usize = 32; const DISCOVERY_CONCURRENCY: usize = 4; +/// Sweep summaries kept until an RPC records them; bounds memory if nothing drains them. +const DISCOVERY_SUMMARY_BACKLOG: usize = 4; +const DISCOVERY_RETRY_DELAY: Duration = Duration::from_millis(750); +// Budget that must remain after the retry delay for a retried request to be useful. +const DISCOVERY_RETRY_FLOOR: Duration = Duration::from_millis(1500); +const MAX_DISCOVERY_LOG_FAILURES: usize = 6; const MAX_CLEANUP_RECORD_BYTES: usize = 16 * 1024; fn webrtc_compatibility_enabled(settings: &Value, auth: &AuthSession) -> bool { @@ -142,6 +148,7 @@ pub struct CloudMatchService { client: Client, active: Mutex>, discovered: Mutex>, + discovery_summaries: Mutex>, conflict: Mutex>, allocation_admission: Mutex<()>, fresh: Mutex>, @@ -204,6 +211,7 @@ impl CloudMatchService { client, active: Mutex::new(None), discovered: Mutex::new(HashMap::new()), + discovery_summaries: Mutex::new(Vec::new()), conflict: Mutex::new(None), allocation_admission: Mutex::new(()), fresh: Mutex::new(None), @@ -958,69 +966,22 @@ impl CloudMatchService { let requested = recovery_region.map_or_else(|| requested_streaming_base(params, settings, auth), Ok)?; let headers = cloudmatch_headers(session_token(auth), device_id)?; - let mut bases = vec![requested.clone()]; let server_info_url = requested .join("v2/serverInfo") .map_err(|_| invalid("Invalid server-info URL"))?; #[cfg(test)] let server_info_url = self.fixture_url(server_info_url); - let server_info = client - .get(server_info_url) - .headers(headers.clone()) - .timeout(DISCOVERY_REQUEST_TIMEOUT) - .send() - .map_err(|error| network("Region discovery failed", error)) - .and_then(|response| { - if !response.status().is_success() { - return Err(response_error("Region discovery failed", response)); - } - let payload = response - .json::() - .map_err(|error| network("Invalid region response", error))?; - if payload["metaData"].as_array().is_none() - || (payload.get("requestStatus").is_some() - && value_i64(&payload["requestStatus"]["statusCode"]) != Some(1)) - { - return Err(upstream("Invalid region response")); - } - Ok(payload) - }); - if let Err(error) = &server_info - && matches!(error.code, "authentication_required" | "http_unauthorized") - { - return Err(error.clone()); - } - if let Ok(payload) = &server_info { - for base in regional_bases(payload) { - if !bases.contains(&base) { - bases.push(base); - } - } - } - let incomplete = server_info.is_err() || bases.len() > MAX_DISCOVERY_REGIONS; - bases.truncate(MAX_DISCOVERY_REGIONS); - let mut sessions = discover_sessions(&bases, deadline, incomplete, |base, timeout| { - let url = base - .join("v2/session") - .map_err(|_| invalid("Invalid active-session URL"))?; - #[cfg(test)] - let url = self.fixture_url(url); - let response = client - .get(url) - .headers(headers.clone()) - .timeout(timeout) - .send() - .map_err(|error| network("Active-session discovery failed", error))?; - let payload = - read_cloudmatch_response("Active-session discovery failed", response, false)?; - let sessions = payload["sessions"] - .as_array() - .ok_or_else(|| upstream("Invalid active-session response"))?; - Ok(sessions - .iter() - .filter_map(|session| remote_session_info(session, base)) - .collect()) - })?; + let mut log = DiscoveryLog::default(); + let sessions = self.discover_remote_sessions( + &client, + &headers, + requested, + server_info_url, + deadline, + &mut log, + ); + self.push_discovery_summary(log.summary()); + let mut sessions = sessions?; if let Some(pending) = self.pending_cleanup(auth) && !sessions .iter() @@ -1045,6 +1006,95 @@ impl CloudMatchService { } } + /// Queues one sweep's diagnostics summary. Every sweep keeps its own line, including the + /// first sweep of a token-renewal retry; only the oldest lines beyond the bound are dropped. + fn push_discovery_summary(&self, summary: String) { + let mut summaries = self + .discovery_summaries + .lock() + .expect("CloudMatch discovery state poisoned"); + if summaries.len() >= DISCOVERY_SUMMARY_BACKLOG { + summaries.remove(0); + } + summaries.push(summary); + } + + /// Returns the diagnostics summaries of the discovery sweeps since the last call, oldest + /// first. + pub(crate) fn take_discovery_summaries(&self) -> Vec { + std::mem::take( + &mut *self + .discovery_summaries + .lock() + .expect("CloudMatch discovery state poisoned"), + ) + } + + fn discover_remote_sessions( + &self, + client: &Client, + headers: &HeaderMap, + requested: Url, + server_info_url: Url, + deadline: Instant, + log: &mut DiscoveryLog, + ) -> Result, ServiceError> { + let mut server_info = fetch_server_info( + client, + server_info_url.clone(), + headers, + DISCOVERY_REQUEST_TIMEOUT, + ); + log.server_info.push(attempt_outcome(&server_info)); + if let Err((_, kind)) = &server_info + && kind.is_transient() + && has_retry_budget(deadline) + { + crate::requests::current().sleep(DISCOVERY_RETRY_DELAY)?; + let timeout = deadline + .saturating_duration_since(Instant::now()) + .min(DISCOVERY_REQUEST_TIMEOUT); + server_info = fetch_server_info(client, server_info_url, headers, timeout); + log.server_info.push(attempt_outcome(&server_info)); + } + let server_info = server_info.map_err(|(error, _)| error); + if let Err(error) = &server_info + && is_auth_failure(error) + { + return Err(error.clone()); + } + let mut regions = vec![("selected".to_owned(), requested)]; + if let Ok(payload) = &server_info { + for (name, base) in regional_bases(payload) { + match regions.iter_mut().find(|(_, known)| known == &base) { + Some(region) => region.0 = name, + None => regions.push((name, base)), + } + } + } + let incomplete = server_info.is_err() || regions.len() > MAX_DISCOVERY_REGIONS; + regions.truncate(MAX_DISCOVERY_REGIONS); + let (names, bases): (Vec<_>, Vec<_>) = regions.into_iter().unzip(); + log.attempts = Mutex::new(vec![Vec::new(); names.len()]); + log.regions = names; + let log = &*log; + discover_sessions(&bases, deadline, incomplete, |base, timeout| { + let url = base.join("v2/session").map_err(|_| { + ( + invalid("Invalid active-session URL"), + FailureKind::Unchecked, + ) + })?; + #[cfg(test)] + let url = self.fixture_url(url); + let result = fetch_region_sessions(client, url, headers, base, timeout); + if let Some(index) = bases.iter().position(|known| known == base) { + log.record(index, attempt_outcome(&result)); + } + result + }) + } + pub fn claim( &self, params: &Value, @@ -1379,13 +1429,7 @@ impl CloudMatchService { let mut last_error = None; for attempt in 0..=2 { match client.get(url.clone()).headers(headers.clone()).send() { - Ok(response) - if attempt < 2 - && matches!( - response.status().as_u16(), - 408 | 425 | 429 | 500 | 502 | 503 | 504 - ) => - { + Ok(response) if attempt < 2 && is_transient_status(response.status().as_u16()) => { thread::sleep(Duration::from_millis(if attempt == 0 { 250 } else { 750 })); } Ok(response) => { @@ -1453,6 +1497,7 @@ impl CloudMatchService { }; Ok(regional_bases(&payload) .into_iter() + .map(|(_, base)| base) .find(|base| { !prefer_regional || !base.host_str().unwrap_or_default().starts_with("np-") }) @@ -2536,30 +2581,95 @@ fn discover_sessions( bases: &[Url], deadline: Instant, mut incomplete: bool, - fetch: impl Fn(&Url, Duration) -> Result, ServiceError> + Sync, + fetch: impl Fn(&Url, Duration) -> Result, RegionFailure> + Sync, ) -> Result, ServiceError> { - let fetch = &fetch; let cancellation = crate::requests::current(); + let all = (0..bases.len()).collect::>(); + let mut results = fetch_regions(bases, &all, deadline, &cancellation, &fetch); + cancellation.check()?; + // A single transient regional failure must not fail discovery on its own. While nothing was + // found and no region rejected the credentials, retry the transiently failed regions once, + // after a short delay, if the budget still leaves room for a useful attempt. Other failures + // stay final, so an incomplete sweep still fails closed below. + let transient = all + .into_iter() + .filter(|index| { + results[*index] + .as_ref() + .is_err_and(|(_, kind)| kind.is_transient()) + }) + .collect::>(); + if !transient.is_empty() + && !results + .iter() + .any(|result| result.as_ref().is_ok_and(|found| !found.is_empty())) + && !results.iter().any(|result| { + result + .as_ref() + .is_err_and(|(error, _)| is_auth_failure(error)) + }) + && has_retry_budget(deadline) + { + cancellation.sleep(DISCOVERY_RETRY_DELAY)?; + let retried = fetch_regions(bases, &transient, deadline, &cancellation, &fetch); + cancellation.check()?; + for (index, result) in transient.into_iter().zip(retried) { + results[index] = result; + } + } + let mut sessions = Vec::new(); + for result in results { + match result { + Ok(found) => { + for session in found { + if !sessions + .iter() + .any(|known: &Value| known["sessionId"] == session["sessionId"]) + { + sessions.push(session); + } + } + } + Err((error, _)) if is_auth_failure(&error) => return Err(error), + Err(_) => incomplete = true, + } + } + if sessions.is_empty() && (incomplete || bases.is_empty()) { + return Err(discovery_failed()); + } + Ok(sessions) +} + +/// Fetches `bases[index]` for each of `indices` with bounded concurrency. Results are returned +/// in the order of `indices`. +fn fetch_regions( + bases: &[Url], + indices: &[usize], + deadline: Instant, + cancellation: &crate::requests::Cancellation, + fetch: &(impl Fn(&Url, Duration) -> Result, RegionFailure> + Sync), +) -> Vec, RegionFailure>> { let mut results = thread::scope(|scope| { - let workers = (0..DISCOVERY_CONCURRENCY.min(bases.len())) + let workers = (0..DISCOVERY_CONCURRENCY.min(indices.len())) .map(|worker| { - let cancellation = cancellation.clone(); scope.spawn(move || { - bases + indices .iter() .enumerate() .skip(worker) .step_by(DISCOVERY_CONCURRENCY) - .map(|(index, base)| { + .map(|(position, index)| { let remaining = deadline.saturating_duration_since(Instant::now()); - let result = cancellation.check().and_then(|()| { - if remaining.is_zero() { - Err(discovery_failed()) - } else { - fetch(base, remaining.min(DISCOVERY_REQUEST_TIMEOUT)) + let result = match cancellation.check() { + Err(error) => Err((error, FailureKind::Unchecked)), + Ok(()) if remaining.is_zero() => { + Err((discovery_failed(), FailureKind::Unchecked)) + } + Ok(()) => { + fetch(&bases[*index], remaining.min(DISCOVERY_REQUEST_TIMEOUT)) } - }); - (index, result) + }; + (position, result) }) .collect::>() }) @@ -2570,31 +2680,291 @@ fn discover_sessions( .flat_map(|worker| worker.join().expect("CloudMatch discovery worker panicked")) .collect::>() }); - cancellation.check()?; - results.sort_by_key(|(index, _)| *index); - let mut sessions = Vec::new(); - for (_, result) in results { - match result { - Ok(found) => { - for session in found { - if !sessions - .iter() - .any(|known: &Value| known["sessionId"] == session["sessionId"]) - { - sessions.push(session); - } - } + results.sort_by_key(|(position, _)| *position); + results.into_iter().map(|(_, result)| result).collect() +} + +/// A failed regional request: the error for the caller and its URL-free classification. +type RegionFailure = (ServiceError, FailureKind); + +fn is_auth_failure(error: &ServiceError) -> bool { + matches!(error.code, "authentication_required" | "http_unauthorized") +} + +fn has_retry_budget(deadline: Instant) -> bool { + deadline.saturating_duration_since(Instant::now()) + > DISCOVERY_RETRY_DELAY + DISCOVERY_RETRY_FLOOR +} + +/// HTTP statuses after which one more CloudMatch read can succeed. Session polling and +/// discovery retry the same set. +fn is_transient_status(status: u16) -> bool { + matches!(status, 408 | 425 | 429 | 500 | 502 | 503 | 504) +} + +fn fetch_server_info( + client: &Client, + url: Url, + headers: &HeaderMap, + timeout: Duration, +) -> Result { + let response = client + .get(url) + .headers(headers.clone()) + .timeout(timeout) + .send() + .map_err(|error| request_failure("Region discovery failed", error))?; + let status = response.status(); + if !status.is_success() { + return Err(( + response_error("Region discovery failed", response), + FailureKind::Http(status.as_u16()), + )); + } + let payload = response + .json::() + .map_err(|error| request_failure("Invalid region response", error))?; + if payload["metaData"].as_array().is_none() { + return Err((upstream("Invalid region response"), FailureKind::Body)); + } + if payload.get("requestStatus").is_some() + && value_i64(&payload["requestStatus"]["statusCode"]) != Some(1) + { + return Err((upstream("Invalid region response"), FailureKind::Rejected)); + } + Ok(payload) +} + +fn fetch_region_sessions( + client: &Client, + url: Url, + headers: &HeaderMap, + base: &Url, + timeout: Duration, +) -> Result, (ServiceError, FailureKind)> { + const CONTEXT: &str = "Active-session discovery failed"; + let response = client + .get(url) + .headers(headers.clone()) + .timeout(timeout) + .send() + .map_err(|error| request_failure(CONTEXT, error))?; + let status = response.status(); + let payload = response.json::(); + let kind = match &payload { + _ if !status.is_success() => FailureKind::Http(status.as_u16()), + Err(error) => FailureKind::of(error), + Ok(_) => FailureKind::Rejected, + }; + let payload = validate_cloudmatch_response(CONTEXT, status, payload, false) + .map_err(|error| (error, kind))?; + let sessions = payload["sessions"].as_array().ok_or_else(|| { + ( + upstream("Invalid active-session response"), + FailureKind::Body, + ) + })?; + Ok(sessions + .iter() + .filter_map(|session| remote_session_info(session, base)) + .collect()) +} + +fn request_failure(context: &str, error: reqwest::Error) -> (ServiceError, FailureKind) { + (network(context, &error), FailureKind::of(&error)) +} + +fn attempt_outcome(result: &Result) -> Result<(), FailureKind> { + result.as_ref().map(|_| ()).map_err(|(_, kind)| *kind) +} + +/// URL-free classification of one failed discovery request, for diagnostics. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FailureKind { + Http(u16), + Timeout, + Dns, + Tls, + Connect, + /// The connection was closed or reset under the request, as with a stale keep-alive + /// connection. + Closed, + /// Invalid JSON or a payload without the expected fields. + Body, + /// CloudMatch answered with a non-success `requestStatus`. + Rejected, + /// Another transport failure. + Network, + /// Never sent: the deadline or the request cancellation ended the sweep first. + Unchecked, +} + +impl FailureKind { + /// Whether one more attempt can plausibly succeed. As in session polling, that is the + /// transient HTTP statuses and any transport failure. Other statuses, malformed bodies, + /// rejected requests and unsent requests are final. + fn is_transient(self) -> bool { + match self { + Self::Http(status) => is_transient_status(status), + Self::Timeout + | Self::Dns + | Self::Tls + | Self::Connect + | Self::Closed + | Self::Network => true, + Self::Body | Self::Rejected | Self::Unchecked => false, + } + } + + /// Classifies a failed send or body read. The `timeout`, `dns`, `tls` and `connect` stages + /// come from the shared GFN transport classifier; discovery adds closed connections and + /// invalid bodies. + fn of(error: &reqwest::Error) -> Self { + match crate::gfn::transport_failure(error) { + Some("timeout") => Self::Timeout, + Some("dns") => Self::Dns, + Some("tls") => Self::Tls, + Some(_) => Self::Connect, + None if connection_closed(error) => Self::Closed, + None if error.is_decode() => Self::Body, + None => Self::Network, + } + } +} + +/// Whether the connection was closed or reset under the request. hyper reports a pooled +/// keep-alive connection that the server closed first as "connection closed before message +/// completed". +fn connection_closed(error: &reqwest::Error) -> bool { + let mut source = std::error::Error::source(error); + while let Some(cause) = source { + if cause.to_string().starts_with("connection closed") { + return true; + } + source = match cause.downcast_ref::() { + Some(io) + if matches!( + io.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + ) => + { + return true; } - Err(error) if matches!(error.code, "authentication_required" | "http_unauthorized") => { - return Err(error); + Some(io) => io.get_ref().map(|inner| inner as _), + None => cause.source(), + }; + } + false +} + +impl std::fmt::Display for FailureKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Http(status) => write!(formatter, "http={status}"), + Self::Timeout => formatter.write_str("timeout"), + Self::Dns => formatter.write_str("dns"), + Self::Tls => formatter.write_str("tls"), + Self::Connect => formatter.write_str("connect"), + Self::Closed => formatter.write_str("closed"), + Self::Body => formatter.write_str("body"), + Self::Rejected => formatter.write_str("rejected"), + Self::Network => formatter.write_str("network"), + Self::Unchecked => formatter.write_str("unchecked"), + } + } +} + +type Attempts = Vec>; + +/// Bounded record of one discovery sweep for `current.log`. It names regions by their +/// CloudMatch metadata key and never holds URLs, tokens or error messages. +#[derive(Default)] +struct DiscoveryLog { + server_info: Attempts, + regions: Vec, + attempts: Mutex>, +} + +impl DiscoveryLog { + fn record(&self, region: usize, outcome: Result<(), FailureKind>) { + if let Some(attempts) = self + .attempts + .lock() + .expect("CloudMatch discovery log poisoned") + .get_mut(region) + { + attempts.push(outcome); + } + } + + /// For example `serverInfo=ok regions=17 failed=1 retried=1 recovered=1 + /// failures=NP-AMS-03:closed#1`. Attempts are numbered from 1; a region that was never + /// requested (deadline or cancellation) is reported as `unchecked`. + fn summary(self) -> String { + let attempts = self + .attempts + .into_inner() + .expect("CloudMatch discovery log poisoned"); + let failures_of = |name: &str, attempts: &Attempts| { + attempts + .iter() + .enumerate() + .filter_map(|(attempt, outcome)| { + outcome + .err() + .map(|kind| format!("{name}:{kind}#{}", attempt + 1)) + }) + .collect::>() + }; + let mut failures = failures_of("serverInfo", &self.server_info); + let (mut failed, mut retried, mut recovered) = (0, 0, 0); + for (index, (name, attempts)) in self.regions.iter().zip(&attempts).enumerate() { + let name = region_label(index, name); + failed += usize::from(attempts.first().is_none_or(Result::is_err)); + retried += usize::from(attempts.len() > 1); + recovered += + usize::from(attempts.len() > 1 && attempts.last().is_some_and(Result::is_ok)); + if attempts.is_empty() { + failures.push(format!("{name}:unchecked#1")); } - Err(_) => incomplete = true, + failures.extend(failures_of(&name, attempts)); + } + let server_info = match self.server_info.as_slice() { + [Ok(())] => "ok", + [Err(_), Ok(())] => "recovered", + _ => "failed", + }; + let mut summary = format!( + "serverInfo={server_info} regions={} failed={failed} retried={retried} recovered={recovered}", + self.regions.len() + ); + if !failures.is_empty() { + let omitted = failures.len().saturating_sub(MAX_DISCOVERY_LOG_FAILURES); + failures.truncate(MAX_DISCOVERY_LOG_FAILURES); + if omitted > 0 { + failures.push(format!("+{omitted}")); + } + summary.push_str(" failures="); + summary.push_str(&failures.join(",")); } + summary } - if sessions.is_empty() && (incomplete || bases.is_empty()) { - return Err(discovery_failed()); +} + +/// Logs a region by its metadata key only when that key looks like a region identifier. +fn region_label(index: usize, name: &str) -> String { + if (1..=24).contains(&name.len()) + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + name.to_owned() + } else { + format!("region{index}") } - Ok(sessions) } fn discovery_failed() -> ServiceError { @@ -2605,7 +2975,8 @@ fn discovery_failed() -> ServiceError { } } -fn regional_bases(payload: &Value) -> Vec { +/// Returns the advertised regional endpoints with their metadata names, local region first. +fn regional_bases(payload: &Value) -> Vec<(String, Url)> { let metadata = payload["metaData"].as_array().cloned().unwrap_or_default(); let value_for = |key: &str| { metadata.iter().find_map(|entry| { @@ -2630,9 +3001,11 @@ fn regional_bases(payload: &Value) -> Vec { for name in names { if let Some(raw) = value_for(&name) && let Ok(base) = trusted_cloudmatch_base(&raw) - && !result.iter().any(|existing: &Url| existing == &base) + && !result + .iter() + .any(|(_, existing): &(String, Url)| existing == &base) { - result.push(base); + result.push((name, base)); } } result @@ -4956,7 +5329,7 @@ mod tests { false, |base, _| { if base == &bases[0] { - Err(upstream("failed region")) + Err(failure(FailureKind::Body)) } else { Ok(if found { vec![json!({"sessionId":"seat"})] @@ -4992,7 +5365,7 @@ mod tests { ); assert_eq!( discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { - Err(upstream("failed")) + Err(failure(FailureKind::Body)) }) .unwrap_err() .code, @@ -5040,10 +5413,13 @@ mod tests { fn discovery_preserves_authentication_failures_and_cancellation() { let bases = [trusted_cloudmatch_base(DEFAULT_STREAMING_BASE).unwrap()]; let error = discover_sessions(&bases, Instant::now() + DISCOVERY_TIMEOUT, false, |_, _| { - Err(ServiceError { - code: "authentication_required", - message: "Expired credentials".to_owned(), - }) + Err(( + ServiceError { + code: "authentication_required", + message: "Expired credentials".to_owned(), + }, + FailureKind::Http(403), + )) }) .unwrap_err(); assert_eq!(error.code, "authentication_required"); @@ -5060,6 +5436,443 @@ mod tests { assert_eq!(error.code, "cancelled"); } + fn retry_bases() -> [Url; 3] { + [ + "https://first.nvidiagrid.net", + "https://second.nvidiagrid.net", + "https://third.nvidiagrid.net", + ] + .map(|url| trusted_cloudmatch_base(url).unwrap()) + } + + /// A failed regional request of the given kind, with the error the caller would see. + fn failure(kind: FailureKind) -> RegionFailure { + (network("Active-session discovery failed", kind), kind) + } + + /// Runs discovery over `bases` and returns the result with the number of calls per base. + fn discover_counting( + bases: &[Url], + deadline: Instant, + fetch: impl Fn(usize, usize) -> Result, RegionFailure> + Sync, + ) -> (Result, ServiceError>, Vec) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let calls = bases + .iter() + .map(|_| AtomicUsize::new(0)) + .collect::>(); + let result = discover_sessions(bases, deadline, false, |base, timeout| { + assert!(timeout <= DISCOVERY_REQUEST_TIMEOUT); + let index = bases.iter().position(|known| known == base).unwrap(); + fetch(index, calls[index].fetch_add(1, Ordering::SeqCst)) + }); + let calls = calls.iter().map(|calls| calls.load(Ordering::SeqCst)); + (result, calls.collect()) + } + + #[test] + fn discovery_retries_a_transient_region_failure_once() { + let started = Instant::now(); + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, call| { + if index == 0 && call == 0 { + Err(failure(FailureKind::Timeout)) + } else { + Ok(vec![]) + } + }, + ); + assert!(result.unwrap().is_empty()); + assert_eq!(calls, [2, 1, 1]); + assert!(started.elapsed() >= DISCOVERY_RETRY_DELAY); + } + + #[test] + fn discovery_still_fails_closed_when_the_retry_fails() { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| { + if index == 0 { + Err(failure(FailureKind::Closed)) + } else { + Ok(vec![]) + } + }, + ); + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + assert_eq!(calls, [2, 1, 1]); + } + + #[test] + fn discovery_does_not_retry_authentication_failures() { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| match index { + 0 => Err(( + ServiceError { + code: "http_unauthorized", + message: "Expired credentials".to_owned(), + }, + FailureKind::Http(401), + )), + 1 => Err(failure(FailureKind::Timeout)), + _ => Ok(vec![]), + }, + ); + assert_eq!(result.unwrap_err().code, "http_unauthorized"); + assert_eq!(calls, [1, 1, 1]); + } + + #[test] + fn discovery_does_not_retry_non_transient_regional_failures() { + for kind in [ + FailureKind::Http(400), + FailureKind::Http(404), + FailureKind::Http(501), + FailureKind::Body, + FailureKind::Rejected, + ] { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| { + if index == 0 { + Err(failure(kind)) + } else { + Ok(vec![]) + } + }, + ); + // The retry delay only runs before a retry, so one call per region also proves + // that no delay was spent. + assert_eq!( + result.unwrap_err().code, + "session_discovery_failed", + "{kind}" + ); + assert_eq!(calls, [1, 1, 1], "{kind}"); + } + } + + #[test] + fn failure_kinds_separate_transient_from_final_failures() { + // The HTTP statuses are the ones session polling (`get_session`) retries. + for kind in [ + FailureKind::Http(408), + FailureKind::Http(425), + FailureKind::Http(429), + FailureKind::Http(500), + FailureKind::Http(502), + FailureKind::Http(503), + FailureKind::Http(504), + FailureKind::Timeout, + FailureKind::Dns, + FailureKind::Tls, + FailureKind::Connect, + FailureKind::Closed, + FailureKind::Network, + ] { + assert!(kind.is_transient(), "{kind}"); + } + for kind in [ + FailureKind::Http(302), + FailureKind::Http(400), + FailureKind::Http(401), + FailureKind::Http(403), + FailureKind::Http(404), + FailureKind::Http(501), + FailureKind::Http(505), + FailureKind::Body, + FailureKind::Rejected, + FailureKind::Unchecked, + ] { + assert!(!kind.is_transient(), "{kind}"); + } + } + + #[test] + fn discovery_skips_the_retry_once_a_session_is_found() { + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_TIMEOUT, + |index, _| match index { + 0 => Err(failure(FailureKind::Timeout)), + 1 => Ok(vec![json!({"sessionId":"seat"})]), + _ => Ok(vec![]), + }, + ); + assert_eq!(result.unwrap().len(), 1); + assert_eq!(calls, [1, 1, 1]); + } + + #[test] + fn discovery_skips_the_retry_without_budget() { + // Exactly the minimum budget at the start, so less than it remains after the first + // pass, while the first pass itself has ample time even on a loaded machine. + let (result, calls) = discover_counting( + &retry_bases(), + Instant::now() + DISCOVERY_RETRY_DELAY + DISCOVERY_RETRY_FLOOR, + |index, _| { + if index == 0 { + Err(failure(FailureKind::Timeout)) + } else { + Ok(vec![]) + } + }, + ); + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + assert_eq!(calls, [1, 1, 1]); + } + + #[test] + fn discovery_retry_delay_honours_cancellation() { + let bases = retry_bases(); + let requests = std::sync::Arc::new(crate::requests::Requests::default()); + let permit = requests.admit("discovery", "session.remote.list").unwrap(); + let regions = bases.len(); + let (fetched_tx, fetched_rx) = std::sync::mpsc::channel(); + let started = Instant::now(); + let (result, calls) = thread::scope(|scope| { + let requests = &requests; + scope.spawn(move || { + // Cancel once every region has been requested. This usually lands inside the + // retry delay; landing just before it must end discovery the same way, so a + // slow machine only loses timing margin, never the outcome. + for _ in 0..regions { + fetched_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + thread::sleep(Duration::from_millis(50)); + requests.cancel("discovery"); + }); + crate::requests::scope(permit.token.clone(), || { + discover_counting(&bases, Instant::now() + DISCOVERY_TIMEOUT, |index, _| { + let _ = fetched_tx.send(()); + if index == 0 { + Err(failure(FailureKind::Timeout)) + } else { + Ok(vec![]) + } + }) + }) + }); + assert_eq!(result.unwrap_err().code, "cancelled"); + assert_eq!(calls, [1, 1, 1]); + assert!(started.elapsed() < DISCOVERY_RETRY_DELAY); + } + + #[test] + fn remote_sessions_retries_server_info_and_records_the_sweep() { + let session_list = json!({"requestStatus":{"statusCode":1},"sessions":[]}); + let (base, server) = session_server( + vec![ + (503, json!({})), + ( + 200, + json!({"metaData":[ + {"key":"local-region","value":"NP-TEST-01"}, + {"key":"NP-TEST-01","value":"https://np-test-01.nvidiagrid.net/"} + ]}), + ), + (200, session_list.clone()), + (200, session_list), + ], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.set_test_control_base(base); + let response = service + .remote_sessions(&json!({}), &json!({}), &conflict_auth(), "device") + .unwrap(); + assert_eq!(response, json!({"sessions":[]})); + assert_eq!( + server.join().unwrap(), + [ + "GET /v2/serverInfo HTTP/1.1", + "GET /v2/serverInfo HTTP/1.1", + "GET /v2/session HTTP/1.1", + "GET /v2/session HTTP/1.1", + ] + ); + assert_eq!( + service.take_discovery_summaries(), + [ + "serverInfo=recovered regions=2 failed=0 retried=0 recovered=0 \ + failures=serverInfo:http=503#1" + ] + ); + assert!(service.take_discovery_summaries().is_empty()); + } + + #[test] + fn remote_sessions_retries_a_transient_regional_failure() { + // The selected endpoint is also the only advertised region, so requests are sequential. + let (base, server) = session_server( + vec![ + ( + 200, + json!({"metaData":[ + {"key":"local-region","value":"NP-TEST-01"}, + {"key":"NP-TEST-01","value":DEFAULT_STREAMING_BASE} + ]}), + ), + (503, json!({})), + (200, json!({"requestStatus":{"statusCode":1},"sessions":[]})), + ], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.set_test_control_base(base); + let response = service + .remote_sessions(&json!({}), &json!({}), &conflict_auth(), "device") + .unwrap(); + assert_eq!(response, json!({"sessions":[]})); + assert_eq!( + server.join().unwrap(), + [ + "GET /v2/serverInfo HTTP/1.1", + "GET /v2/session HTTP/1.1", + "GET /v2/session HTTP/1.1", + ] + ); + assert_eq!( + service.take_discovery_summaries(), + ["serverInfo=ok regions=1 failed=1 retried=1 recovered=1 \ + failures=NP-TEST-01:http=503#1"] + ); + } + + #[test] + fn remote_sessions_does_not_retry_final_failures() { + // A 404 from region discovery and a rejected regional answer are each requested once, + // and the incomplete sweep still fails closed. + let (base, server) = session_server( + vec![ + (404, json!({})), + (200, json!({"requestStatus":{"statusCode":4}})), + ], + |_| {}, + ); + let mut service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.set_test_control_base(base); + let error = service + .remote_sessions(&json!({}), &json!({}), &conflict_auth(), "device") + .unwrap_err(); + assert_eq!(error.code, "session_discovery_failed"); + assert_eq!( + server.join().unwrap(), + ["GET /v2/serverInfo HTTP/1.1", "GET /v2/session HTTP/1.1"] + ); + assert_eq!( + service.take_discovery_summaries(), + [ + "serverInfo=failed regions=1 failed=1 retried=0 recovered=0 \ + failures=serverInfo:http=404#1,selected:rejected#1" + ] + ); + } + + #[test] + fn discovery_summaries_keep_every_sweep_until_recorded() { + // A token-renewal retry runs a second sweep before the RPC records anything; both + // sweeps must still reach the log, and an undrained backlog stays bounded. + let service = CloudMatchService::new(Client::builder().no_proxy().build().unwrap()); + service.push_discovery_summary("first".to_owned()); + service.push_discovery_summary("second".to_owned()); + assert_eq!(service.take_discovery_summaries(), ["first", "second"]); + assert!(service.take_discovery_summaries().is_empty()); + for index in 0..DISCOVERY_SUMMARY_BACKLOG + 2 { + service.push_discovery_summary(format!("sweep{index}")); + } + let kept = service.take_discovery_summaries(); + assert_eq!(kept.len(), DISCOVERY_SUMMARY_BACKLOG); + assert_eq!(kept.first().map(String::as_str), Some("sweep2")); + assert_eq!(kept.last().map(String::as_str), Some("sweep5")); + } + + #[test] + fn discovery_summary_names_regions_and_stays_bounded() { + let log = DiscoveryLog { + server_info: vec![Ok(())], + regions: vec![ + "selected".into(), + "NP-AMS-03".into(), + "https://np-ams-04.nvidiagrid.net/".into(), + "NP-FRK-01".into(), + ], + attempts: Mutex::new(vec![ + vec![Ok(())], + vec![Err(FailureKind::Closed), Ok(())], + vec![Err(FailureKind::Http(503)), Err(FailureKind::Timeout)], + vec![], + ]), + }; + assert_eq!( + log.summary(), + "serverInfo=ok regions=4 failed=3 retried=2 recovered=1 \ + failures=NP-AMS-03:closed#1,region2:http=503#1,region2:timeout#2,NP-FRK-01:unchecked#1" + ); + + let log = DiscoveryLog { + server_info: vec![Err(FailureKind::Connect), Err(FailureKind::Connect)], + regions: (0..MAX_DISCOVERY_REGIONS) + .map(|index| format!("NP-REGION-{index:02}")) + .collect(), + attempts: Mutex::new(vec![ + vec![Err(FailureKind::Closed), Err(FailureKind::Closed)]; + MAX_DISCOVERY_REGIONS + ]), + }; + let summary = log.summary(); + assert!(summary.starts_with( + "serverInfo=failed regions=32 failed=32 retried=32 recovered=0 \ + failures=serverInfo:connect#1,serverInfo:connect#2,NP-REGION-00:closed#1," + )); + assert!(summary.ends_with(",+60"), "{summary}"); + // Diagnostics redact whitespace-separated words containing '@' or URLs. + assert!(summary.len() < 300 && !summary.contains('@') && !summary.contains("://")); + } + + #[test] + fn failure_kind_tells_closed_connections_from_timeouts() { + use std::io::{BufRead, BufReader}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/v2/session", listener.local_addr().unwrap()); + let (timed_out_tx, timed_out_rx) = std::sync::mpsc::channel::<()>(); + let server = thread::spawn(move || { + // Read the request, then close without answering, like a stale keep-alive peer. + let (stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 0 && line != "\r\n" { + line.clear(); + } + drop(stream); + // Never answer, and keep the connection open until the client has given up. + let (stream, _) = listener.accept().unwrap(); + let _ = timed_out_rx.recv_timeout(Duration::from_secs(30)); + drop(stream); + }); + let client = Client::builder().no_proxy().build().unwrap(); + let closed = client.get(&url).send().unwrap_err(); + assert_eq!(FailureKind::of(&closed), FailureKind::Closed, "{closed:?}"); + let timeout = client + .get(&url) + .timeout(Duration::from_millis(100)) + .send() + .unwrap_err(); + let _ = timed_out_tx.send(()); + assert_eq!( + FailureKind::of(&timeout), + FailureKind::Timeout, + "{timeout:?}" + ); + server.join().unwrap(); + } + fn cloudmatch_response(status: u16, body: &str) -> Response { use std::io::{BufRead, BufReader, Write}; use std::net::TcpListener; diff --git a/native/opennow-core/src/gfn.rs b/native/opennow-core/src/gfn.rs index f16afe21d..d7ab657b2 100644 --- a/native/opennow-core/src/gfn.rs +++ b/native/opennow-core/src/gfn.rs @@ -379,7 +379,8 @@ impl ServiceError { } } -fn transport_failure(error: &reqwest::Error) -> Option<&'static str> { +/// The transport stage a request failed at, when the HTTP client identifies it. +pub(crate) fn transport_failure(error: &reqwest::Error) -> Option<&'static str> { if error.is_timeout() { return Some("timeout"); } @@ -2124,6 +2125,12 @@ impl GfnService { Ok(result) } + /// The diagnostics summaries of the session-discovery sweeps since the last call, oldest + /// first. + pub fn take_discovery_summaries(&self) -> Vec { + self.cloudmatch.take_discovery_summaries() + } + pub fn claim_session(&self, params: &Value, settings: &Value) -> Result { let mut routing = crate::store_requests::lock(&self.session_routing)?; let _operation = crate::store_requests::lock(&self.auth_operation)?; diff --git a/native/opennow-core/src/main.rs b/native/opennow-core/src/main.rs index eec276816..6ab8e4f53 100644 --- a/native/opennow-core/src/main.rs +++ b/native/opennow-core/src/main.rs @@ -868,17 +868,15 @@ fn dispatch( .map_err(gfn_error), "session.active.get" => { let settings = core.settings.lock().expect("settings poisoned").all(); - core.gfn - .reconcile_active_session(params, &settings) - .map(|value| (value, None)) - .map_err(gfn_error) + let result = core.gfn.reconcile_active_session(params, &settings); + record_discovery(core); + result.map(|value| (value, None)).map_err(gfn_error) } "session.remote.list" => { let settings = core.settings.lock().expect("settings poisoned").all(); - core.gfn - .remote_sessions(params, &settings) - .map(|value| (value, None)) - .map_err(gfn_error) + let result = core.gfn.remote_sessions(params, &settings); + record_discovery(core); + result.map(|value| (value, None)).map_err(gfn_error) } "session.claim" => { let settings = core.settings.lock().expect("settings poisoned").all(); @@ -1597,6 +1595,12 @@ fn ensure_install_id(core: &AppCore) -> Result { Ok(install_id) } +fn record_discovery(core: &AppCore) { + for summary in core.gfn.take_discovery_summaries() { + core.diagnostics.record("session", "discovery", summary); + } +} + fn upstream_http_status(message: &str) -> Option { let (_, detail) = message.split_once(" (")?; let status = detail.get(..3)?.parse::().ok()?; diff --git a/native/opennow-core/src/requests.rs b/native/opennow-core/src/requests.rs index 5d2179600..9eaf068eb 100644 --- a/native/opennow-core/src/requests.rs +++ b/native/opennow-core/src/requests.rs @@ -42,6 +42,20 @@ impl Cancellation { .expect("request receipt poisoned"); *accepted && !self.cancelled() } + + /// Waits for `duration`, returning early with `cancelled` if the request is cancelled. + /// Must not be called from inside `commit`, which holds the same lock. + pub fn sleep(&self, duration: Duration) -> Result<(), ServiceError> { + let accepted = self.0.accepted.lock().expect("request receipt poisoned"); + drop( + self.0 + .wake + .wait_timeout_while(accepted, duration, |_| !self.cancelled()) + .expect("request receipt poisoned"), + ); + self.check() + } + pub fn check(&self) -> Result<(), ServiceError> { if self.cancelled() { Err(ServiceError { @@ -188,6 +202,28 @@ mod tests { }); } #[test] + fn cancellation_interrupts_a_request_sleep() { + let started = std::time::Instant::now(); + assert!( + Cancellation::default() + .sleep(Duration::from_millis(20)) + .is_ok() + ); + assert!(started.elapsed() >= Duration::from_millis(20)); + + let requests = Arc::new(Requests::default()); + let permit = requests.admit("discovery", "session.remote.list").unwrap(); + let started = std::time::Instant::now(); + std::thread::scope(|scope| { + let token = permit.token.clone(); + let worker = scope.spawn(move || token.sleep(Duration::from_secs(10))); + std::thread::sleep(Duration::from_millis(50)); + requests.cancel("discovery"); + assert_eq!(worker.join().unwrap().unwrap_err().code, "cancelled"); + }); + assert!(started.elapsed() < Duration::from_secs(5)); + } + #[test] fn background_load_reserves_control_capacity_until_workers_really_exit() { let requests = Arc::new(Requests::default()); let mut permits = Vec::new(); From 8abf4c5559c68ea2f19d1acfdd5b8d74425c9642 Mon Sep 17 00:00:00 2001 From: Evan Brown <153393779+etgbrown@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:21:13 -0500 Subject: [PATCH 2/2] fix(core): retry a discovery timeout at once with a shorter timeout A region that kept timing out got a second full 3 s request after the 750 ms delay, so a persistent timeout took 6.75 s to fail discovery instead of 3 s. - The retry pass now uses its own 1.5 s per-request timeout. - A timed-out request is retried at once, since it already waited out its timeout. Fast failures still wait 750 ms first. - The retry budget check is now the delay plus 1.5 s. - v2/serverInfo is retried the same way. A region that keeps timing out now fails the sweep after about 4.5 s. Timeouts stay retryable: in the live runs, both launches the retry rescued were timeouts that succeeded on the second attempt. Co-Authored-By: Claude Opus 5.5 --- docs/core-protocol.md | 6 +- native/opennow-core/src/cloudmatch.rs | 115 ++++++++++++++++++++------ 2 files changed, 96 insertions(+), 25 deletions(-) diff --git a/docs/core-protocol.md b/docs/core-protocol.md index a66b2f37a..4ebb2f725 100644 --- a/docs/core-protocol.md +++ b/docs/core-protocol.md @@ -701,8 +701,10 @@ Discovery deduplicates session IDs and uses at most four concurrent requests, a three-second per-request timeout, a 12-second total budget (including region discovery), and at most 32 regional endpoints. While no session has been found and no region rejected the credentials, a region or region-discovery request that failed -transiently is retried once, 750 ms later, if more than 2.25 seconds of the budget -remain. Transient means HTTP 408, 425, 429, 500, 502, 503, or 504 (the statuses +transiently is retried once with a 1.5-second timeout. A fast failure is retried +750 ms later, and a timeout, which has already waited three seconds, at once. The +retry runs only if more than the delay plus 1.5 seconds of the budget remain, so a +region that keeps timing out fails the sweep after about 4.5 seconds. Transient means HTTP 408, 425, 429, 500, 502, 503, or 504 (the statuses session polling retries) or a transport failure such as a timeout or a closed connection; other HTTP statuses, invalid bodies, and rejected `requestStatus` values are final. If no sessions were found but a region failed, returned invalid data, or diff --git a/native/opennow-core/src/cloudmatch.rs b/native/opennow-core/src/cloudmatch.rs index 6815c507b..29f3172a5 100644 --- a/native/opennow-core/src/cloudmatch.rs +++ b/native/opennow-core/src/cloudmatch.rs @@ -25,9 +25,13 @@ const MAX_DISCOVERY_REGIONS: usize = 32; const DISCOVERY_CONCURRENCY: usize = 4; /// Sweep summaries kept until an RPC records them; bounds memory if nothing drains them. const DISCOVERY_SUMMARY_BACKLOG: usize = 4; +/// Pause before retrying a fast failure, such as a refused or reset connection, so a transient +/// condition can clear. A timed-out request has already waited out its timeout. const DISCOVERY_RETRY_DELAY: Duration = Duration::from_millis(750); -// Budget that must remain after the retry delay for a retried request to be useful. -const DISCOVERY_RETRY_FLOOR: Duration = Duration::from_millis(1500); +/// Per-request timeout of the retry, and the budget that must remain after the delay for the +/// retry to run. A region that answers normally replies well within it, and it bounds what a +/// region that keeps timing out adds to a failing sweep. +const DISCOVERY_RETRY_REQUEST_TIMEOUT: Duration = Duration::from_millis(1500); const MAX_DISCOVERY_LOG_FAILURES: usize = 6; const MAX_CLEANUP_RECORD_BYTES: usize = 16 * 1024; @@ -1048,12 +1052,12 @@ impl CloudMatchService { log.server_info.push(attempt_outcome(&server_info)); if let Err((_, kind)) = &server_info && kind.is_transient() - && has_retry_budget(deadline) + && has_retry_budget(deadline, retry_delay([*kind])) { - crate::requests::current().sleep(DISCOVERY_RETRY_DELAY)?; + crate::requests::current().sleep(retry_delay([*kind]))?; let timeout = deadline .saturating_duration_since(Instant::now()) - .min(DISCOVERY_REQUEST_TIMEOUT); + .min(DISCOVERY_RETRY_REQUEST_TIMEOUT); server_info = fetch_server_info(client, server_info_url, headers, timeout); log.server_info.push(attempt_outcome(&server_info)); } @@ -2585,12 +2589,21 @@ fn discover_sessions( ) -> Result, ServiceError> { let cancellation = crate::requests::current(); let all = (0..bases.len()).collect::>(); - let mut results = fetch_regions(bases, &all, deadline, &cancellation, &fetch); + let mut results = fetch_regions( + bases, + &all, + deadline, + DISCOVERY_REQUEST_TIMEOUT, + &cancellation, + &fetch, + ); cancellation.check()?; // A single transient regional failure must not fail discovery on its own. While nothing was // found and no region rejected the credentials, retry the transiently failed regions once, - // after a short delay, if the budget still leaves room for a useful attempt. Other failures - // stay final, so an incomplete sweep still fails closed below. + // if the budget still leaves room for a useful attempt. Fast failures wait a short delay + // first; timeouts are retried at once. The retry has a shorter timeout, so a region that + // keeps timing out adds at most DISCOVERY_RETRY_REQUEST_TIMEOUT. Other failures stay final, + // so an incomplete sweep still fails closed below. let transient = all .into_iter() .filter(|index| { @@ -2599,6 +2612,11 @@ fn discover_sessions( .is_err_and(|(_, kind)| kind.is_transient()) }) .collect::>(); + let delay = retry_delay( + transient + .iter() + .filter_map(|index| results[*index].as_ref().err().map(|(_, kind)| *kind)), + ); if !transient.is_empty() && !results .iter() @@ -2608,10 +2626,17 @@ fn discover_sessions( .as_ref() .is_err_and(|(error, _)| is_auth_failure(error)) }) - && has_retry_budget(deadline) + && has_retry_budget(deadline, delay) { - cancellation.sleep(DISCOVERY_RETRY_DELAY)?; - let retried = fetch_regions(bases, &transient, deadline, &cancellation, &fetch); + cancellation.sleep(delay)?; + let retried = fetch_regions( + bases, + &transient, + deadline, + DISCOVERY_RETRY_REQUEST_TIMEOUT, + &cancellation, + &fetch, + ); cancellation.check()?; for (index, result) in transient.into_iter().zip(retried) { results[index] = result; @@ -2640,12 +2665,13 @@ fn discover_sessions( Ok(sessions) } -/// Fetches `bases[index]` for each of `indices` with bounded concurrency. Results are returned -/// in the order of `indices`. +/// Fetches `bases[index]` for each of `indices` with bounded concurrency, each request limited +/// to `request_timeout` and the deadline. Results are returned in the order of `indices`. fn fetch_regions( bases: &[Url], indices: &[usize], deadline: Instant, + request_timeout: Duration, cancellation: &crate::requests::Cancellation, fetch: &(impl Fn(&Url, Duration) -> Result, RegionFailure> + Sync), ) -> Vec, RegionFailure>> { @@ -2665,9 +2691,7 @@ fn fetch_regions( Ok(()) if remaining.is_zero() => { Err((discovery_failed(), FailureKind::Unchecked)) } - Ok(()) => { - fetch(&bases[*index], remaining.min(DISCOVERY_REQUEST_TIMEOUT)) - } + Ok(()) => fetch(&bases[*index], remaining.min(request_timeout)), }; (position, result) }) @@ -2691,9 +2715,18 @@ fn is_auth_failure(error: &ServiceError) -> bool { matches!(error.code, "authentication_required" | "http_unauthorized") } -fn has_retry_budget(deadline: Instant) -> bool { - deadline.saturating_duration_since(Instant::now()) - > DISCOVERY_RETRY_DELAY + DISCOVERY_RETRY_FLOOR +fn has_retry_budget(deadline: Instant, delay: Duration) -> bool { + deadline.saturating_duration_since(Instant::now()) > delay + DISCOVERY_RETRY_REQUEST_TIMEOUT +} + +/// The pause before a retry: none when every failure being retried was a timeout, which already +/// waited out its request, otherwise `DISCOVERY_RETRY_DELAY`. +fn retry_delay(kinds: impl IntoIterator) -> Duration { + if kinds.into_iter().all(|kind| kind == FailureKind::Timeout) { + Duration::ZERO + } else { + DISCOVERY_RETRY_DELAY + } } /// HTTP statuses after which one more CloudMatch read can succeed. Session polling and @@ -5478,7 +5511,7 @@ mod tests { Instant::now() + DISCOVERY_TIMEOUT, |index, call| { if index == 0 && call == 0 { - Err(failure(FailureKind::Timeout)) + Err(failure(FailureKind::Closed)) } else { Ok(vec![]) } @@ -5489,6 +5522,42 @@ mod tests { assert!(started.elapsed() >= DISCOVERY_RETRY_DELAY); } + #[test] + fn a_region_that_keeps_timing_out_fails_after_one_shorter_retry() { + // A timed-out request has already waited out its timeout, so its retry skips the delay + // and gets the shorter retry timeout: about 4.5 s to fail instead of 3 + 0.75 + 3 s. + let bases = retry_bases(); + let timeouts = std::sync::Mutex::new(Vec::new()); + let started = Instant::now(); + let result = discover_sessions( + &bases, + Instant::now() + DISCOVERY_TIMEOUT, + false, + |base, timeout| { + if base != &bases[0] { + return Ok(vec![]); + } + timeouts.lock().unwrap().push(timeout); + thread::sleep(timeout); + Err(failure(FailureKind::Timeout)) + }, + ); + let elapsed = started.elapsed(); + assert_eq!(result.unwrap_err().code, "session_discovery_failed"); + assert_eq!( + *timeouts.lock().unwrap(), + [DISCOVERY_REQUEST_TIMEOUT, DISCOVERY_RETRY_REQUEST_TIMEOUT] + ); + assert!(elapsed >= DISCOVERY_REQUEST_TIMEOUT + DISCOVERY_RETRY_REQUEST_TIMEOUT); + // Below the delayed retry this replaces, with the delay's 750 ms as scheduling slack. + assert!( + elapsed + < DISCOVERY_REQUEST_TIMEOUT + + DISCOVERY_RETRY_DELAY + + DISCOVERY_RETRY_REQUEST_TIMEOUT + ); + } + #[test] fn discovery_still_fails_closed_when_the_retry_fails() { let (result, calls) = discover_counting( @@ -5615,10 +5684,10 @@ mod tests { // pass, while the first pass itself has ample time even on a loaded machine. let (result, calls) = discover_counting( &retry_bases(), - Instant::now() + DISCOVERY_RETRY_DELAY + DISCOVERY_RETRY_FLOOR, + Instant::now() + DISCOVERY_RETRY_DELAY + DISCOVERY_RETRY_REQUEST_TIMEOUT, |index, _| { if index == 0 { - Err(failure(FailureKind::Timeout)) + Err(failure(FailureKind::Closed)) } else { Ok(vec![]) } @@ -5652,7 +5721,7 @@ mod tests { discover_counting(&bases, Instant::now() + DISCOVERY_TIMEOUT, |index, _| { let _ = fetched_tx.send(()); if index == 0 { - Err(failure(FailureKind::Timeout)) + Err(failure(FailureKind::Closed)) } else { Ok(vec![]) }