From 0aea2f7d9e691a788d193a9652de752804e7905c Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Wed, 4 Nov 2026 04:19:49 +0000 Subject: [PATCH 1/8] PeerRpc port skeleton and reqwest adapter; drop the unused transport seam domain::cluster::peer_rpc::PeerRpcError keeps the peer's HTTP status apart from an unreachable peer (callers act on 404/405/409/429/503). ports::peer_rpc::PeerRpc is the port; adapters::cluster::http_peer_rpc implements it on PeerClient's pooled clients. The HttpTransport seam in adapters/cluster/transport.rs, never wired up, is deleted. MvHandoffRequest moves to domain::sharding::types. --- .../src/adapters/cluster/http_peer_rpc.rs | 130 +++++++++++++ hyperbytedb/src/adapters/cluster/mod.rs | 2 +- hyperbytedb/src/adapters/cluster/transport.rs | 182 ------------------ hyperbytedb/src/application/mv_handoff.rs | 8 +- hyperbytedb/src/domain/cluster/mod.rs | 1 + hyperbytedb/src/domain/cluster/peer_rpc.rs | 44 +++++ hyperbytedb/src/domain/sharding/types.rs | 8 + hyperbytedb/src/ports/mod.rs | 1 + hyperbytedb/src/ports/peer_rpc.rs | 9 + 9 files changed, 195 insertions(+), 190 deletions(-) create mode 100644 hyperbytedb/src/adapters/cluster/http_peer_rpc.rs delete mode 100644 hyperbytedb/src/adapters/cluster/transport.rs create mode 100644 hyperbytedb/src/domain/cluster/peer_rpc.rs create mode 100644 hyperbytedb/src/ports/peer_rpc.rs diff --git a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs new file mode 100644 index 0000000..397e2e0 --- /dev/null +++ b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs @@ -0,0 +1,130 @@ +//! [`PeerRpc`] over HTTP, on the pooled clients `PeerClient` keeps (they +//! carry the cluster credential). + +use std::sync::Arc; +use std::time::Duration; + +use crate::adapters::cluster::peer_client::PeerClient; +use crate::domain::cluster::peer_rpc::PeerRpcError; +use crate::ports::peer_rpc::PeerRpc; + +pub struct HttpPeerRpc { + peers: Arc, +} + +impl HttpPeerRpc { + #[must_use] + pub fn new(peers: Arc) -> Self { + Self { peers } + } + + /// POST `req` as JSON to `http://{addr}{path}`; a non-success status is + /// [`PeerRpcError::Status`] with the body, a failed send `Unreachable`. + pub async fn post_json( + &self, + addr: &str, + path: &str, + req: &Req, + timeout: Duration, + ) -> Result { + let resp = self + .peers + .http_client() + .post(format!("http://{addr}{path}")) + .json(req) + .timeout(timeout) + .send() + .await; + Self::checked(resp).await + } + + /// GET `http://{addr}{path}`, with the same error mapping. + pub async fn get( + &self, + addr: &str, + path: &str, + timeout: Duration, + ) -> Result { + let resp = self + .peers + .http_client() + .get(format!("http://{addr}{path}")) + .timeout(timeout) + .send() + .await; + Self::checked(resp).await + } + + async fn checked( + resp: Result, + ) -> Result { + let resp = resp.map_err(|e| PeerRpcError::Unreachable(e.to_string()))?; + if resp.status().is_success() { + return Ok(resp); + } + let code = resp.status().as_u16(); + let body = resp.text().await.unwrap_or_default(); + Err(PeerRpcError::Status { code, body }) + } +} + +#[async_trait::async_trait] +impl PeerRpc for HttpPeerRpc {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::adapters::cluster::replication_log::ReplicationLog; + use crate::domain::cluster::membership::{ClusterMembership, new_shared}; + + fn rpc(dir: &tempfile::TempDir) -> HttpPeerRpc { + let pc = PeerClient::new( + 1, + "127.0.0.1:1".into(), + new_shared(ClusterMembership::default()), + Arc::new(ReplicationLog::open(dir.path().join("repl")).unwrap()), + 3, + 64, + 2, + 1 << 20, + Default::default(), + ); + HttpPeerRpc::new(Arc::new(pc)) + } + + /// "Answered 503" (back-pressure, retried) and "unreachable" must stay + /// distinct: callers retry one and not the other. + #[tokio::test] + async fn status_and_transport_errors_stay_distinct() { + let app = axum::Router::new().route( + "/x", + axum::routing::post(|| async { (axum::http::StatusCode::SERVICE_UNAVAILABLE, "busy") }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let dir = tempfile::tempdir().unwrap(); + let rpc = rpc(&dir); + let err = rpc + .post_json(&addr, "/x", &serde_json::json!({}), Duration::from_secs(5)) + .await + .unwrap_err(); + assert_eq!( + err, + PeerRpcError::Status { + code: 503, + body: "busy".into() + } + ); + + let closed = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let closed_addr = closed.local_addr().unwrap().to_string(); + drop(closed); + let err = rpc + .get(&closed_addr, "/x", Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(matches!(err, PeerRpcError::Unreachable(_)), "{err:?}"); + } +} diff --git a/hyperbytedb/src/adapters/cluster/mod.rs b/hyperbytedb/src/adapters/cluster/mod.rs index eb82583..ac9f679 100644 --- a/hyperbytedb/src/adapters/cluster/mod.rs +++ b/hyperbytedb/src/adapters/cluster/mod.rs @@ -2,8 +2,8 @@ pub mod cluster_auth; pub mod dns_cache; pub mod forward_queue; pub mod hinted_handoff; +pub mod http_peer_rpc; pub mod peer_client; pub mod raft; pub mod replication_log; pub mod sync_client; -pub mod transport; diff --git a/hyperbytedb/src/adapters/cluster/transport.rs b/hyperbytedb/src/adapters/cluster/transport.rs deleted file mode 100644 index 98655fb..0000000 --- a/hyperbytedb/src/adapters/cluster/transport.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! Pluggable HTTP transport for cluster RPCs. -//! -//! `PeerClient`, `SyncClient`, and the OpenRaft `Network` implementation -//! currently hardcode [`reqwest::Client`]. The [`HttpTransport`] trait is the -//! seam for alternative transports (e.g. in-memory routing for tests). -//! -//! ### Current status -//! -//! This module introduces the trait and the [`ReqwestTransport`] implementation -//! without yet rewiring `PeerClient`, `SyncClient`, or the raft `Network` to -//! consume it. - -use std::time::Duration; - -use async_trait::async_trait; -use bytes::Bytes; - -/// One RPC request destined for a peer node. -#[derive(Debug, Clone)] -pub struct TransportRequest { - /// Absolute URL, e.g. `http://10.0.0.3:8086/cluster/replicate/v1/lp`. - /// Transport implementations may ignore the hostname but are required to - /// preserve the path and query string. - pub url: String, - /// HTTP method. We keep it a string so transports don't need to depend - /// on any specific http crate version. - pub method: &'static str, - /// Headers as a `Vec` (order-preserving) so transports that care about - /// header order can serialize deterministically. - pub headers: Vec<(String, String)>, - pub body: Bytes, - /// Peer node id, if known. Optional hint for transports that route by id. - pub peer_node_id: Option, - /// Overall timeout for the request. - pub timeout: Duration, -} - -/// The peer's response. -#[derive(Debug, Clone)] -pub struct TransportResponse { - pub status: u16, - pub headers: Vec<(String, String)>, - pub body: Bytes, -} - -/// Errors returned from a transport send. Kept intentionally simple — the -/// production code paths already translate into [`crate::error::HyperbytedbError`] variants -/// like `PeerUnreachable` / `ReplicationTimeout`. -#[derive(Debug, thiserror::Error)] -pub enum TransportError { - #[error("peer unreachable: {0}")] - Unreachable(String), - #[error("request timed out after {0:?}")] - Timeout(Duration), - #[error("transport error: {0}")] - Other(String), -} - -/// HTTP-style transport for cluster RPCs. -/// -/// Implementations must be `Send + Sync` and `Clone` — the production code -/// passes the transport through many async tasks and cloning it is expected -/// to be cheap (e.g. `reqwest::Client` internally refcounts). -#[async_trait] -pub trait HttpTransport: Send + Sync + 'static { - async fn send(&self, request: TransportRequest) -> Result; -} - -/// Default transport backed by `reqwest::Client`. Behaviour is equivalent -/// to the direct `reqwest` calls the cluster code uses today. -#[derive(Clone)] -pub struct ReqwestTransport { - client: reqwest::Client, -} - -impl ReqwestTransport { - pub fn new(client: reqwest::Client) -> Self { - Self { client } - } -} - -#[async_trait] -impl HttpTransport for ReqwestTransport { - async fn send(&self, request: TransportRequest) -> Result { - let method = match request.method { - "GET" => reqwest::Method::GET, - "POST" => reqwest::Method::POST, - "PUT" => reqwest::Method::PUT, - "DELETE" => reqwest::Method::DELETE, - "PATCH" => reqwest::Method::PATCH, - "HEAD" => reqwest::Method::HEAD, - other => { - return Err(TransportError::Other(format!( - "unsupported http method: {other}" - ))); - } - }; - let mut builder = self - .client - .request(method, &request.url) - .timeout(request.timeout) - .body(request.body.to_vec()); - for (k, v) in &request.headers { - builder = builder.header(k, v); - } - let resp = builder.send().await.map_err(|e| { - if e.is_timeout() { - TransportError::Timeout(request.timeout) - } else if e.is_connect() { - TransportError::Unreachable(e.to_string()) - } else { - TransportError::Other(e.to_string()) - } - })?; - let status = resp.status().as_u16(); - let mut headers = Vec::with_capacity(resp.headers().len()); - for (k, v) in resp.headers() { - if let Ok(s) = v.to_str() { - headers.push((k.as_str().to_string(), s.to_string())); - } - } - let body = resp.bytes().await.map_err(|e| { - if e.is_timeout() { - TransportError::Timeout(request.timeout) - } else { - TransportError::Other(e.to_string()) - } - })?; - Ok(TransportResponse { - status, - headers, - body, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn transport_request_is_clone_and_debug() { - let req = TransportRequest { - url: "http://peer/lp".into(), - method: "POST", - headers: vec![("Content-Type".into(), "text/plain".into())], - body: Bytes::from_static(b"measurement,tag=a value=1"), - peer_node_id: Some(7), - timeout: Duration::from_secs(5), - }; - let _cloned = req.clone(); - let formatted = format!("{req:?}"); - assert!(formatted.contains("peer_node_id")); - } - - #[test] - fn unsupported_method_produces_other_error() { - let client = reqwest::Client::new(); - let transport = ReqwestTransport::new(client); - let req = TransportRequest { - url: "http://example.invalid/".into(), - method: "TRACE", - headers: Vec::new(), - body: Bytes::new(), - peer_node_id: None, - timeout: Duration::from_millis(10), - }; - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("current_thread runtime"); - let err = rt - .block_on(transport.send(req)) - .expect_err("trace should error"); - match err { - TransportError::Other(msg) => { - assert!(msg.contains("TRACE"), "error was: {msg}") - } - _ => panic!("expected Other, got {err:?}"), - } - } -} diff --git a/hyperbytedb/src/application/mv_handoff.rs b/hyperbytedb/src/application/mv_handoff.rs index 61893ee..ab53abf 100644 --- a/hyperbytedb/src/application/mv_handoff.rs +++ b/hyperbytedb/src/application/mv_handoff.rs @@ -32,13 +32,7 @@ pub const HANDOFF_CHUNK_ROWS: u64 = 50_000; /// handoff sends, so every retried range is recognised. const DEDUP_WINDOW: u64 = 100_000; -/// `POST /internal/mv/handoff`, to the departing node. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MvHandoffRequest { - /// The node that keeps the rollups (`host:port`). - pub to_addr: String, -} +pub use crate::domain::sharding::types::MvHandoffRequest; /// `POST /internal/mv/handoff-chunk`, to the receiving node: one range of /// one destination table. diff --git a/hyperbytedb/src/domain/cluster/mod.rs b/hyperbytedb/src/domain/cluster/mod.rs index d293ab1..167cb27 100644 --- a/hyperbytedb/src/domain/cluster/mod.rs +++ b/hyperbytedb/src/domain/cluster/mod.rs @@ -1,5 +1,6 @@ pub mod forward; pub mod membership; +pub mod peer_rpc; pub mod raft_request; pub mod replication_wire; pub mod sync; diff --git a/hyperbytedb/src/domain/cluster/peer_rpc.rs b/hyperbytedb/src/domain/cluster/peer_rpc.rs new file mode 100644 index 0000000..ddb7272 --- /dev/null +++ b/hyperbytedb/src/domain/cluster/peer_rpc.rs @@ -0,0 +1,44 @@ +//! How a call to a peer failed: the peer answered with an error status, or +//! it could not be reached. Callers act on the status (409 stale epoch, +//! 429/503 back-pressure, 404/405 an older peer), so it is kept. + +use std::fmt; + +use crate::error::HyperbytedbError; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PeerRpcError { + /// The peer answered with a non-success status. + Status { code: u16, body: String }, + /// The request did not complete (connect, send or read failed, or it + /// timed out). + Unreachable(String), +} + +impl PeerRpcError { + /// The peer's status code, if it answered. + #[must_use] + pub fn status(&self) -> Option { + match self { + Self::Status { code, .. } => Some(*code), + Self::Unreachable(_) => None, + } + } +} + +impl fmt::Display for PeerRpcError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Status { code, body } => write!(f, "HTTP {code}: {body}"), + Self::Unreachable(e) => f.write_str(e), + } + } +} + +impl std::error::Error for PeerRpcError {} + +impl From for HyperbytedbError { + fn from(e: PeerRpcError) -> Self { + HyperbytedbError::PeerUnreachable(e.to_string()) + } +} diff --git a/hyperbytedb/src/domain/sharding/types.rs b/hyperbytedb/src/domain/sharding/types.rs index 49e47b9..345cf4d 100644 --- a/hyperbytedb/src/domain/sharding/types.rs +++ b/hyperbytedb/src/domain/sharding/types.rs @@ -556,3 +556,11 @@ pub struct ShardMvBackfillRequest { pub dest_rp: String, pub dest_measurement: String, } + +/// `POST /internal/mv/handoff`, to the departing node. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MvHandoffRequest { + /// The node that keeps the rollups (`host:port`). + pub to_addr: String, +} diff --git a/hyperbytedb/src/ports/mod.rs b/hyperbytedb/src/ports/mod.rs index b9e6208..ee175ca 100644 --- a/hyperbytedb/src/ports/mod.rs +++ b/hyperbytedb/src/ports/mod.rs @@ -5,6 +5,7 @@ pub mod ingestion; pub mod leadership; pub mod metadata; pub mod peer_http; +pub mod peer_rpc; pub mod points_sink; pub mod query; pub mod replication; diff --git a/hyperbytedb/src/ports/peer_rpc.rs b/hyperbytedb/src/ports/peer_rpc.rs new file mode 100644 index 0000000..43d2cb0 --- /dev/null +++ b/hyperbytedb/src/ports/peer_rpc.rs @@ -0,0 +1,9 @@ +//! Calls to peer nodes, one typed method per internal endpoint. The +//! application uses this instead of building HTTP requests itself. + +use std::sync::Arc; + +#[async_trait::async_trait] +pub trait PeerRpc: Send + Sync {} + +pub type SharedPeerRpc = Arc; From dbc1bf5997e1ed8bacdb3b8c9884e17514a960a9 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Wed, 4 Nov 2026 04:21:06 +0000 Subject: [PATCH 2/8] Thread PeerRpc through the sharding routing context ShardRoutingContext carries the PeerRpc port and the cluster membership next to PeerClient, which the following commits stop using. --- hyperbytedb/src/application/shard_routing.rs | 8 ++++++++ hyperbytedb/src/bootstrap.rs | 4 ++++ hyperbytedb/tests/common/sharding_cluster.rs | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/hyperbytedb/src/application/shard_routing.rs b/hyperbytedb/src/application/shard_routing.rs index b8f3358..7c04c3d 100644 --- a/hyperbytedb/src/application/shard_routing.rs +++ b/hyperbytedb/src/application/shard_routing.rs @@ -31,6 +31,10 @@ pub struct ShardRoutingContext { pub config: ShardingConfig, pub node_id: u64, pub peer_client: Arc, + /// Calls to peers (replaces direct HTTP through `peer_client`). + pub peer_rpc: crate::ports::peer_rpc::SharedPeerRpc, + /// Cluster membership, for resolving node addresses. + pub membership: crate::domain::cluster::membership::SharedMembership, pub region_write_stats: Arc, /// Shares one request per destination node among concurrent writes. pub forward_coalescer: Arc, @@ -1249,6 +1253,10 @@ mod scatter_tests { location_cache: Arc::new(ShardLocationCache::new()), config, node_id, + peer_rpc: Arc::new(crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::new( + peer_client.clone(), + )), + membership: peer_client.membership().clone(), peer_client, region_write_stats: Arc::new(RegionWriteStats::new()), forward_coalescer: Default::default(), diff --git a/hyperbytedb/src/bootstrap.rs b/hyperbytedb/src/bootstrap.rs index fd7eb91..3cebbb6 100644 --- a/hyperbytedb/src/bootstrap.rs +++ b/hyperbytedb/src/bootstrap.rs @@ -363,6 +363,10 @@ pub async fn build_services(config: &HyperbytedbConfig) -> anyhow::Result Date: Fri, 6 Nov 2026 12:13:08 +0000 Subject: [PATCH 3/8] Shard data path goes through PeerRpc PeerRpc gains shard_bootstrap, shard_write, shard_write_batch, shard_query (the Arrow reply as a byte stream, still decoded chunk by chunk), shard_metadata and shard_delete. The routing, query, metadata scatter and forward-coalescer call sites use them and keep their status handling (409 stale epoch, 429/503 capacity, 400/422 refused, 404/405 older peer) by matching PeerRpcError. error_chain moves to crate::error so the adapter can use it. --- .../src/adapters/cluster/http_peer_rpc.rs | 160 ++++++++++++++++-- .../src/application/forward_coalescer.rs | 86 +++++----- .../src/application/query_arrow_series.rs | 77 ++++++++- hyperbytedb/src/application/query_service.rs | 66 ++++---- .../src/application/shard_metadata_scatter.rs | 37 ++-- hyperbytedb/src/application/shard_routing.rs | 142 ++++++---------- hyperbytedb/src/domain/cluster/peer_rpc.rs | 11 ++ hyperbytedb/src/error.rs | 13 ++ hyperbytedb/src/ports/peer_rpc.rs | 64 ++++++- 9 files changed, 446 insertions(+), 210 deletions(-) diff --git a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs index 397e2e0..50e2978 100644 --- a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs +++ b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs @@ -1,21 +1,50 @@ -//! [`PeerRpc`] over HTTP, on the pooled clients `PeerClient` keeps (they -//! carry the cluster credential). +//! [`PeerRpc`] over HTTP. Requests go out on `PeerClient`'s pooled clients +//! (they carry the cluster credential), one per call, as the call sites did. use std::sync::Arc; use std::time::Duration; +use bytes::Bytes; + use crate::adapters::cluster::peer_client::PeerClient; -use crate::domain::cluster::peer_rpc::PeerRpcError; -use crate::ports::peer_rpc::PeerRpc; +use crate::domain::cluster::peer_rpc::{PeerRpcError, ShardWriteTarget}; +use crate::domain::sharding::types::{ + ShardBootstrapRequest, ShardDeleteRequest, ShardMetadataRequest, ShardQueryRequest, + ShardWriteBatchResponse, +}; +use crate::error::error_chain; +use crate::ports::peer_rpc::{ByteStream, PeerRpc}; + +enum Clients { + Pooled(Arc), + Single(reqwest::Client), +} pub struct HttpPeerRpc { - peers: Arc, + clients: Clients, } impl HttpPeerRpc { #[must_use] pub fn new(peers: Arc) -> Self { - Self { peers } + Self { + clients: Clients::Pooled(peers), + } + } + + /// Every request on one client (tests, tools). + #[must_use] + pub fn with_client(client: reqwest::Client) -> Self { + Self { + clients: Clients::Single(client), + } + } + + fn client(&self) -> &reqwest::Client { + match &self.clients { + Clients::Pooled(p) => p.http_client(), + Clients::Single(c) => c, + } } /// POST `req` as JSON to `http://{addr}{path}`; a non-success status is @@ -28,8 +57,7 @@ impl HttpPeerRpc { timeout: Duration, ) -> Result { let resp = self - .peers - .http_client() + .client() .post(format!("http://{addr}{path}")) .json(req) .timeout(timeout) @@ -46,8 +74,7 @@ impl HttpPeerRpc { timeout: Duration, ) -> Result { let resp = self - .peers - .http_client() + .client() .get(format!("http://{addr}{path}")) .timeout(timeout) .send() @@ -58,7 +85,7 @@ impl HttpPeerRpc { async fn checked( resp: Result, ) -> Result { - let resp = resp.map_err(|e| PeerRpcError::Unreachable(e.to_string()))?; + let resp = resp.map_err(|e| PeerRpcError::Unreachable(error_chain(&e)))?; if resp.status().is_success() { return Ok(resp); } @@ -66,10 +93,119 @@ impl HttpPeerRpc { let body = resp.text().await.unwrap_or_default(); Err(PeerRpcError::Status { code, body }) } + + async fn body(resp: reqwest::Response) -> Result { + resp.bytes() + .await + .map_err(|e| PeerRpcError::Unreachable(error_chain(&e))) + } } #[async_trait::async_trait] -impl PeerRpc for HttpPeerRpc {} +impl PeerRpc for HttpPeerRpc { + async fn shard_bootstrap( + &self, + addr: &str, + req: &ShardBootstrapRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + self.post_json(addr, "/internal/shard/bootstrap", req, timeout) + .await + .map(|_| ()) + } + + async fn shard_write( + &self, + addr: &str, + target: &ShardWriteTarget, + body: Bytes, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + let mut req = self + .client() + .post(format!("http://{addr}/internal/shard/write")) + .header("Content-Type", "application/octet-stream") + .header("X-HyperbyteDB-Shard-Db", &target.db) + .header("X-HyperbyteDB-Shard-Rp", &target.rp) + .header( + "X-HyperbyteDB-Shard-Region-Id", + target.region_id.to_string(), + ) + .header("X-HyperbyteDB-Shard-Epoch", &target.epoch_json) + .body(body); + if let Some(p) = target.precision.as_deref() { + req = req.header("X-HyperbyteDB-Shard-Precision", p); + } + Self::checked(req.timeout(timeout).send().await) + .await + .map(|_| ()) + } + + async fn shard_write_batch( + &self, + addr: &str, + body: Bytes, + timeout: Duration, + ) -> Result { + let req = crate::telemetry::inject( + self.client() + .post(format!("http://{addr}/internal/shard/write-batch")) + .header("Content-Type", "application/octet-stream"), + ) + .body(body) + .timeout(timeout); + let resp = Self::checked(req.send().await).await?; + let status = resp.status(); + resp.json::() + .await + .map_err(|e| PeerRpcError::Unreachable(format!("{status}: {e}"))) + } + + async fn shard_query( + &self, + addr: &str, + req: &ShardQueryRequest, + timeout: Duration, + ) -> Result { + let resp = self + .post_json(addr, "/internal/shard/query", req, timeout) + .await?; + // Chunk by chunk, so a large Arrow reply is decoded as it arrives + // rather than buffered whole. + let stream = futures::stream::unfold(Some(resp), |state| async move { + let mut resp = state?; + match resp.chunk().await { + Ok(Some(chunk)) => Some((Ok(chunk), Some(resp))), + Ok(None) => None, + Err(e) => Some((Err(PeerRpcError::Unreachable(error_chain(&e))), None)), + } + }); + Ok(Box::pin(stream)) + } + + async fn shard_metadata( + &self, + addr: &str, + req: &ShardMetadataRequest, + timeout: Duration, + ) -> Result { + let resp = self + .post_json(addr, "/internal/shard/metadata", req, timeout) + .await?; + Self::body(resp).await + } + + async fn shard_delete( + &self, + addr: &str, + req: &ShardDeleteRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + self.post_json(addr, "/internal/shard/delete", req, timeout) + .await + .map(|_| ()) + } +} #[cfg(test)] mod tests { diff --git a/hyperbytedb/src/application/forward_coalescer.rs b/hyperbytedb/src/application/forward_coalescer.rs index 900901a..1531d1c 100644 --- a/hyperbytedb/src/application/forward_coalescer.rs +++ b/hyperbytedb/src/application/forward_coalescer.rs @@ -19,9 +19,8 @@ use std::time::{Duration, Instant}; use metrics::{counter, histogram}; use tokio::sync::{Semaphore, mpsc, oneshot}; -use crate::domain::sharding::{ - ShardWriteBatch, ShardWriteBatchResponse, ShardWritePartResult, ShardWriteRequest, -}; +use crate::domain::sharding::{ShardWriteBatch, ShardWritePartResult, ShardWriteRequest}; +use crate::ports::peer_rpc::{PeerRpc, SharedPeerRpc}; /// What a destination said about one write's parts. #[derive(Debug, Clone)] @@ -88,7 +87,7 @@ impl ForwardCoalescer { /// writes' parts for that node. pub async fn send( &self, - client: &reqwest::Client, + rpc: &SharedPeerRpc, addr: &str, timeout: Duration, parts: Vec, @@ -100,10 +99,10 @@ impl ForwardCoalescer { bytes, reply, }; - if let Err(mpsc::error::SendError(job)) = self.lane(client, addr, timeout).send(job) { + if let Err(mpsc::error::SendError(job)) = self.lane(rpc, addr, timeout).send(job) { // The lane's loop has gone; start a new one. self.lanes.lock().remove(addr); - if self.lane(client, addr, timeout).send(job).is_err() { + if self.lane(rpc, addr, timeout).send(job).is_err() { return NodeSendOutcome::Failed(format!("forward lane to {addr} closed")); } } @@ -113,7 +112,7 @@ impl ForwardCoalescer { fn lane( &self, - client: &reqwest::Client, + rpc: &SharedPeerRpc, addr: &str, timeout: Duration, ) -> mpsc::UnboundedSender { @@ -126,7 +125,7 @@ impl ForwardCoalescer { let (tx, rx) = mpsc::unbounded_channel(); tokio::spawn(lane_loop( rx, - client.clone(), + rpc.clone(), addr.to_string(), timeout, self.limits, @@ -138,7 +137,7 @@ impl ForwardCoalescer { async fn lane_loop( mut rx: mpsc::UnboundedReceiver, - client: reqwest::Client, + rpc: SharedPeerRpc, addr: String, timeout: Duration, limits: Limits, @@ -176,16 +175,16 @@ async fn lane_loop( Err(_) => break, } } - let client = client.clone(); + let rpc = rpc.clone(); let addr = addr.clone(); tokio::spawn(async move { - send_jobs(&client, &addr, timeout, jobs).await; + send_jobs(rpc.as_ref(), &addr, timeout, jobs).await; drop(permit); }); } } -async fn send_jobs(client: &reqwest::Client, addr: &str, timeout: Duration, jobs: Vec) { +async fn send_jobs(rpc: &dyn PeerRpc, addr: &str, timeout: Duration, jobs: Vec) { let counts: Vec = jobs.iter().map(|j| j.parts.len()).collect(); let mut replies = Vec::with_capacity(jobs.len()); let mut parts = Vec::with_capacity(counts.iter().sum()); @@ -196,7 +195,7 @@ async fn send_jobs(client: &reqwest::Client, addr: &str, timeout: Duration, jobs histogram!("hyperbytedb_shard_forward_writes_per_request").record(replies.len() as f64); counter!("hyperbytedb_shard_forward_requests_total").increment(1); - let outcome = post_batch(client, addr, timeout, parts).await; + let outcome = post_batch(rpc, addr, timeout, parts).await; match outcome { NodeSendOutcome::Results(results) if results.len() == counts.iter().sum::() => { let mut results = results.into_iter(); @@ -225,13 +224,13 @@ async fn send_jobs(client: &reqwest::Client, addr: &str, timeout: Duration, jobs /// Send `parts` to the node at `addr` in a request of their own: the path /// when `sharding.forward_coalesce` is off. pub async fn send_direct( - client: &reqwest::Client, + rpc: &dyn PeerRpc, addr: &str, timeout: Duration, parts: Vec, ) -> NodeSendOutcome { let n = parts.len(); - match post_batch(client, addr, timeout, parts).await { + match post_batch(rpc, addr, timeout, parts).await { NodeSendOutcome::Results(results) if results.len() != n => { NodeSendOutcome::Failed(format!( "forward batch to {addr}: {} results for {n} parts", @@ -243,46 +242,28 @@ pub async fn send_direct( } async fn post_batch( - client: &reqwest::Client, + rpc: &dyn PeerRpc, addr: &str, timeout: Duration, parts: Vec, ) -> NodeSendOutcome { + use crate::domain::cluster::peer_rpc::PeerRpcError; let body = match bincode::serialize(&ShardWriteBatch { parts }) { Ok(b) => b, Err(e) => return NodeSendOutcome::Failed(format!("encode forward batch: {e}")), }; let start = Instant::now(); - let resp = crate::telemetry::inject( - client - .post(format!("http://{addr}/internal/shard/write-batch")) - .header("Content-Type", "application/octet-stream"), - ) - .body(body) - .timeout(timeout) - .send() - .await; - let outcome = match resp { - Err(e) => NodeSendOutcome::Failed(crate::application::shard_routing::error_chain(&e)), - Ok(r) => { - let status = r.status(); - if status == reqwest::StatusCode::NOT_FOUND - || status == reqwest::StatusCode::METHOD_NOT_ALLOWED - { - NodeSendOutcome::Unsupported - } else if status == reqwest::StatusCode::SERVICE_UNAVAILABLE - || status == reqwest::StatusCode::TOO_MANY_REQUESTS - { - NodeSendOutcome::Capacity - } else if status.is_success() { - match r.json::().await { - Ok(parsed) => NodeSendOutcome::Results(parsed.results), - Err(e) => NodeSendOutcome::Failed(format!("{status}: {e}")), - } - } else { - NodeSendOutcome::Failed(status.to_string()) - } - } + let outcome = match rpc.shard_write_batch(addr, body.into(), timeout).await { + Ok(parsed) => NodeSendOutcome::Results(parsed.results), + // An older build without the batch endpoint. + Err(PeerRpcError::Status { + code: 404 | 405, .. + }) => NodeSendOutcome::Unsupported, + Err(PeerRpcError::Status { + code: 503 | 429, .. + }) => NodeSendOutcome::Capacity, + Err(PeerRpcError::Status { code, .. }) => NodeSendOutcome::Failed(format!("HTTP {code}")), + Err(PeerRpcError::Unreachable(e)) => NodeSendOutcome::Failed(e), }; let label = match &outcome { NodeSendOutcome::Results(_) => "ok", @@ -298,6 +279,7 @@ async fn post_batch( mod tests { use super::*; use crate::domain::sharding::ShardEpoch; + use crate::domain::sharding::types::ShardWriteBatchResponse; use std::sync::atomic::{AtomicUsize, Ordering}; fn part(region_id: u64) -> ShardWriteRequest { @@ -355,7 +337,11 @@ mod tests { async fn concurrent_writes_share_requests_and_get_their_own_results() { let (addr, requests) = mock_node(Duration::from_millis(100), false).await; let coalescer = Arc::new(ForwardCoalescer::new(2, 512 * 1024, 256)); - let client = reqwest::Client::new(); + let client: SharedPeerRpc = Arc::new( + crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ), + ); let writes = (0..40u64).map(|w| { let coalescer = coalescer.clone(); let client = client.clone(); @@ -391,7 +377,11 @@ mod tests { async fn a_refused_request_is_capacity_for_every_write_in_it() { let (addr, _) = mock_node(Duration::from_millis(50), true).await; let coalescer = Arc::new(ForwardCoalescer::new(1, 512 * 1024, 256)); - let client = reqwest::Client::new(); + let client: SharedPeerRpc = Arc::new( + crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ), + ); let writes = (0..8u64).map(|w| { let coalescer = coalescer.clone(); let client = client.clone(); diff --git a/hyperbytedb/src/application/query_arrow_series.rs b/hyperbytedb/src/application/query_arrow_series.rs index aa1f55c..15576db 100644 --- a/hyperbytedb/src/application/query_arrow_series.rs +++ b/hyperbytedb/src/application/query_arrow_series.rs @@ -323,19 +323,21 @@ fn cell(col: &ArrayRef, row: usize, is_time: bool, epoch: Option<&str>) -> Value /// Decode a peer's `/internal/shard/query` response, an Arrow IPC stream, /// into series as its chunks arrive. An empty body is an empty result. pub async fn series_from_ipc_response( - mut resp: reqwest::Response, + mut stream: futures::stream::BoxStream< + 'static, + Result, + >, measurement: &str, epoch: Option<&str>, group_by_tags: &[String], ) -> Result, HyperbytedbError> { + use futures::StreamExt as _; let mut decoder = arrow::ipc::reader::StreamDecoder::new(); let mut builder = SeriesBuilder::new(measurement, epoch, group_by_tags); let mut received = false; - while let Some(chunk) = resp - .chunk() - .await - .map_err(|e| HyperbytedbError::PeerUnreachable(format!("shard query stream: {e}")))? - { + while let Some(chunk) = stream.next().await { + let chunk = chunk + .map_err(|e| HyperbytedbError::PeerUnreachable(format!("shard query stream: {e}")))?; received = true; let mut buf = arrow::buffer::Buffer::from_vec(chunk.to_vec()); while !buf.is_empty() { @@ -394,6 +396,69 @@ mod tests { (b.finish(), batches) } + /// A peer's Arrow IPC reply decodes the same however its bytes are + /// chunked on the wire: here whole, and split into 3-byte chunks. + #[tokio::test] + async fn shard_query_decodes_a_multi_chunk_stream() { + use arrow::array::{Float64Array, TimestampNanosecondArray}; + use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; + use arrow::record_batch::RecordBatch; + let schema = std::sync::Arc::new(Schema::new(vec![ + Field::new( + "__time", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + false, + ), + Field::new("v", DataType::Float64, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + std::sync::Arc::new( + TimestampNanosecondArray::from(vec![1_000_000_000i64, 2_000_000_000]) + .with_timezone("UTC"), + ), + std::sync::Arc::new(Float64Array::from(vec![1.5, 2.5])), + ], + ) + .unwrap(); + let mut ipc = Vec::new(); + { + let mut w = arrow::ipc::writer::StreamWriter::try_new(&mut ipc, &schema).unwrap(); + w.write(&batch).unwrap(); + w.finish().unwrap(); + } + let stream_of = |chunks: Vec| { + Box::pin(futures::stream::iter( + chunks + .into_iter() + .map(Ok::<_, crate::domain::cluster::peer_rpc::PeerRpcError>), + )) as futures::stream::BoxStream<'static, _> + }; + let whole = series_from_ipc_response( + stream_of(vec![bytes::Bytes::from(ipc.clone())]), + "m", + None, + &[], + ) + .await + .unwrap(); + let split = series_from_ipc_response( + stream_of(ipc.chunks(3).map(bytes::Bytes::copy_from_slice).collect()), + "m", + None, + &[], + ) + .await + .unwrap(); + assert_eq!(whole.len(), 1); + assert_eq!(whole[0].values.len(), 2); + assert_eq!( + serde_json::to_string(&whole).unwrap(), + serde_json::to_string(&split).unwrap() + ); + } + const VALUES: &str = "SELECT toDateTime64('2026-01-02 03:04:05.123456789', 9, 'UTC') + toIntervalSecond(number) AS __time, \ toFloat64(number) AS f_int, number + 0.5 AS f_frac, toFloat64('inf') AS f_inf, \ -toFloat64('inf') AS f_ninf, toFloat64('nan') AS f_nan, toFloat64(-0.) AS f_negzero, \ diff --git a/hyperbytedb/src/application/query_service.rs b/hyperbytedb/src/application/query_service.rs index 370c6f1..11047d8 100644 --- a/hyperbytedb/src/application/query_service.rs +++ b/hyperbytedb/src/application/query_service.rs @@ -2508,28 +2508,23 @@ async fn try_query_sharded_region( let req = req.clone(); let addr = addr.to_string(); async move { - let url = format!("http://{addr}/internal/shard/query"); - let resp = ctx - .peer_client - .http_client() - .post(&url) - .json(&req) - .timeout(timeout) - .send() - .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - let status = resp.status(); - if status == reqwest::StatusCode::CONFLICT { - return Err(HyperbytedbError::StaleShardEpoch { region_id }); - } - if !status.is_success() { - let body = resp.text().await.unwrap_or_default(); - return Err(HyperbytedbError::PeerUnreachable(format!( - "shard query to peer {peer_id} failed: {status} {body}" - ))); - } + use crate::domain::cluster::peer_rpc::PeerRpcError; + let stream = match ctx.peer_rpc.shard_query(&addr, &req, timeout).await { + Ok(stream) => stream, + Err(PeerRpcError::Status { code: 409, .. }) => { + return Err(HyperbytedbError::StaleShardEpoch { region_id }); + } + Err(PeerRpcError::Status { code, body }) => { + return Err(HyperbytedbError::PeerUnreachable(format!( + "shard query to peer {peer_id} failed: {code} {body}" + ))); + } + Err(PeerRpcError::Unreachable(e)) => { + return Err(HyperbytedbError::PeerUnreachable(e)); + } + }; crate::application::query_arrow_series::series_from_ipc_response( - resp, + stream, measurement, epoch, resolved_group_by_tags, @@ -2593,23 +2588,22 @@ async fn query_materialized_dest_node( &ctx.config, crate::application::shard_peer_resolution::ScatterKind::Query, ); - let resp = ctx - .peer_client - .http_client() - .post(format!("http://{addr}/internal/shard/query")) - .json(&req) - .timeout(timeout) - .send() + let stream = ctx + .peer_rpc + .shard_query(addr, &req, timeout) .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - if !resp.status().is_success() { - return Err(HyperbytedbError::PeerUnreachable(format!( - "materialized query to node {node_id} failed: {}", - resp.status() - ))); - } + .map_err(|e| match e { + crate::domain::cluster::peer_rpc::PeerRpcError::Status { code, .. } => { + HyperbytedbError::PeerUnreachable(format!( + "materialized query to node {node_id} failed: {code}" + )) + } + crate::domain::cluster::peer_rpc::PeerRpcError::Unreachable(e) => { + HyperbytedbError::PeerUnreachable(e) + } + })?; crate::application::query_arrow_series::series_from_ipc_response( - resp, + stream, measurement, epoch, resolved_group_by_tags, diff --git a/hyperbytedb/src/application/shard_metadata_scatter.rs b/hyperbytedb/src/application/shard_metadata_scatter.rs index 9458629..5d93ad9 100644 --- a/hyperbytedb/src/application/shard_metadata_scatter.rs +++ b/hyperbytedb/src/application/shard_metadata_scatter.rs @@ -45,27 +45,22 @@ async fn fetch_remote_metadata( let req = req.clone(); let addr = addr.to_string(); async move { - let url = format!("http://{addr}/internal/shard/metadata"); - let resp = ctx - .peer_client - .http_client() - .post(&url) - .json(&req) - .timeout(timeout) - .send() - .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - if resp.status() == reqwest::StatusCode::CONFLICT { - return Err(HyperbytedbError::StaleShardEpoch { region_id }); - } - if !resp.status().is_success() { - return Err(HyperbytedbError::PeerUnreachable(format!( - "metadata scatter to peer {peer_id} failed: {}", - resp.status() - ))); - } - resp.json() - .await + use crate::domain::cluster::peer_rpc::PeerRpcError; + let body = match ctx.peer_rpc.shard_metadata(&addr, &req, timeout).await { + Ok(body) => body, + Err(PeerRpcError::Status { code: 409, .. }) => { + return Err(HyperbytedbError::StaleShardEpoch { region_id }); + } + Err(PeerRpcError::Status { code, .. }) => { + return Err(HyperbytedbError::PeerUnreachable(format!( + "metadata scatter to peer {peer_id} failed: {code}" + ))); + } + Err(PeerRpcError::Unreachable(e)) => { + return Err(HyperbytedbError::PeerUnreachable(e)); + } + }; + serde_json::from_slice(&body) .map_err(|e| HyperbytedbError::Internal(e.to_string().into())) } }, diff --git a/hyperbytedb/src/application/shard_routing.rs b/hyperbytedb/src/application/shard_routing.rs index 7c04c3d..1039225 100644 --- a/hyperbytedb/src/application/shard_routing.rs +++ b/hyperbytedb/src/application/shard_routing.rs @@ -197,25 +197,21 @@ pub async fn ensure_measurement_bootstrapped( bootstrap_measurement_local(ctx, db, rp, measurement).await?; } } else if let Some(addr) = leader_addr { - let url = format!("http://{addr}/internal/shard/bootstrap"); let req = ShardBootstrapRequest { db: db.to_string(), rp: rp.to_string(), measurement: measurement.to_string(), }; - let resp = ctx - .peer_client - .http_client() - .post(&url) - .json(&req) - .timeout(timeout) - .send() - .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - if !resp.status().is_success() { - return Err(HyperbytedbError::ShardMap( - format!("bootstrap failed: {}", resp.status()).into(), - )); + match ctx.peer_rpc.shard_bootstrap(addr, &req, timeout).await { + Ok(()) => {} + Err(crate::domain::cluster::peer_rpc::PeerRpcError::Status { code, .. }) => { + return Err(HyperbytedbError::ShardMap( + format!("bootstrap failed: {code}").into(), + )); + } + Err(crate::domain::cluster::peer_rpc::PeerRpcError::Unreachable(e)) => { + return Err(HyperbytedbError::PeerUnreachable(e)); + } } } else { return Err(HyperbytedbError::ClusterUnavailable( @@ -641,7 +637,7 @@ pub async fn forward_shard_writes_by_node( } } let sent: Vec = parts.iter().map(|p| p.region_id).collect(); - let client = ctx.peer_client.http_client().clone(); + let rpc = ctx.peer_rpc.clone(); let coalescer = ctx.forward_coalescer.clone(); let coalesce = ctx.config.forward_coalesce; let span = tracing::debug_span!( @@ -653,10 +649,13 @@ pub async fn forward_shard_writes_by_node( sends.push(tracing::Instrument::instrument( async move { let outcome = if coalesce { - coalescer.send(&client, &addr, timeout, parts).await + coalescer.send(&rpc, &addr, timeout, parts).await } else { crate::application::forward_coalescer::send_direct( - &client, &addr, timeout, parts, + rpc.as_ref(), + &addr, + timeout, + parts, ) .await }; @@ -764,19 +763,6 @@ pub(crate) fn refused(message: &str, points: usize) -> HyperbytedbError { } /// What to do with one part of a batch response. -/// `e` and its causes, joined: reqwest's own message ("error sending request -/// for url ...") hides whether the peer refused, reset or timed out. -pub(crate) fn error_chain(e: &dyn std::error::Error) -> String { - let mut out = e.to_string(); - let mut cause = e.source(); - while let Some(c) = cause { - out.push_str(": "); - out.push_str(&c.to_string()); - cause = c.source(); - } - out -} - #[derive(Debug)] pub(crate) enum PartOutcome { Done(Result<(), HyperbytedbError>), @@ -899,51 +885,43 @@ async fn try_forward_shard_write_to_region( let precision = precision_owned.clone(); let epoch_header = epoch_header.clone(); async move { - let url = format!("http://{addr}/internal/shard/write"); - let mut req = ctx - .peer_client - .http_client() - .post(&url) - .header("Content-Type", "application/octet-stream") - .header("X-HyperbyteDB-Shard-Db", &db) - .header("X-HyperbyteDB-Shard-Rp", &rp) - .header("X-HyperbyteDB-Shard-Region-Id", region_id.to_string()) - .header("X-HyperbyteDB-Shard-Epoch", epoch_header) - .body(body); - if let Some(p) = precision.as_deref() { - req = req.header("X-HyperbyteDB-Shard-Precision", p); - } - let resp = req - .timeout(timeout) - .send() + use crate::domain::cluster::peer_rpc::{PeerRpcError, ShardWriteTarget}; + let target = ShardWriteTarget { + db: db.clone(), + rp: rp.clone(), + region_id, + epoch_json: epoch_header, + precision, + }; + match ctx + .peer_rpc + .shard_write(&addr, &target, body.into(), timeout) .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - - if resp.status() == reqwest::StatusCode::CONFLICT { - ctx.location_cache - .invalidate_measurement(&MeasurementKey::new(&db, &rp, &meas)); - return Err(HyperbytedbError::StaleShardEpoch { region_id }); - } - // The primary is shedding load: say so, as a retryable 503, - // rather than a 502 the client counts as a hard failure. - if resp.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE - || resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS { - return Err(HyperbytedbError::ClusterUnavailable(format!( + Ok(()) => Ok(()), + Err(PeerRpcError::Status { code: 409, .. }) => { + ctx.location_cache + .invalidate_measurement(&MeasurementKey::new(&db, &rp, &meas)); + Err(HyperbytedbError::StaleShardEpoch { region_id }) + } + // The primary is shedding load: say so, as a retryable 503, + // rather than a 502 the client counts as a hard failure. + Err(PeerRpcError::Status { + code: 503 | 429, .. + }) => Err(HyperbytedbError::ClusterUnavailable(format!( "region {region_id} primary {peer_id} is at write capacity" - ))); - } - if matches!(resp.status().as_u16(), 400 | 422) { - let body = resp.text().await.unwrap_or_default(); - return Err(refused(&body, point_count)); - } - if !resp.status().is_success() { - return Err(HyperbytedbError::PeerUnreachable(format!( - "forward write to peer {peer_id} failed: {}", - resp.status() - ))); + ))), + Err(PeerRpcError::Status { + code: 400 | 422, + body, + }) => Err(refused(&body, point_count)), + Err(PeerRpcError::Status { code, .. }) => { + Err(HyperbytedbError::PeerUnreachable(format!( + "forward write to peer {peer_id} failed: {code}" + ))) + } + Err(PeerRpcError::Unreachable(e)) => Err(HyperbytedbError::PeerUnreachable(e)), } - Ok(()) } }, ) @@ -1087,17 +1065,9 @@ pub async fn scatter_delete_to_regions( }, }; let timeout = Duration::from_millis(ctx.config.scatter_peer_timeout_ms.max(1)); - let url = format!("http://{addr}/internal/shard/delete"); - let resp = ctx - .peer_client - .http_client() - .post(&url) - .json(&req) - .timeout(timeout) - .send() - .await; - match resp { - Ok(r) if r.status() == reqwest::StatusCode::CONFLICT => { + use crate::domain::cluster::peer_rpc::PeerRpcError; + match ctx.peer_rpc.shard_delete(&addr, &req, timeout).await { + Err(PeerRpcError::Status { code: 409, .. }) => { refresh_location_cache(ctx).await?; outcome.failed += 1; counter!("hyperbytedb_shard_delete_failures_total", "reason" => "stale_epoch") @@ -1107,17 +1077,17 @@ pub async fn scatter_delete_to_regions( "shard delete stale epoch; retry on next request" ); } - Ok(r) if !r.status().is_success() => { + Err(PeerRpcError::Status { code, .. }) => { outcome.failed += 1; counter!("hyperbytedb_shard_delete_failures_total", "reason" => "http_status") .increment(1); tracing::warn!( - status = %r.status(), + status = code, region_id = region.region_id, "shard delete scatter failed for region" ); } - Err(e) => { + Err(e @ PeerRpcError::Unreachable(_)) => { outcome.failed += 1; counter!("hyperbytedb_shard_delete_failures_total", "reason" => "transport") .increment(1); @@ -1127,7 +1097,7 @@ pub async fn scatter_delete_to_regions( "shard delete scatter failed for region" ); } - _ => { + Ok(()) => { outcome.applied += 1; } } diff --git a/hyperbytedb/src/domain/cluster/peer_rpc.rs b/hyperbytedb/src/domain/cluster/peer_rpc.rs index ddb7272..1058ed1 100644 --- a/hyperbytedb/src/domain/cluster/peer_rpc.rs +++ b/hyperbytedb/src/domain/cluster/peer_rpc.rs @@ -6,6 +6,17 @@ use std::fmt; use crate::error::HyperbytedbError; +/// Where a forwarded `/internal/shard/write` goes: the headers it carries. +#[derive(Debug, Clone)] +pub struct ShardWriteTarget { + pub db: String, + pub rp: String, + pub region_id: u64, + /// The region epoch, JSON-encoded, as the receiver parses it. + pub epoch_json: String, + pub precision: Option, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum PeerRpcError { /// The peer answered with a non-success status. diff --git a/hyperbytedb/src/error.rs b/hyperbytedb/src/error.rs index 9075f6a..bb476b7 100644 --- a/hyperbytedb/src/error.rs +++ b/hyperbytedb/src/error.rs @@ -231,6 +231,19 @@ impl From for HyperbytedbError { } } +/// `e` and its causes, joined: reqwest's own message ("error sending request +/// for url ...") hides whether the peer refused, reset or timed out. +pub fn error_chain(e: &dyn std::error::Error) -> String { + let mut out = e.to_string(); + let mut cause = e.source(); + while let Some(c) = cause { + out.push_str(": "); + out.push_str(&c.to_string()); + cause = c.source(); + } + out +} + #[cfg(test)] mod tests { use super::{ChainedError, HyperbytedbError}; diff --git a/hyperbytedb/src/ports/peer_rpc.rs b/hyperbytedb/src/ports/peer_rpc.rs index 43d2cb0..87690f6 100644 --- a/hyperbytedb/src/ports/peer_rpc.rs +++ b/hyperbytedb/src/ports/peer_rpc.rs @@ -2,8 +2,70 @@ //! application uses this instead of building HTTP requests itself. use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use futures::stream::BoxStream; + +use crate::domain::cluster::peer_rpc::{PeerRpcError, ShardWriteTarget}; +use crate::domain::sharding::types::{ + ShardBootstrapRequest, ShardDeleteRequest, ShardMetadataRequest, ShardQueryRequest, + ShardWriteBatchResponse, +}; + +/// A peer's reply body as it arrives. +pub type ByteStream = BoxStream<'static, Result>; #[async_trait::async_trait] -pub trait PeerRpc: Send + Sync {} +pub trait PeerRpc: Send + Sync { + /// `POST /internal/shard/bootstrap`. + async fn shard_bootstrap( + &self, + addr: &str, + req: &ShardBootstrapRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError>; + + /// `POST /internal/shard/write`: line protocol for one region. + async fn shard_write( + &self, + addr: &str, + target: &ShardWriteTarget, + body: Bytes, + timeout: Duration, + ) -> Result<(), PeerRpcError>; + + /// `POST /internal/shard/write-batch`: a bincode `ShardWriteBatch`. + async fn shard_write_batch( + &self, + addr: &str, + body: Bytes, + timeout: Duration, + ) -> Result; + + /// `POST /internal/shard/query`: the Arrow IPC reply, as it streams. + async fn shard_query( + &self, + addr: &str, + req: &ShardQueryRequest, + timeout: Duration, + ) -> Result; + + /// `POST /internal/shard/metadata`: the JSON reply body. + async fn shard_metadata( + &self, + addr: &str, + req: &ShardMetadataRequest, + timeout: Duration, + ) -> Result; + + /// `POST /internal/shard/delete`. + async fn shard_delete( + &self, + addr: &str, + req: &ShardDeleteRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError>; +} pub type SharedPeerRpc = Arc; From 2f0f16e337c1ceb02ca5f3ea98cbf3ab4e2967f0 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Tue, 10 Nov 2026 14:16:13 +0000 Subject: [PATCH 4/8] Transfers, backfill, handoff and heartbeats go through PeerRpc --- .../src/adapters/cluster/http_peer_rpc.rs | 205 +++++++++++++++++- .../src/adapters/http/shard_handlers.rs | 7 +- hyperbytedb/src/application/cluster/drain.rs | 12 +- .../application/runtime/shard_heartbeat.rs | 44 ++-- hyperbytedb/src/application/shard_routing.rs | 11 + .../src/application/shard_scheduler.rs | 190 ++++++++-------- hyperbytedb/src/application/shard_transfer.rs | 148 +++++-------- .../src/application/sharded_mv_backfill.rs | 41 ++-- hyperbytedb/src/ports/peer_rpc.rs | 80 ++++++- hyperbytedb/src/server.rs | 11 +- hyperbytedb/tests/common/sharding_cluster.rs | 4 +- .../tests/sharding_cluster_integration.rs | 12 +- 12 files changed, 500 insertions(+), 265 deletions(-) diff --git a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs index 50e2978..91aa9ab 100644 --- a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs +++ b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs @@ -8,9 +8,13 @@ use bytes::Bytes; use crate::adapters::cluster::peer_client::PeerClient; use crate::domain::cluster::peer_rpc::{PeerRpcError, ShardWriteTarget}; +use crate::domain::sharding::transfer::{ + ShardRehomeRequest, ShardTransferPayload, TRANSFER_PAYLOAD_HEADER, +}; use crate::domain::sharding::types::{ - ShardBootstrapRequest, ShardDeleteRequest, ShardMetadataRequest, ShardQueryRequest, - ShardWriteBatchResponse, + FEATURE_CROSS_EPOCH_MERGE, MvHandoffRequest, NODE_FEATURES_HEADER, REHOME_PROGRESS_HEADER, + RegionHeartbeat, ShardBootstrapRequest, ShardDeleteRequest, ShardMetadataRequest, + ShardMvBackfillRequest, ShardQueryRequest, ShardWriteBatchResponse, }; use crate::error::error_chain; use crate::ports::peer_rpc::{ByteStream, PeerRpc}; @@ -24,6 +28,15 @@ pub struct HttpPeerRpc { clients: Clients, } +/// The port on `pc`'s pooled clients, with its membership. +#[must_use] +pub fn peers_for(pc: Arc) -> crate::ports::peer_rpc::Peers { + crate::ports::peer_rpc::Peers { + membership: pc.membership().clone(), + rpc: Arc::new(HttpPeerRpc::new(pc)), + } +} + impl HttpPeerRpc { #[must_use] pub fn new(peers: Arc) -> Self { @@ -205,6 +218,146 @@ impl PeerRpc for HttpPeerRpc { .await .map(|_| ()) } + + async fn shard_transfer_push( + &self, + addr: &str, + mut payload: ShardTransferPayload, + timeout: Duration, + ) -> Result { + use base64::Engine as _; + // Bytes: the fallback below resends it without a copy. + let body = Bytes::from(payload.body.take().unwrap_or_default()); + let header = serde_json::to_vec(&payload) + .map(|json| base64::engine::general_purpose::STANDARD.encode(json)) + .map_err(|e| PeerRpcError::Unreachable(format!("transfer header: {e}")))?; + let raw = self + .client() + .post(format!("http://{addr}/internal/shard/transfer/raw")) + .header(TRANSFER_PAYLOAD_HEADER, header) + .header(reqwest::header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(body.clone()) + .timeout(timeout) + .send() + .await + .map_err(|e| PeerRpcError::Unreachable(error_chain(&e)))?; + let resp = if raw.status() == reqwest::StatusCode::NOT_FOUND { + // An older node, or a region the receiver does not know: the JSON + // route answers the second the same way. + payload.body = Some(body.to_vec()); + self.post_json(addr, "/internal/shard/transfer", &payload, timeout) + .await? + } else { + Self::checked(Ok(raw)).await? + }; + Ok(resp.text().await.unwrap_or_default()) + } + + async fn shard_transfer_ack( + &self, + addr: &str, + payload: &ShardTransferPayload, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + self.post_json(addr, "/internal/shard/transfer", payload, timeout) + .await + .map(|_| ()) + } + + async fn shard_mv_backfill( + &self, + addr: &str, + req: &ShardMvBackfillRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + self.post_json(addr, "/internal/shard/mv-backfill", req, timeout) + .await + .map(|_| ()) + } + + async fn mv_handoff( + &self, + addr: &str, + req: &MvHandoffRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + self.post_json(addr, "/internal/mv/handoff", req, timeout) + .await + .map(|_| ()) + } + + async fn shard_heartbeat( + &self, + addr: &str, + hb: &RegionHeartbeat, + timeout: Duration, + ) -> Result<(), PeerRpcError> { + let resp = self + .client() + .post(format!("http://{addr}/internal/shard/heartbeat")) + .header(NODE_FEATURES_HEADER, FEATURE_CROSS_EPOCH_MERGE) + .json(hb) + .timeout(timeout) + .send() + .await; + Self::checked(resp).await.map(|_| ()) + } + + async fn shard_rehome( + &self, + addr: &str, + req: &ShardRehomeRequest, + idle: Duration, + max: Duration, + ) -> Result { + let resp = self + .client() + .post(format!("http://{addr}/internal/shard/rehome")) + .header(REHOME_PROGRESS_HEADER, "1") + .json(req) + .timeout(max) + .send() + .await; + let mut resp = Self::checked(resp).await?; + let mut raw = Vec::new(); + loop { + match tokio::time::timeout(idle, resp.chunk()).await { + Err(_) => { + return Err(PeerRpcError::Unreachable(format!( + "rehome silent for {idle:?}" + ))); + } + Ok(Err(e)) => return Err(PeerRpcError::Unreachable(error_chain(&e))), + Ok(Ok(None)) => break, + Ok(Ok(Some(chunk))) => raw.extend_from_slice(&chunk), + } + } + serde_json::from_slice(&raw) + .map_err(|e| PeerRpcError::Unreachable(format!("rehome result: {e}"))) + } + + async fn sync_manifest( + &self, + addr: &str, + timeout: Duration, + ) -> Result { + let resp = self.get(addr, "/internal/sync/manifest", timeout).await?; + let body = Self::body(resp).await?; + serde_json::from_slice(&body) + .map_err(|e| PeerRpcError::Unreachable(format!("parse sync manifest: {e}"))) + } + + async fn shard_map( + &self, + addr: &str, + timeout: Duration, + ) -> Result { + let resp = self.get(addr, "/internal/shard/map", timeout).await?; + let body = Self::body(resp).await?; + let json: crate::domain::sharding::ShardMapJson = serde_json::from_slice(&body) + .map_err(|e| PeerRpcError::Unreachable(format!("parse shard map: {e}")))?; + Ok(crate::domain::sharding::ShardMap::from(json)) + } } #[cfg(test)] @@ -263,4 +416,52 @@ mod tests { .unwrap_err(); assert!(matches!(err, PeerRpcError::Unreachable(_)), "{err:?}"); } + + /// An older peer without the raw route answers 404; the push is then + /// sent once as JSON, as before. + #[tokio::test] + async fn transfer_push_falls_back_to_json_on_404() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let raw = Arc::new(AtomicUsize::new(0)); + let json = Arc::new(AtomicUsize::new(0)); + let (r, j) = (raw.clone(), json.clone()); + let app = axum::Router::new() + .route( + "/internal/shard/transfer/raw", + axum::routing::post(move || { + r.fetch_add(1, Ordering::SeqCst); + async { axum::http::StatusCode::NOT_FOUND } + }), + ) + .route( + "/internal/shard/transfer", + axum::routing::post(move || { + j.fetch_add(1, Ordering::SeqCst); + async { "applied=3" } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let rpc = HttpPeerRpc::with_client(reqwest::Client::new()); + let payload = crate::domain::sharding::transfer::ShardTransferPayload::ack( + "db", + "autogen", + "cpu", + 1, + 0, + u64::MAX, + Default::default(), + 1, + 7, + ); + let body = rpc + .shard_transfer_push(&addr, payload, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(body, "applied=3"); + assert_eq!(raw.load(Ordering::SeqCst), 1); + assert_eq!(json.load(Ordering::SeqCst), 1); + } } diff --git a/hyperbytedb/src/adapters/http/shard_handlers.rs b/hyperbytedb/src/adapters/http/shard_handlers.rs index 7fb7a64..4b84ee8 100644 --- a/hyperbytedb/src/adapters/http/shard_handlers.rs +++ b/hyperbytedb/src/adapters/http/shard_handlers.rs @@ -1440,10 +1440,11 @@ async fn run_rehome( serde_json::json!({"error": "no peer client"}), ); }; + let peers = crate::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()); let region = ®ion; let outcome = match if req.stage { crate::application::shard_transfer::stage_region_transfer_data( - peer_client, + &peers, &state.metadata, &state.wal, Some(&state.query_port), @@ -1457,7 +1458,7 @@ async fn run_rehome( .await } else { push_region_transfer_data( - peer_client, + &peers, &state.metadata, &state.wal, Some(&state.query_port), @@ -1490,7 +1491,7 @@ async fn run_rehome( if !req.stage && let Err(e) = complete_region_transfer( - peer_client, + &peers, &state.metadata, Some(&state.points_sink), state.node_id, diff --git a/hyperbytedb/src/application/cluster/drain.rs b/hyperbytedb/src/application/cluster/drain.rs index f3f9375..c537391 100644 --- a/hyperbytedb/src/application/cluster/drain.rs +++ b/hyperbytedb/src/application/cluster/drain.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use crate::adapters::cluster::peer_client::PeerClient; use crate::adapters::cluster::raft::HyperbytedbRaft; use crate::application::shard_routing::ShardRoutingContext; use crate::application::shard_transfer::run_region_transfer; @@ -29,7 +28,6 @@ pub struct DrainService { replication_log: Arc, wal: Arc, shard_routing: Option>, - peer_client: Option>, metadata: Option>, points_sink: Option>, query_port: Option>, @@ -60,7 +58,6 @@ impl DrainService { replication_log, wal, shard_routing: None, - peer_client: None, metadata: None, points_sink: None, query_port: None, @@ -74,7 +71,6 @@ impl DrainService { pub fn with_sharding( mut self, ctx: Arc, - peer_client: Arc, metadata: Arc, points_sink: Arc, query_port: Arc, @@ -82,7 +78,6 @@ impl DrainService { max_points_per_request: usize, ) -> Self { self.shard_routing = Some(ctx); - self.peer_client = Some(peer_client); self.metadata = Some(metadata); self.points_sink = Some(points_sink); self.query_port = Some(query_port); @@ -197,10 +192,7 @@ impl DrainService { Some(c) => c, None => return Ok(()), }; - let pc = self - .peer_client - .as_ref() - .ok_or_else(|| HyperbytedbError::ClusterUnavailable("no peer client".into()))?; + let peers = ctx.peers(); let metadata = self .metadata .as_ref() @@ -224,7 +216,7 @@ impl DrainService { if copy { run_region_transfer( - pc, + &peers, metadata, &self.wal, self.query_port.as_ref(), diff --git a/hyperbytedb/src/application/runtime/shard_heartbeat.rs b/hyperbytedb/src/application/runtime/shard_heartbeat.rs index 7e1568a..4f5b20a 100644 --- a/hyperbytedb/src/application/runtime/shard_heartbeat.rs +++ b/hyperbytedb/src/application/runtime/shard_heartbeat.rs @@ -5,16 +5,15 @@ use std::sync::Arc; use std::time::Duration; use metrics::counter; -use reqwest::StatusCode; use tokio::sync::watch; -use crate::adapters::cluster::peer_client::PeerClient; use crate::application::runtime::RegionWriteStats; use crate::domain::chdb_naming::unquoted_table_name; use crate::domain::cluster::membership::SharedMembership; use crate::domain::sharding::{RegionHeartbeat, ShardMap, ShardRegion}; use crate::ports::leadership::LeadershipPort; use crate::ports::metadata::MetadataPort; +use crate::ports::peer_rpc::{PeerRpc, SharedPeerRpc}; use crate::ports::query::QueryPort; use crate::ports::sharding::ShardMapPort; @@ -36,7 +35,7 @@ struct ByteEstimateCache { #[allow(clippy::too_many_arguments)] pub async fn run_region_heartbeat_reporter( node_id: u64, - peer_client: Arc, + peer_rpc: SharedPeerRpc, membership: SharedMembership, shard_map: Arc, leadership: Arc, @@ -57,7 +56,7 @@ pub async fn run_region_heartbeat_reporter( byte_estimates.tick = byte_estimates.tick.wrapping_add(1); if let Err(e) = report_once( node_id, - &peer_client, + peer_rpc.as_ref(), &membership, shard_map.as_ref(), leadership.as_ref(), @@ -83,7 +82,7 @@ pub async fn run_region_heartbeat_reporter( #[allow(clippy::too_many_arguments)] async fn report_once( node_id: u64, - peer_client: &PeerClient, + peer_rpc: &dyn PeerRpc, membership: &SharedMembership, shard_map: &dyn ShardMapPort, leadership: &dyn LeadershipPort, @@ -197,8 +196,7 @@ async fn report_once( epoch: region.epoch, split_key_hint, }; - let url = format!("http://{leader_addr}/internal/shard/heartbeat"); - if let Err(reason) = post_region_heartbeat(peer_client, &url, &hb).await { + if let Err(reason) = post_region_heartbeat(peer_rpc, &leader_addr, &hb).await { if reason == "stale_epoch" { let fresh_map = shard_map.snapshot().await.map_err(|e| e.to_string())?; if let Some(fresh) = lookup_region( @@ -210,7 +208,7 @@ async fn report_once( ) { hb.epoch = fresh.epoch; if let Err(retry_reason) = - post_region_heartbeat(peer_client, &url, &hb).await + post_region_heartbeat(peer_rpc, &leader_addr, &hb).await { record_heartbeat_failure(retry_reason); } @@ -240,30 +238,20 @@ fn lookup_region<'a>( } async fn post_region_heartbeat( - peer_client: &PeerClient, - url: &str, + peer_rpc: &dyn PeerRpc, + leader_addr: &str, hb: &RegionHeartbeat, ) -> Result<(), &'static str> { - let resp = peer_client - .http_client() - .post(url) - .header( - crate::domain::sharding::types::NODE_FEATURES_HEADER, - crate::domain::sharding::types::FEATURE_CROSS_EPOCH_MERGE, - ) - .json(hb) - .timeout(Duration::from_secs(5)) - .send() + use crate::domain::cluster::peer_rpc::PeerRpcError; + match peer_rpc + .shard_heartbeat(leader_addr, hb, Duration::from_secs(5)) .await - .map_err(|_| "transport")?; - - if resp.status() == StatusCode::CONFLICT { - return Err("stale_epoch"); - } - if !resp.status().is_success() { - return Err("http_status"); + { + Ok(()) => Ok(()), + Err(PeerRpcError::Status { code: 409, .. }) => Err("stale_epoch"), + Err(PeerRpcError::Status { .. }) => Err("http_status"), + Err(PeerRpcError::Unreachable(_)) => Err("transport"), } - Ok(()) } fn record_heartbeat_failure(reason: &'static str) { diff --git a/hyperbytedb/src/application/shard_routing.rs b/hyperbytedb/src/application/shard_routing.rs index 1039225..655a78b 100644 --- a/hyperbytedb/src/application/shard_routing.rs +++ b/hyperbytedb/src/application/shard_routing.rs @@ -42,6 +42,17 @@ pub struct ShardRoutingContext { pub runtime_settings: Arc, } +impl ShardRoutingContext { + /// The peer port and membership, for the transfer functions. + #[must_use] + pub fn peers(&self) -> crate::ports::peer_rpc::Peers { + crate::ports::peer_rpc::Peers { + rpc: self.peer_rpc.clone(), + membership: self.membership.clone(), + } + } +} + pub struct PointBuckets { /// region_id -> points this node primaries locally pub local: HashMap>, diff --git a/hyperbytedb/src/application/shard_scheduler.rs b/hyperbytedb/src/application/shard_scheduler.rs index b219414..24d4890 100644 --- a/hyperbytedb/src/application/shard_scheduler.rs +++ b/hyperbytedb/src/application/shard_scheduler.rs @@ -5,7 +5,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use metrics::counter; use tokio::sync::watch; -use crate::adapters::cluster::peer_client::PeerClient; use crate::adapters::cluster::raft::HyperbytedbRaft; use crate::application::runtime_settings as rs; use crate::application::shard_peer_resolution::is_active_peer; @@ -24,6 +23,7 @@ use crate::domain::sharding::{ }; use crate::error::HyperbytedbError; use crate::ports::metadata::MetadataPort; +use crate::ports::peer_rpc::{PeerRpc, Peers}; use crate::ports::query::QueryPort; use crate::ports::sharding::ShardMapPort; use crate::ports::wal::WalPort; @@ -254,7 +254,7 @@ pub struct ShardScheduler { shard_map: Arc, membership: SharedMembership, raft: HyperbytedbRaft, - peer_client: Option>, + peers: Option, metadata: Arc, wal: Arc, query_port: Option>, @@ -347,7 +347,7 @@ impl ShardScheduler { shard_map: Arc, membership: SharedMembership, raft: HyperbytedbRaft, - peer_client: Option>, + peers: Option, metadata: Arc, wal: Arc, query_port: Option>, @@ -364,7 +364,7 @@ impl ShardScheduler { shard_map, membership, raft, - peer_client, + peers, metadata, wal, query_port, @@ -423,11 +423,11 @@ impl ShardScheduler { peer: u64, addr: &str, cluster_ver: u64, - pc: &PeerClient, + peers: &Peers, ) -> Option<(u64, u64)> { let cell = self.probe_cell(&self.peer_map_versions, peer).await; *cell - .get_or_init(|| self.probe_peer_map_version(peer, addr, cluster_ver, pc)) + .get_or_init(|| self.probe_peer_map_version(peer, addr, cluster_ver, peers)) .await } @@ -454,9 +454,15 @@ impl ShardScheduler { peer: u64, addr: &str, cluster_ver: u64, - pc: &PeerClient, + peers: &Peers, ) -> Option<(u64, u64)> { - match pc.fetch_map_version(addr).await { + // 10 s: the pooled peer client's default, which this probe used. + match peers + .rpc + .shard_map(addr, Duration::from_secs(10)) + .await + .map(|m| m.map_version) + { Ok(version) => Some((version, cluster_ver)), Err(e) => { // Warn, not debug: the shipped image pins the log level to @@ -1386,7 +1392,7 @@ impl ShardScheduler { key: &MeasurementKey, region: &ShardRegion, ) -> Option<(MoveTier, PlacementStep)> { - if self.is_rollup_dest(key).await || self.peer_client.is_none() { + if self.is_rollup_dest(key).await || self.peers.is_none() { return None; } let map = self.shard_map.snapshot().await.ok()?; @@ -1539,13 +1545,8 @@ impl ShardScheduler { return Ok(()); } - let new_primary = pick_best_primary( - &self.membership, - region, - self.node_id, - self.peer_client.as_deref(), - ) - .await; + let new_primary = + pick_best_primary(&self.membership, region, self.node_id, self.peers.as_ref()).await; let Some(new_primary) = new_primary else { // No serving peer survives. If a learner does, promote it and hand // it primacy in the same tick rather than leaving the region with @@ -1603,7 +1604,7 @@ impl ShardScheduler { // watermark lags the best known for this region. Moving primary to a // data-less node converts a visible availability blip into silent // empty reads; blocking keeps reads routed at whoever holds the data. - if self.peer_client.is_some() { + if self.peers.is_some() { let progress = self.region_progress_for_peers(region).await; // `flatten` drops unknowns rather than reading them as 0: only // progress we actually heard take part in the comparison. @@ -1930,7 +1931,7 @@ impl ShardScheduler { if self.rollup_dests.read().await.is_empty() { return true; } - let Some(pc) = self.peer_client.clone() else { + let Some(peers) = self.peers.clone() else { return true; }; let mut handoffs = self.rollup_handoffs.lock().await; @@ -1961,23 +1962,22 @@ impl ShardScheduler { let req = crate::application::mv_handoff::MvHandoffRequest { to_addr: to.clone(), }; - let result = pc - .http_client() - .post(format!("http://{from}/internal/mv/handoff")) - .json(&req) - .timeout(Duration::from_secs(30 * 60)) - .send() + let result = peers + .rpc + .mv_handoff(&from, &req, Duration::from_secs(30 * 60)) .await; let mut handoffs = handoffs.lock().await; let state = handoffs.entry(node_id).or_default(); state.running = false; match result { - Ok(resp) if resp.status().is_success() => { + Ok(()) => { state.done = true; tracing::info!(node_id, to = %to, "view rollups handed off"); } // A node too old to hand off: it leaves as before. - Ok(resp) if resp.status() == reqwest::StatusCode::NOT_FOUND => { + Err(crate::domain::cluster::peer_rpc::PeerRpcError::Status { + code: 404, .. + }) => { state.done = true; tracing::warn!( node_id, @@ -1986,8 +1986,12 @@ impl ShardScheduler { } other => { let why = match other { - Ok(resp) => format!("status {}", resp.status()), + Err(crate::domain::cluster::peer_rpc::PeerRpcError::Status { + code, + .. + }) => format!("status {code}"), Err(e) => e.to_string(), + Ok(()) => String::new(), }; let since = *state .failing_since @@ -2059,7 +2063,7 @@ impl ShardScheduler { } // No peer client means nothing can be staged; skip quietly rather than // erroring every tick. - if self.peer_client.is_none() { + if self.peers.is_none() { return Ok(false); } // Re-read: earlier steps of the tick can bump epochs, so the caller's @@ -2218,7 +2222,7 @@ impl ShardScheduler { "promoting despite watermark guard: primary is departing" ); } - if !evacuating && self.peer_client.is_some() { + if !evacuating && self.peers.is_some() { let progress = self.region_progress_for_peers(region).await; let candidate_p = progress.get(&new_primary).copied().flatten(); let best_p = progress.values().copied().flatten().max(); @@ -2288,12 +2292,12 @@ impl ShardScheduler { to_peer: u64, max_bytes_per_sec: u64, ) -> Result { - let Some(pc) = self.peer_client.as_ref() else { + let Some(peers) = self.peers.as_ref() else { return Err(HyperbytedbError::ShardMap("no peer client".into())); }; if region.primary == self.node_id { let outcome = stage_region_transfer_data( - pc, + peers, &self.metadata, &self.wal, self.query_port.as_ref(), @@ -2320,7 +2324,7 @@ impl ShardScheduler { // counter, and its only error log was the `debug!` in the tick. A // leader failing to stage onto every region's primary was silent. match request_region_stage( - pc.as_ref(), + peers.rpc.as_ref(), &self.membership, region.primary, key, @@ -2354,7 +2358,7 @@ impl ShardScheduler { cluster_ver: u64, candidates: &[(u64, String)], ) -> Option { - let pc = self.peer_client.as_ref()?; + let peers = self.peers.as_ref()?; // Preserve the original accounting: no candidates is not a stall. if candidates.is_empty() { self.clear_placement_stall(region_id).await; @@ -2362,7 +2366,7 @@ impl ShardScheduler { } for (id, addr) in candidates { let Some((ver, probed_against)) = self - .cached_peer_map_version(*id, addr, cluster_ver, pc) + .cached_peer_map_version(*id, addr, cluster_ver, peers) .await else { continue; @@ -2408,7 +2412,7 @@ impl ShardScheduler { { return Ok(()); } - if self.peer_client.is_none() { + if self.peers.is_none() { return Ok(()); } @@ -2522,39 +2526,27 @@ const REHOME_MAX_SECS: u64 = 6 * 3600; /// on total time: a large region under write load took longer than any fixed /// total, and each timeout threw the copy away to start again from nothing. async fn post_rehome( - http: &reqwest::Client, - url: &str, + rpc: &dyn PeerRpc, + addr: &str, req: &ShardRehomeRequest, idle: Duration, ) -> Result { - use crate::domain::sharding::types::REHOME_PROGRESS_HEADER; - let unreachable = |e: String| HyperbytedbError::PeerUnreachable(format!("{url}: {e}")); - let mut resp = http - .post(url) - .header(REHOME_PROGRESS_HEADER, "1") - .json(req) - .timeout(Duration::from_secs(REHOME_MAX_SECS)) - .send() + use crate::domain::cluster::peer_rpc::PeerRpcError; + let body = match rpc + .shard_rehome(addr, req, idle, Duration::from_secs(REHOME_MAX_SECS)) .await - .map_err(|e| unreachable(e.to_string()))?; - if !resp.status().is_success() { - let status = resp.status().as_u16(); - return Err(HyperbytedbError::TransferRejected { - status, - reason: crate::application::shard_transfer::rejection_reason(resp).await, - }); - } - let mut raw = Vec::new(); - loop { - match tokio::time::timeout(idle, resp.chunk()).await { - Err(_) => return Err(unreachable(format!("rehome silent for {idle:?}"))), - Ok(Err(e)) => return Err(unreachable(e.to_string())), - Ok(Ok(None)) => break, - Ok(Ok(Some(chunk))) => raw.extend_from_slice(&chunk), + { + Ok(body) => body, + Err(PeerRpcError::Status { code, body }) => { + return Err(HyperbytedbError::TransferRejected { + status: code, + reason: crate::application::shard_transfer::rejection_reason(&body), + }); } - } - let body: serde_json::Value = - serde_json::from_slice(&raw).map_err(|e| unreachable(format!("rehome result: {e}")))?; + Err(PeerRpcError::Unreachable(e)) => { + return Err(HyperbytedbError::PeerUnreachable(format!("{addr}: {e}"))); + } + }; // Streamed results carry their status in the body: the 200 went out // before the copy finished. let status = body @@ -2699,7 +2691,7 @@ pub fn current_region<'a>( /// Ask the current primary to stage `[start, end)` onto a joiner that is /// not yet a committed peer. async fn request_region_stage( - peer_client: &PeerClient, + rpc: &dyn PeerRpc, membership: &SharedMembership, target_node: u64, key: &MeasurementKey, @@ -2729,13 +2721,7 @@ async fn request_region_stage( // Accepted and ignored by the receiver; retained only for wire compat. dest_skip_wal_through: 0, }; - let body = post_rehome( - peer_client.http_client(), - &format!("http://{addr}/internal/shard/rehome"), - &req, - Duration::from_secs(REHOME_IDLE_SECS), - ) - .await?; + let body = post_rehome(rpc, &addr, &req, Duration::from_secs(REHOME_IDLE_SECS)).await?; // Bytes, not points: the budget is a bandwidth throttle. An older peer that // does not report `bytes` contributes 0 rather than a point count — // under-charging is a convergence-rate cost, while charging points as if @@ -2747,7 +2733,7 @@ async fn pick_best_primary( membership: &SharedMembership, region: &ShardRegion, self_id: u64, - peer_client: Option<&PeerClient>, + peers: Option<&Peers>, ) -> Option { let m = membership.read().await; let candidates: Vec = region @@ -2771,11 +2757,11 @@ async fn pick_best_primary( return None; } - if let Some(pc) = peer_client { + if let Some(peers) = peers { let wms = futures::future::join_all(candidates.iter().map(|&candidate| async move { ( candidate, - peer_region_wal_watermark(pc, candidate, region.region_id).await, + peer_region_wal_watermark(peers, candidate, region.region_id).await, ) })) .await; @@ -2807,23 +2793,16 @@ async fn pick_best_primary( /// not reach this node" is not "this node has applied nothing". Collapsing /// them let one timed-out probe read as a data-less candidate, which the /// promotion guard then acted on. -async fn peer_region_wal_watermark( - peer_client: &PeerClient, - peer_id: u64, - region_id: u64, -) -> Option { - let membership = peer_client.membership().read().await; +async fn peer_region_wal_watermark(peers: &Peers, peer_id: u64, region_id: u64) -> Option { + let membership = peers.membership.read().await; let addr = membership.get_node(peer_id)?.addr.clone(); drop(membership); - let url = format!("http://{addr}/internal/sync/manifest"); - let Ok(resp) = peer_client.http_client().get(&url).send().await else { - return None; - }; - let Ok(body) = resp.text().await else { - return None; - }; - let Ok(manifest) = serde_json::from_str::(&body) + // 10 s: the pooled peer client's default, which this probe used. + let Ok(manifest) = peers + .rpc + .sync_manifest(&addr, Duration::from_secs(10)) + .await else { return None; }; @@ -2932,6 +2911,7 @@ impl Drop for ClearOnDrop<'_> { #[cfg(test)] mod tests { + use crate::adapters::cluster::peer_client::PeerClient; /// Seconds since the epoch, for regions' `transfer_first_seen`. fn now_unix() -> i64 { std::time::SystemTime::now() @@ -4017,7 +3997,9 @@ mod tests { shard_map, bootstrap.membership.clone(), raft, - Some(peer_client), + Some(crate::adapters::cluster::http_peer_rpc::peers_for( + peer_client, + )), metadata, wal, None, @@ -5263,13 +5245,14 @@ mod tests { 8 * 1024 * 1024, Default::default(), ); + let peers = crate::adapters::cluster::http_peer_rpc::peers_for(Arc::new(pc)); assert_eq!( - pick_best_primary(&membership, &learner_only, 9, Some(&pc)).await, + pick_best_primary(&membership, &learner_only, 9, Some(&peers)).await, None, "the watermark path must exclude learners too" ); assert_eq!( - pick_best_primary(&membership, ®ion, 9, Some(&pc)).await, + pick_best_primary(&membership, ®ion, 9, Some(&peers)).await, Some(3), "and must still pick the committed peer" ); @@ -5665,6 +5648,13 @@ mod tests { format!("http://{addr}/internal/shard/rehome") } + /// `host:port` of a URL from `spawn_rehome_peer`. + fn rehome_addr(url: &str) -> String { + url.trim_start_matches("http://") + .trim_end_matches("/internal/shard/rehome") + .to_string() + } + fn sample_rehome_request() -> ShardRehomeRequest { ShardRehomeRequest { db: "db".into(), @@ -5689,18 +5679,24 @@ mod tests { /// remote copy of a scale-up ran past it and was restarted from nothing. #[tokio::test] async fn a_rehome_times_out_on_silence_not_on_total_time() { - let http = reqwest::Client::new(); + let http = crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ); let idle = Duration::from_millis(300); let req = sample_rehome_request(); // ~1 s in all, never silent for longer than 100 ms. let url = spawn_rehome_peer(10, Duration::from_millis(100), r#"{"ok":true,"bytes":7}"#).await; - let body = post_rehome(&http, &url, &req, idle).await.unwrap(); + let body = post_rehome(&http, &rehome_addr(&url), &req, idle) + .await + .unwrap(); assert_eq!(body["bytes"], 7); let url = spawn_rehome_peer(1, Duration::from_millis(800), r#"{"ok":true}"#).await; - let err = post_rehome(&http, &url, &req, idle).await.unwrap_err(); + let err = post_rehome(&http, &rehome_addr(&url), &req, idle) + .await + .unwrap_err(); assert!( matches!(err, HyperbytedbError::PeerUnreachable(ref m) if m.contains("silent")), "{err}" @@ -5717,8 +5713,10 @@ mod tests { ) .await; let err = post_rehome( - &reqwest::Client::new(), - &url, + &crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ), + &rehome_addr(&url), &sample_rehome_request(), Duration::from_secs(5), ) diff --git a/hyperbytedb/src/application/shard_transfer.rs b/hyperbytedb/src/application/shard_transfer.rs index d2ffb18..ff1fd04 100644 --- a/hyperbytedb/src/application/shard_transfer.rs +++ b/hyperbytedb/src/application/shard_transfer.rs @@ -603,7 +603,7 @@ const TRANSFER_ACK_TIMEOUT_SECS: u64 = 30; #[allow(clippy::too_many_arguments)] pub async fn push_region_transfer( - peer_client: &crate::adapters::cluster::peer_client::PeerClient, + peers: &crate::ports::peer_rpc::Peers, source_node_id: u64, key: &MeasurementKey, region: &ShardRegion, @@ -614,7 +614,7 @@ pub async fn push_region_transfer( done: bool, stage: bool, ) -> Result<(u64, u64), HyperbytedbError> { - let membership = peer_client.membership().read().await; + let membership = peers.membership.read().await; let addr = membership .get_node(dest_primary) .map(|n| n.addr.clone()) @@ -653,72 +653,40 @@ pub async fn push_region_transfer( done, ) }; - let resp = send_transfer_push(peer_client.http_client(), &addr, payload).await?; - if !resp.status().is_success() { - let status = resp.status().as_u16(); - return Err(HyperbytedbError::TransferRejected { - status, - reason: rejection_reason(resp).await, - }); - } - let applied = parse_applied_from_response(&resp.text().await.unwrap_or_default()); + use crate::domain::cluster::peer_rpc::PeerRpcError; + let text = match peers + .rpc + .shard_transfer_push( + &addr, + payload, + Duration::from_secs(TRANSFER_PUSH_TIMEOUT_SECS), + ) + .await + { + Ok(text) => text, + Err(PeerRpcError::Status { code, body }) => { + return Err(HyperbytedbError::TransferRejected { + status: code, + reason: rejection_reason(&body), + }); + } + Err(PeerRpcError::Unreachable(e)) => return Err(HyperbytedbError::PeerUnreachable(e)), + }; + let applied = parse_applied_from_response(&text); counter!("hyperbytedb_shard_transfer_bytes").increment(wire_bytes); counter!("hyperbytedb_shard_transfers_total").increment(1); Ok((applied, wire_bytes)) } -/// Send one push: the line protocol as the raw body, the rest of the payload -/// in a header. A node without the raw route (an older build) answers 404, -/// and gets the JSON push instead. -async fn send_transfer_push( - client: &reqwest::Client, - addr: &str, - mut payload: ShardTransferPayload, -) -> Result { - use base64::Engine as _; - let unreachable = |e: reqwest::Error| HyperbytedbError::PeerUnreachable(e.to_string()); - let timeout = Duration::from_secs(TRANSFER_PUSH_TIMEOUT_SECS); - // Bytes: the fallback below resends it without a copy. - let body = bytes::Bytes::from(payload.body.take().unwrap_or_default()); - let header = serde_json::to_vec(&payload) - .map(|json| base64::engine::general_purpose::STANDARD.encode(json)) - .map_err(|e| HyperbytedbError::Internal(format!("transfer header: {e}").into()))?; - let resp = client - .post(format!("http://{addr}/internal/shard/transfer/raw")) - .header( - crate::domain::sharding::transfer::TRANSFER_PAYLOAD_HEADER, - header, - ) - .header(reqwest::header::CONTENT_TYPE, "text/plain; charset=utf-8") - .body(body.clone()) - .timeout(timeout) - .send() - .await - .map_err(unreachable)?; - if resp.status() != reqwest::StatusCode::NOT_FOUND { - return Ok(resp); - } - // An older node, or a region the receiver does not know: the JSON route - // answers the second the same way. - payload.body = Some(body.to_vec()); - client - .post(format!("http://{addr}/internal/shard/transfer")) - .json(&payload) - .timeout(timeout) - .send() - .await - .map_err(unreachable) -} - pub async fn ack_region_transfer( - peer_client: &crate::adapters::cluster::peer_client::PeerClient, + peers: &crate::ports::peer_rpc::Peers, source_node_id: u64, key: &MeasurementKey, region: &ShardRegion, ack_node: u64, transfer_id: u64, ) -> Result<(), HyperbytedbError> { - let membership = peer_client.membership().read().await; + let membership = peers.membership.read().await; let addr = membership .get_node(ack_node) .map(|n| n.addr.clone()) @@ -736,21 +704,23 @@ pub async fn ack_region_transfer( source_node_id, transfer_id, ); - let url = format!("http://{addr}/internal/shard/transfer"); - let resp = peer_client - .http_client() - .post(&url) - .json(&payload) - .timeout(Duration::from_secs(TRANSFER_ACK_TIMEOUT_SECS)) - .send() + match peers + .rpc + .shard_transfer_ack( + &addr, + &payload, + Duration::from_secs(TRANSFER_ACK_TIMEOUT_SECS), + ) .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - if !resp.status().is_success() { - return Err(HyperbytedbError::ShardMap( - format!("transfer ack failed: {}", resp.status()).into(), - )); + { + Ok(()) => Ok(()), + Err(crate::domain::cluster::peer_rpc::PeerRpcError::Status { code, .. }) => Err( + HyperbytedbError::ShardMap(format!("transfer ack failed: {code}").into()), + ), + Err(crate::domain::cluster::peer_rpc::PeerRpcError::Unreachable(e)) => { + Err(HyperbytedbError::PeerUnreachable(e)) + } } - Ok(()) } fn new_transfer_id() -> u64 { @@ -808,7 +778,7 @@ fn pacing_delay(rate: u64, sent: u64, elapsed: std::time::Duration) -> std::time /// move only sees the whole remote stage, after it is done. #[allow(clippy::too_many_arguments)] async fn transfer_region_data( - peer_client: &Arc, + peers: &crate::ports::peer_rpc::Peers, metadata: &Arc, wal: &Arc, query_port: Option<&Arc>, @@ -862,7 +832,7 @@ async fn transfer_region_data( export_region_points_from_wal_chunked(wal, key, region, cursor, max_points, 0).await?; if !body.is_empty() { let pushed = push_region_transfer( - peer_client.as_ref(), + peers, source_node_id, key, region, @@ -932,7 +902,7 @@ async fn transfer_region_data( } let (body, count, chunk_end) = (chunk.body, chunk.points, chunk.next); let push = push_region_transfer( - peer_client.as_ref(), + peers, source_node_id, key, region, @@ -996,7 +966,7 @@ async fn transfer_region_data( /// applied. Ownership-aware: the destination validates the committed range. #[allow(clippy::too_many_arguments)] pub async fn push_region_transfer_data( - peer_client: &Arc, + peers: &crate::ports::peer_rpc::Peers, metadata: &Arc, wal: &Arc, query_port: Option<&Arc>, @@ -1007,7 +977,7 @@ pub async fn push_region_transfer_data( max_points: usize, ) -> Result { transfer_region_data( - peer_client, + peers, metadata, wal, query_port, @@ -1027,7 +997,7 @@ pub async fn push_region_transfer_data( /// Best-effort by design — callers log failures and proceed to commit. #[allow(clippy::too_many_arguments)] pub async fn stage_region_transfer_data( - peer_client: &Arc, + peers: &crate::ports::peer_rpc::Peers, metadata: &Arc, wal: &Arc, query_port: Option<&Arc>, @@ -1039,7 +1009,7 @@ pub async fn stage_region_transfer_data( max_bytes_per_sec: u64, ) -> Result { let outcome = transfer_region_data( - peer_client, + peers, metadata, wal, query_port, @@ -1072,8 +1042,8 @@ pub async fn stage_region_transfer_data( /// and every caller threw it away at the one place anyone would look. That is /// why a cluster could report 49 staging rejections across 27 regions without /// a single line saying what was actually wrong. -pub(crate) async fn rejection_reason(resp: reqwest::Response) -> Box { - let body = resp.text().await.unwrap_or_default(); +pub(crate) fn rejection_reason(body: &str) -> Box { + let body = body.to_owned(); let mut reason = serde_json::from_str::(&body) .ok() .and_then(|v| { @@ -1097,7 +1067,7 @@ pub(crate) async fn rejection_reason(resp: reqwest::Response) -> Box { #[allow(clippy::too_many_arguments)] pub async fn complete_region_transfer( - peer_client: &Arc, + peers: &crate::ports::peer_rpc::Peers, metadata: &Arc, sink: Option<&Arc>, source_node_id: u64, @@ -1111,7 +1081,7 @@ pub async fn complete_region_transfer( return Ok(()); } ack_region_transfer( - peer_client.as_ref(), + peers, source_node_id, key, region, @@ -1144,7 +1114,7 @@ pub async fn complete_region_transfer( /// dropping it strands data whenever the subsequent ownership change fails. #[allow(clippy::too_many_arguments)] pub async fn run_region_transfer( - peer_client: &Arc, + peers: &crate::ports::peer_rpc::Peers, metadata: &Arc, wal: &Arc, query_port: Option<&Arc>, @@ -1157,7 +1127,7 @@ pub async fn run_region_transfer( drop_source: bool, ) -> Result<(), HyperbytedbError> { let outcome = push_region_transfer_data( - peer_client, + peers, metadata, wal, query_port, @@ -1169,7 +1139,7 @@ pub async fn run_region_transfer( ) .await?; complete_region_transfer( - peer_client, + peers, metadata, sink, source_node_id, @@ -1196,7 +1166,7 @@ mod tests { .status(500) .body(r#"{"error":"region range not found"}"#) .unwrap(); - let got = super::rejection_reason(reqwest::Response::from(resp)).await; + let got = super::rejection_reason(resp.body()); assert_eq!(&*got, "region range not found"); } @@ -1206,7 +1176,7 @@ mod tests { .status(502) .body("upstream exploded") .unwrap(); - let got = super::rejection_reason(reqwest::Response::from(resp)).await; + let got = super::rejection_reason(resp.body()); assert_eq!(&*got, "upstream exploded"); } @@ -1218,7 +1188,7 @@ mod tests { .status(500) .body("") .unwrap(); - let got = super::rejection_reason(reqwest::Response::from(resp)).await; + let got = super::rejection_reason(resp.body()); assert_eq!(&*got, "no body"); } @@ -1231,7 +1201,7 @@ mod tests { .status(500) .body(body) .unwrap(); - let got = super::rejection_reason(reqwest::Response::from(resp)).await; + let got = super::rejection_reason(resp.body()); assert!(got.len() <= 300, "len {}", got.len()); assert!(!got.is_empty()); } @@ -1466,7 +1436,7 @@ mod tests { staging_peers: Vec::new(), }; let outcome = transfer_region_data( - &pc, + &crate::adapters::cluster::http_peer_rpc::peers_for(pc.clone()), &metadata, &wal, Some(&query), @@ -1716,7 +1686,7 @@ mod tests { staging_peers: Vec::new(), }; let outcome = transfer_region_data( - &pc, + &crate::adapters::cluster::http_peer_rpc::peers_for(pc.clone()), &metadata, &wal, Some(&query), diff --git a/hyperbytedb/src/application/sharded_mv_backfill.rs b/hyperbytedb/src/application/sharded_mv_backfill.rs index 11f2593..a724dba 100644 --- a/hyperbytedb/src/application/sharded_mv_backfill.rs +++ b/hyperbytedb/src/application/sharded_mv_backfill.rs @@ -178,31 +178,24 @@ async fn execute_region_backfill_once( let req = req.clone(); let addr = addr.to_string(); async move { - let url = format!("http://{addr}/internal/shard/mv-backfill"); - let resp = ctx - .peer_client - .http_client() - .post(&url) - .json(&req) - .timeout(timeout) - .send() - .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - if resp.status() == reqwest::StatusCode::CONFLICT { - return Err(HyperbytedbError::StaleShardEpoch { region_id }); + use crate::domain::cluster::peer_rpc::PeerRpcError; + match ctx.peer_rpc.shard_mv_backfill(&addr, &req, timeout).await { + Ok(()) => Ok(()), + Err(PeerRpcError::Status { code: 409, .. }) => { + Err(HyperbytedbError::StaleShardEpoch { region_id }) + } + Err(PeerRpcError::Status { code: 503, .. }) => { + Err(HyperbytedbError::ClusterUnavailable(format!( + "peer {peer_id} has not created the view yet" + ))) + } + Err(PeerRpcError::Status { code, .. }) => { + Err(HyperbytedbError::PeerUnreachable(format!( + "shard mv-backfill to peer {peer_id} failed: {code}" + ))) + } + Err(PeerRpcError::Unreachable(e)) => Err(HyperbytedbError::PeerUnreachable(e)), } - if resp.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE { - return Err(HyperbytedbError::ClusterUnavailable(format!( - "peer {peer_id} has not created the view yet" - ))); - } - if !resp.status().is_success() { - return Err(HyperbytedbError::PeerUnreachable(format!( - "shard mv-backfill to peer {peer_id} failed: {}", - resp.status() - ))); - } - Ok(()) } } }) diff --git a/hyperbytedb/src/ports/peer_rpc.rs b/hyperbytedb/src/ports/peer_rpc.rs index 87690f6..f8afa1f 100644 --- a/hyperbytedb/src/ports/peer_rpc.rs +++ b/hyperbytedb/src/ports/peer_rpc.rs @@ -7,10 +7,12 @@ use std::time::Duration; use bytes::Bytes; use futures::stream::BoxStream; +use crate::domain::cluster::membership::SharedMembership; use crate::domain::cluster::peer_rpc::{PeerRpcError, ShardWriteTarget}; +use crate::domain::sharding::transfer::{ShardRehomeRequest, ShardTransferPayload}; use crate::domain::sharding::types::{ - ShardBootstrapRequest, ShardDeleteRequest, ShardMetadataRequest, ShardQueryRequest, - ShardWriteBatchResponse, + MvHandoffRequest, RegionHeartbeat, ShardBootstrapRequest, ShardDeleteRequest, + ShardMetadataRequest, ShardMvBackfillRequest, ShardQueryRequest, ShardWriteBatchResponse, }; /// A peer's reply body as it arrives. @@ -66,6 +68,80 @@ pub trait PeerRpc: Send + Sync { req: &ShardDeleteRequest, timeout: Duration, ) -> Result<(), PeerRpcError>; + + /// A transfer push: the raw route first (`/internal/shard/transfer/raw`, + /// the line protocol as the body), and on 404 (an older peer) the JSON + /// route. Returns the reply text. + async fn shard_transfer_push( + &self, + addr: &str, + payload: ShardTransferPayload, + timeout: Duration, + ) -> Result; + + /// `POST /internal/shard/transfer` with an ack payload. + async fn shard_transfer_ack( + &self, + addr: &str, + payload: &ShardTransferPayload, + timeout: Duration, + ) -> Result<(), PeerRpcError>; + + /// `POST /internal/shard/mv-backfill`. + async fn shard_mv_backfill( + &self, + addr: &str, + req: &ShardMvBackfillRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError>; + + /// `POST /internal/mv/handoff`, to the departing node. + async fn mv_handoff( + &self, + addr: &str, + req: &MvHandoffRequest, + timeout: Duration, + ) -> Result<(), PeerRpcError>; + + /// `POST /internal/shard/heartbeat`, to the leader. + async fn shard_heartbeat( + &self, + addr: &str, + hb: &RegionHeartbeat, + timeout: Duration, + ) -> Result<(), PeerRpcError>; + + /// `POST /internal/shard/rehome` with progress streaming: waits as long + /// as the peer keeps sending (up to `max`), gives up after `idle` of + /// silence. Returns the final JSON body (which carries its own status). + async fn shard_rehome( + &self, + addr: &str, + req: &ShardRehomeRequest, + idle: Duration, + max: Duration, + ) -> Result; + + /// `GET /internal/sync/manifest`. + async fn sync_manifest( + &self, + addr: &str, + timeout: Duration, + ) -> Result; + + /// `GET /internal/shard/map`. + async fn shard_map( + &self, + addr: &str, + timeout: Duration, + ) -> Result; } pub type SharedPeerRpc = Arc; + +/// The port plus the membership its callers resolve node ids against. +#[derive(Clone)] +pub struct Peers { + pub rpc: SharedPeerRpc, + pub membership: SharedMembership, +} diff --git a/hyperbytedb/src/server.rs b/hyperbytedb/src/server.rs index ed81894..deac833 100644 --- a/hyperbytedb/src/server.rs +++ b/hyperbytedb/src/server.rs @@ -230,7 +230,9 @@ async fn serve_inner( map.clone(), membership.clone(), raft.clone(), - peer_client.clone(), + peer_client + .clone() + .map(crate::adapters::cluster::http_peer_rpc::peers_for), app_state.metadata.clone(), app_state.wal.clone(), Some(app_state.query_port.clone()), @@ -252,7 +254,7 @@ async fn serve_inner( ); } - if let (Some(sr), Some(pc), Some(rl)) = ( + if let (Some(sr), Some(_), Some(rl)) = ( app_state.shard_routing.as_ref(), peer_client.as_ref(), &app_state.replication_log, @@ -270,7 +272,6 @@ async fn serve_inner( ) .with_sharding( sr.clone(), - pc.clone(), app_state.metadata.clone(), app_state.points_sink.clone(), app_state.query_port.clone(), @@ -306,7 +307,9 @@ async fn serve_inner( if let Some(pc) = pc { shard_heartbeat::run_region_heartbeat_reporter( node_id, - pc, + Arc::new(crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::new( + pc, + )), m, map, Arc::new(raft), diff --git a/hyperbytedb/tests/common/sharding_cluster.rs b/hyperbytedb/tests/common/sharding_cluster.rs index 8a68e98..2fbeafe 100644 --- a/hyperbytedb/tests/common/sharding_cluster.rs +++ b/hyperbytedb/tests/common/sharding_cluster.rs @@ -276,7 +276,9 @@ pub async fn start_sharded_node( shard_map.clone(), shared_membership.clone(), raft, - Some(peer_client.clone()), + Some(hyperbytedb::adapters::cluster::http_peer_rpc::peers_for( + peer_client.clone(), + )), metadata.clone(), wal.clone(), Some(chdb_adapter.clone() as Arc), diff --git a/hyperbytedb/tests/sharding_cluster_integration.rs b/hyperbytedb/tests/sharding_cluster_integration.rs index 1ce5580..3902b6c 100644 --- a/hyperbytedb/tests/sharding_cluster_integration.rs +++ b/hyperbytedb/tests/sharding_cluster_integration.rs @@ -324,7 +324,7 @@ async fn transfer_moves_flushed_data_and_drops_source_range() { let query_port: Arc = nodes[0].query_port.clone(); run_region_transfer( - &peer_client, + &hyperbytedb::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()), &metadata, &wal, Some(&query_port), @@ -1532,7 +1532,7 @@ async fn joiner_receives_existing_region_as_replica() { let query_port: Arc = node1.query_port.clone(); let key = MeasurementKey::new("p1db", "autogen", "cpu"); let outcome = stage_region_transfer_data( - &peer_client, + &hyperbytedb::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()), &metadata, &wal, Some(&query_port), @@ -1696,7 +1696,7 @@ async fn joiner_takes_region_primary_and_applies_writes_locally() { let key = MeasurementKey::new("p14db", "autogen", "disk"); hyperbytedb::application::shard_transfer::stage_region_transfer_data( - &peer_client, + &hyperbytedb::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()), &metadata, &wal, Some(&query_port), @@ -1750,7 +1750,7 @@ async fn joiner_takes_region_primary_and_applies_writes_locally() { // Transfer-then-commit: rows move and verify before ownership changes. run_region_transfer( - &peer_client, + &hyperbytedb::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()), &metadata, &wal, Some(&query_port), @@ -1938,7 +1938,7 @@ async fn fourth_node_joining_rf_complete_cluster_takes_a_region() { // P1.3 shape: stage before the map commits the newcomer as a peer. let staged = stage_region_transfer_data( - &peer_client, + &hyperbytedb::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()), &metadata, &wal, Some(&query_port), @@ -1986,7 +1986,7 @@ async fn fourth_node_joining_rf_complete_cluster_takes_a_region() { ); run_region_transfer( - &peer_client, + &hyperbytedb::adapters::cluster::http_peer_rpc::peers_for(peer_client.clone()), &metadata, &wal, Some(&query_port), From ed2b940329d8925aba69f5c883a7e5df36b33bdd Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Tue, 10 Nov 2026 14:36:23 +0000 Subject: [PATCH 5/8] Cluster control calls go through PeerRpc --- .../cluster/hinted_handoff_watcher.rs} | 0 .../src/adapters/cluster/http_peer_rpc.rs | 89 ++++++++++++- hyperbytedb/src/adapters/cluster/mod.rs | 1 + .../src/application/cluster/heartbeat.rs | 126 +++++++----------- hyperbytedb/src/application/cluster/mod.rs | 1 - hyperbytedb/src/domain/cluster/peer_rpc.rs | 13 ++ hyperbytedb/src/ports/peer_rpc.rs | 8 ++ hyperbytedb/src/server.rs | 18 ++- 8 files changed, 173 insertions(+), 83 deletions(-) rename hyperbytedb/src/{application/cluster/hinted_handoff.rs => adapters/cluster/hinted_handoff_watcher.rs} (100%) diff --git a/hyperbytedb/src/application/cluster/hinted_handoff.rs b/hyperbytedb/src/adapters/cluster/hinted_handoff_watcher.rs similarity index 100% rename from hyperbytedb/src/application/cluster/hinted_handoff.rs rename to hyperbytedb/src/adapters/cluster/hinted_handoff_watcher.rs diff --git a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs index 91aa9ab..6454530 100644 --- a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs +++ b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs @@ -7,7 +7,7 @@ use std::time::Duration; use bytes::Bytes; use crate::adapters::cluster::peer_client::PeerClient; -use crate::domain::cluster::peer_rpc::{PeerRpcError, ShardWriteTarget}; +use crate::domain::cluster::peer_rpc::{PeerRpcError, ProbeReply, ShardWriteTarget}; use crate::domain::sharding::transfer::{ ShardRehomeRequest, ShardTransferPayload, TRANSFER_PAYLOAD_HEADER, }; @@ -358,6 +358,27 @@ impl PeerRpc for HttpPeerRpc { .map_err(|e| PeerRpcError::Unreachable(format!("parse shard map: {e}")))?; Ok(crate::domain::sharding::ShardMap::from(json)) } + + async fn probe(&self, addr: &str, timeout: Duration) -> ProbeReply { + let get = |path: &str| { + self.client() + .get(format!("http://{addr}{path}")) + .timeout(timeout) + .send() + }; + match get("/health").await { + Ok(resp) => ProbeReply::Health { + ok: resp.status().is_success(), + body: resp.text().await.unwrap_or_default(), + }, + // No answer from `/health`: a live `/ping` keeps a + // reachable-but-unhealthy peer from being taken for crashed. + Err(_) => match get("/ping").await { + Ok(_) => ProbeReply::AliveOnly, + Err(_) => ProbeReply::Unreachable, + }, + } + } } #[cfg(test)] @@ -381,6 +402,72 @@ mod tests { HttpPeerRpc::new(Arc::new(pc)) } + /// A 503 with a body is still a health answer (the body carries the + /// peer's self-reported state); a peer with no `/health` but a live + /// `/ping` is alive only; no listener at all is unreachable. + #[tokio::test] + async fn probe_reports_health_body() { + let serve = |app: axum::Router| async move { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + addr + }; + let health = serve(axum::Router::new().route( + "/health", + axum::routing::get(|| async { + ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + r#"{"state":"syncing"}"#, + ) + }), + )) + .await; + let ping_only = serve(axum::Router::new().route( + "/ping", + axum::routing::get(|| async { axum::http::StatusCode::NO_CONTENT }), + )) + .await; + let slow_health = serve( + axum::Router::new() + .route( + "/health", + axum::routing::get(|| async { + tokio::time::sleep(Duration::from_secs(3)).await; + "late" + }), + ) + .route( + "/ping", + axum::routing::get(|| async { axum::http::StatusCode::NO_CONTENT }), + ), + ) + .await; + + let rpc = HttpPeerRpc::with_client(reqwest::Client::new()); + assert_eq!( + rpc.probe(&slow_health, Duration::from_millis(500)).await, + ProbeReply::AliveOnly + ); + let t = Duration::from_secs(5); + assert_eq!( + rpc.probe(&health, t).await, + ProbeReply::Health { + ok: false, + body: r#"{"state":"syncing"}"#.into() + } + ); + // `/health` 404s here: an answer, so no `/ping` fallback. + assert!(matches!( + rpc.probe(&ping_only, t).await, + ProbeReply::Health { ok: false, .. } + )); + assert_eq!( + rpc.probe("127.0.0.1:1", Duration::from_millis(500)).await, + ProbeReply::Unreachable + ); + } + /// "Answered 503" (back-pressure, retried) and "unreachable" must stay /// distinct: callers retry one and not the other. #[tokio::test] diff --git a/hyperbytedb/src/adapters/cluster/mod.rs b/hyperbytedb/src/adapters/cluster/mod.rs index ac9f679..3ed0211 100644 --- a/hyperbytedb/src/adapters/cluster/mod.rs +++ b/hyperbytedb/src/adapters/cluster/mod.rs @@ -2,6 +2,7 @@ pub mod cluster_auth; pub mod dns_cache; pub mod forward_queue; pub mod hinted_handoff; +pub mod hinted_handoff_watcher; pub mod http_peer_rpc; pub mod peer_client; pub mod raft; diff --git a/hyperbytedb/src/application/cluster/heartbeat.rs b/hyperbytedb/src/application/cluster/heartbeat.rs index 48d0270..7d47f7d 100644 --- a/hyperbytedb/src/application/cluster/heartbeat.rs +++ b/hyperbytedb/src/application/cluster/heartbeat.rs @@ -1,8 +1,8 @@ use std::time::Duration; use crate::domain::cluster::membership::{NodeState, SharedMembership}; -use crate::ports::peer_http::PeerHttp; -use std::sync::Arc; +use crate::domain::cluster::peer_rpc::ProbeReply; +use crate::ports::peer_rpc::{PeerRpc, SharedPeerRpc}; /// Self-reported readiness fields parsed from a peer's `/health` body. /// @@ -192,17 +192,9 @@ pub async fn run_heartbeat_updater( interval: Duration, probe_timeout: Duration, sync_gates_readmit: bool, - peer_http: Arc, + peer_rpc: SharedPeerRpc, mut shutdown_rx: tokio::sync::watch::Receiver, ) { - let client = match peer_http.client_builder().timeout(probe_timeout).build() { - Ok(c) => c, - Err(e) => { - tracing::error!(error = %e, "failed to build heartbeat probe client"); - return; - } - }; - let mut ticker = tokio::time::interval(interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -215,7 +207,15 @@ pub async fn run_heartbeat_updater( loop { tokio::select! { _ = ticker.tick() => { - probe_peers(self_id, &membership, &client, miss_threshold, sync_gates_readmit).await; + probe_peers( + self_id, + &membership, + peer_rpc.as_ref(), + probe_timeout, + miss_threshold, + sync_gates_readmit, + ) + .await; } _ = async { while !*shutdown_rx.borrow() { @@ -232,7 +232,8 @@ pub async fn run_heartbeat_updater( async fn probe_peers( self_id: u64, membership: &SharedMembership, - client: &reqwest::Client, + rpc: &dyn PeerRpc, + probe_timeout: Duration, miss_threshold: u32, sync_gates_readmit: bool, ) { @@ -251,30 +252,18 @@ async fn probe_peers( let futures: Vec<_> = peers .iter() .map(|(peer_id, addr)| { - let health_url = format!("http://{addr}/health"); - let ping_url = format!("http://{addr}/ping"); - let client = client.clone(); let pid = *peer_id; async move { - let health = client.get(&health_url).send().await; - let signal = match health { - Ok(resp) => { - let health_ok = resp.status().is_success(); - let body = resp.text().await.ok().and_then(|t| HealthBody::parse(&t)); - ProbeSignal::Response { health_ok, body } - } - Err(_) => { - // Health endpoint unreachable entirely: fall back to - // liveness ping so reachable-but-unhealthy peers are - // not mistaken for crashed ones. - match client.get(&ping_url).send().await { - Ok(_) => ProbeSignal::Response { - health_ok: false, - body: None, - }, - Err(_) => ProbeSignal::Unreachable, - } - } + let signal = match rpc.probe(addr, probe_timeout).await { + ProbeReply::Health { ok, body } => ProbeSignal::Response { + health_ok: ok, + body: HealthBody::parse(&body), + }, + ProbeReply::AliveOnly => ProbeSignal::Response { + health_ok: false, + body: None, + }, + ProbeReply::Unreachable => ProbeSignal::Unreachable, }; (pid, signal) } @@ -351,6 +340,7 @@ mod tests { /// the demotion. Hysteresis itself is covered separately below. const THRESHOLD: u32 = 5; const MISSES_AT_THRESHOLD: u32 = 5; + use crate::adapters::cluster::http_peer_rpc::HttpPeerRpc; use crate::domain::cluster::membership::{ClusterMembership, NodeInfo, new_shared}; fn make_membership(peer_addrs: &[(u64, &str, NodeState)]) -> SharedMembership { @@ -753,11 +743,9 @@ mod tests { let membership = make_membership(&[(2, addr.as_str(), NodeState::Syncing)]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(500)) - .build() - .unwrap(); - probe_peers(1, &membership, &client, THRESHOLD, true).await; + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(500); + probe_peers(1, &membership, &client, probe_timeout, THRESHOLD, true).await; let m = membership.read().await; assert_eq!(m.get_node(2).unwrap().state, NodeState::Active); @@ -787,12 +775,10 @@ mod tests { (4, unsynced.as_str(), NodeState::Disconnected), (5, leaving.as_str(), NodeState::Disconnected), ]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(500)) - .build() - .unwrap(); + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(500); - probe_peers(1, &membership, &client, THRESHOLD, false).await; + probe_peers(1, &membership, &client, probe_timeout, THRESHOLD, false).await; let m = membership.read().await; assert_eq!(m.get_node(4).unwrap().state, NodeState::Active); @@ -811,11 +797,9 @@ mod tests { let membership = make_membership(&[(2, addr.as_str(), NodeState::Active)]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(500)) - .build() - .unwrap(); - probe_peers(1, &membership, &client, THRESHOLD, true).await; + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(500); + probe_peers(1, &membership, &client, probe_timeout, THRESHOLD, true).await; let m = membership.read().await; assert_eq!(m.get_node(2).unwrap().state, NodeState::Syncing); @@ -827,11 +811,9 @@ mod tests { let membership = make_membership(&[(2, addr.as_str(), NodeState::Syncing)]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(500)) - .build() - .unwrap(); - probe_peers(1, &membership, &client, THRESHOLD, true).await; + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(500); + probe_peers(1, &membership, &client, probe_timeout, THRESHOLD, true).await; let m = membership.read().await; assert_eq!( @@ -860,12 +842,10 @@ mod tests { let membership = make_membership(&[(2, addr.as_str(), NodeState::Syncing)]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(500)) - .build() - .unwrap(); + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(500); for _ in 0..6 { - probe_peers(1, &membership, &client, THRESHOLD, true).await; + probe_peers(1, &membership, &client, probe_timeout, THRESHOLD, true).await; } let m = membership.read().await; @@ -884,13 +864,11 @@ mod tests { // longer demotes, because a demotion now moves data. The old // single-probe assertion encoded the pre-hysteresis contract. let membership = make_membership(&[(1, "127.0.0.1:19999", NodeState::Active)]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(100)) - .build() - .unwrap(); + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(100); for probe in 1..THRESHOLD { - probe_peers(0, &membership, &client, THRESHOLD, true).await; + probe_peers(0, &membership, &client, probe_timeout, THRESHOLD, true).await; assert_eq!( membership.read().await.get_node(1).unwrap().state, NodeState::Active, @@ -898,7 +876,7 @@ mod tests { ); } - probe_peers(0, &membership, &client, THRESHOLD, true).await; + probe_peers(0, &membership, &client, probe_timeout, THRESHOLD, true).await; assert_eq!( membership.read().await.get_node(1).unwrap().state, NodeState::Disconnected, @@ -938,12 +916,10 @@ mod tests { async fn draining_peer_stays_draining_when_unreachable() { let membership = make_membership(&[(1, "127.0.0.1:19999", NodeState::Draining)]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(100)) - .build() - .unwrap(); + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(100); - probe_peers(0, &membership, &client, THRESHOLD, true).await; + probe_peers(0, &membership, &client, probe_timeout, THRESHOLD, true).await; let m = membership.read().await; assert_eq!(m.get_node(1).unwrap().state, NodeState::Draining); @@ -952,11 +928,9 @@ mod tests { #[tokio::test] async fn no_peers_is_noop() { let membership = make_membership(&[]); - let client = reqwest::Client::builder() - .timeout(Duration::from_millis(100)) - .build() - .unwrap(); + let client = HttpPeerRpc::with_client(reqwest::Client::new()); + let probe_timeout = Duration::from_millis(100); - probe_peers(0, &membership, &client, THRESHOLD, true).await; + probe_peers(0, &membership, &client, probe_timeout, THRESHOLD, true).await; } } diff --git a/hyperbytedb/src/application/cluster/mod.rs b/hyperbytedb/src/application/cluster/mod.rs index a132a4b..a8ddbe6 100644 --- a/hyperbytedb/src/application/cluster/mod.rs +++ b/hyperbytedb/src/application/cluster/mod.rs @@ -1,4 +1,3 @@ pub mod drain; pub mod heartbeat; -pub mod hinted_handoff; pub mod sync_manifest; diff --git a/hyperbytedb/src/domain/cluster/peer_rpc.rs b/hyperbytedb/src/domain/cluster/peer_rpc.rs index 1058ed1..d4e52e6 100644 --- a/hyperbytedb/src/domain/cluster/peer_rpc.rs +++ b/hyperbytedb/src/domain/cluster/peer_rpc.rs @@ -17,6 +17,19 @@ pub struct ShardWriteTarget { pub precision: Option, } +/// What one health probe of a peer got back. Unlike other calls, an error +/// status is an answer here: its body still carries the peer's +/// self-reported state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProbeReply { + /// `/health` answered; `ok` is whether its status was a success. + Health { ok: bool, body: String }, + /// `/health` did not answer but `/ping` did. + AliveOnly, + /// Neither answered. + Unreachable, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum PeerRpcError { /// The peer answered with a non-success status. diff --git a/hyperbytedb/src/ports/peer_rpc.rs b/hyperbytedb/src/ports/peer_rpc.rs index f8afa1f..11b6f28 100644 --- a/hyperbytedb/src/ports/peer_rpc.rs +++ b/hyperbytedb/src/ports/peer_rpc.rs @@ -135,6 +135,14 @@ pub trait PeerRpc: Send + Sync { addr: &str, timeout: Duration, ) -> Result; + + /// `GET /health`, falling back to `GET /ping` when `/health` gets no + /// answer; each request is bounded by `timeout`. + async fn probe( + &self, + addr: &str, + timeout: Duration, + ) -> crate::domain::cluster::peer_rpc::ProbeReply; } pub type SharedPeerRpc = Arc; diff --git a/hyperbytedb/src/server.rs b/hyperbytedb/src/server.rs index deac833..e0731a1 100644 --- a/hyperbytedb/src/server.rs +++ b/hyperbytedb/src/server.rs @@ -5,9 +5,9 @@ use std::sync::Arc; use std::time::Duration; +use crate::adapters::cluster::hinted_handoff_watcher; use crate::adapters::http::router::{build_control_router, build_router}; use crate::application::cluster::heartbeat; -use crate::application::cluster::hinted_handoff; use crate::application::continuous_query_service::ContinuousQueryService; use crate::application::disk_monitor; use crate::application::retention_service::RetentionService; @@ -123,8 +123,16 @@ async fn serve_inner( // Under sharding `needs_sync` is not a readiness signal; see // `decide_transition_with`. let sync_gates_readmit = !config.sharding.enabled; - let peer_http: Arc = - Arc::new(app_state.cluster_auth.clone()); + // Its own client, so probes never queue behind the write pool. + let probe_rpc: crate::ports::peer_rpc::SharedPeerRpc = { + use crate::ports::peer_http::PeerHttp; + let auth = &app_state.cluster_auth; + Arc::new( + crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + auth.build(auth.client_builder()), + ), + ) + }; Some(tokio::spawn(async move { heartbeat::run_heartbeat_updater( self_id, @@ -133,7 +141,7 @@ async fn serve_inner( hb_interval, probe_timeout, sync_gates_readmit, - peer_http, + probe_rpc, rx, ) .await; @@ -197,7 +205,7 @@ async fn serve_inner( let self_id = config.cluster.node_id; let rx = service_shutdown_rx.clone(); Some(tokio::spawn(async move { - hinted_handoff::run_hinted_handoff_watcher(self_id, pc, m, rx).await; + hinted_handoff_watcher::run_hinted_handoff_watcher(self_id, pc, m, rx).await; })) } else { None From 2f020285e4e47e1fe2e290c50b6c9f51e55aa434 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Tue, 10 Nov 2026 14:55:24 +0000 Subject: [PATCH 6/8] Raft proposals go through ConsensusPort --- .../src/adapters/cluster/http_peer_rpc.rs | 18 + .../src/adapters/cluster/raft/consensus.rs | 306 ++++++++++++++++ hyperbytedb/src/adapters/cluster/raft/mod.rs | 1 + hyperbytedb/src/application/cluster/drain.rs | 17 +- .../src/application/peer_query_service.rs | 327 ++---------------- hyperbytedb/src/application/query_service.rs | 2 +- hyperbytedb/src/application/shard_routing.rs | 30 +- .../src/application/shard_scheduler.rs | 46 ++- hyperbytedb/src/bootstrap.rs | 1 - hyperbytedb/src/ports/consensus.rs | 14 + hyperbytedb/src/ports/mod.rs | 1 + hyperbytedb/src/ports/peer_rpc.rs | 8 + hyperbytedb/src/server.rs | 42 ++- hyperbytedb/tests/common/sharding_cluster.rs | 19 +- hyperbytedb/tests/raft_integration.rs | 10 +- hyperbytedb/tests/security_auth.rs | 10 +- .../tests/sharding_failover_integration.rs | 12 +- hyperbytedb/tests/sync_quorum_integration.rs | 10 +- 18 files changed, 494 insertions(+), 380 deletions(-) create mode 100644 hyperbytedb/src/adapters/cluster/raft/consensus.rs create mode 100644 hyperbytedb/src/ports/consensus.rs diff --git a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs index 6454530..4f06dae 100644 --- a/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs +++ b/hyperbytedb/src/adapters/cluster/http_peer_rpc.rs @@ -359,6 +359,24 @@ impl PeerRpc for HttpPeerRpc { Ok(crate::domain::sharding::ShardMap::from(json)) } + async fn raft_client_write( + &self, + addr: &str, + req: &crate::domain::cluster::raft_request::ClusterRequest, + timeout: Duration, + ) -> Result { + let resp = self + .post_json(addr, "/cluster/raft/client-write", req, timeout) + .await?; + let body = Self::body(resp).await?; + serde_json::from_slice(&body).map_err(|e| { + PeerRpcError::Unreachable(format!( + "invalid leader response: {e}: {}", + String::from_utf8_lossy(&body) + )) + }) + } + async fn probe(&self, addr: &str, timeout: Duration) -> ProbeReply { let get = |path: &str| { self.client() diff --git a/hyperbytedb/src/adapters/cluster/raft/consensus.rs b/hyperbytedb/src/adapters/cluster/raft/consensus.rs new file mode 100644 index 0000000..14727e5 --- /dev/null +++ b/hyperbytedb/src/adapters/cluster/raft/consensus.rs @@ -0,0 +1,306 @@ +//! [`ConsensusPort`] over OpenRaft: propose locally, or forward to the +//! leader over [`PeerRpc`] when this node is a follower. + +use std::time::Duration; + +use openraft::BasicNode; +use openraft::error::ForwardToLeader; + +use super::HyperbytedbRaft; +use crate::domain::cluster::membership::SharedMembership; +use crate::domain::cluster::raft_request::{ClusterRequest, ClusterResponse}; +use crate::error::HyperbytedbError; +use crate::ports::consensus::ConsensusPort; +use crate::ports::leadership::LeadershipPort; +use crate::ports::peer_rpc::{PeerRpc, SharedPeerRpc}; + +/// Proposals forwarded to a leader that cannot be reached are retried this +/// many times in all, re-resolving the leader each time. +const MAX_ATTEMPTS: u32 = 3; +/// A forwarded proposal's deadline (the old dedicated client's timeout). +const FORWARD_TIMEOUT: Duration = Duration::from_secs(10); + +pub struct RaftConsensus { + raft: HyperbytedbRaft, + rpc: SharedPeerRpc, + /// Addresses for a leader openraft names by id only. + membership: Option, +} + +impl RaftConsensus { + #[must_use] + pub fn new( + raft: HyperbytedbRaft, + rpc: SharedPeerRpc, + membership: Option, + ) -> Self { + Self { + raft, + rpc, + membership, + } + } + + async fn leader_addr( + &self, + forward: &ForwardToLeader, + ) -> Result { + let metrics = self.raft.metrics().borrow().clone(); + let membership_addrs = match &self.membership { + Some(m) => m + .read() + .await + .nodes + .values() + .map(|node| (node.node_id, node.addr.clone())) + .collect::>(), + None => Vec::new(), + }; + resolve_leader_addr_with_lookup( + forward, + metrics.current_leader, + |leader_id| { + metrics + .membership_config + .membership() + .get_node(&leader_id) + .map(|node| node.addr.clone()) + }, + |leader_id| { + membership_addrs + .iter() + .find(|(id, _)| *id == leader_id) + .map(|(_, addr)| addr.clone()) + }, + ) + } +} + +impl LeadershipPort for RaftConsensus { + fn current_leader(&self) -> Option { + self.raft.metrics().borrow().current_leader + } +} + +#[async_trait::async_trait] +impl ConsensusPort for RaftConsensus { + async fn propose(&self, req: ClusterRequest) -> Result { + for attempt in 0..MAX_ATTEMPTS { + let e = match self.raft.client_write(req.clone()).await { + Ok(resp) => return Ok(resp.data), + Err(e) => e, + }; + let Some(fwd) = e.forward_to_leader::() else { + tracing::error!(error = %e, "raft proposal failed"); + return Err(HyperbytedbError::Internal( + format!("raft replication failed: {e}").into(), + )); + }; + tracing::debug!( + leader_id = ?fwd.leader_id, + leader_addr = ?fwd.leader_node.as_ref().map(|n| n.addr.as_str()), + attempt, + "forwarding proposal to raft leader" + ); + let result = match self.leader_addr(fwd).await { + Ok(addr) => forward(self.rpc.as_ref(), &addr, &req).await, + Err(e) => Err(e), + }; + match result { + Ok(resp) => return Ok(resp), + Err(e) if attempt + 1 >= MAX_ATTEMPTS => return Err(e), + Err(e) => { + tracing::warn!(error = %e, attempt, "forward to raft leader failed; retrying"); + } + } + } + Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "raft replication failed after retries", + ))) + } +} + +/// Send `req` to the leader at `leader_addr` and return its answer. +pub(crate) async fn forward( + rpc: &dyn PeerRpc, + leader_addr: &str, + req: &ClusterRequest, +) -> Result { + rpc.raft_client_write(leader_addr, req, FORWARD_TIMEOUT) + .await + .map_err(|e| { + HyperbytedbError::Internal( + format!("raft replication failed: forward to leader {leader_addr}: {e}").into(), + ) + }) +} + +fn resolve_leader_addr_with_lookup( + forward: &ForwardToLeader, + fallback_leader_id: Option, + raft_lookup: impl Fn(u64) -> Option, + membership_lookup: impl Fn(u64) -> Option, +) -> Result { + if let Some(node) = forward.leader_node.as_ref() { + return Ok(node.addr.clone()); + } + + let leader_id = forward.leader_id.or(fallback_leader_id); + let leader_id = leader_id.ok_or_else(|| { + HyperbytedbError::Internal(crate::error::ChainedError::new( + "raft replication failed: forward to leader but leader id is unknown", + )) + })?; + + raft_lookup(leader_id) + .or_else(|| membership_lookup(leader_id)) + .ok_or_else(|| { + HyperbytedbError::Internal( + format!("raft replication failed: leader {leader_id} has no known address").into(), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::adapters::cluster::http_peer_rpc::HttpPeerRpc; + use crate::domain::sharding::ops::ShardMapOp; + use std::sync::{Arc, Mutex}; + + async fn stub_leader( + status: axum::http::StatusCode, + reply: ClusterResponse, + ) -> (String, Arc>>) { + let seen = Arc::new(Mutex::new(Vec::new())); + let rec = seen.clone(); + let app = axum::Router::new().route( + "/cluster/raft/client-write", + axum::routing::post(move |axum::Json(v): axum::Json| { + let rec = rec.clone(); + let reply = reply.clone(); + async move { + rec.lock().unwrap().push(v); + (status, axum::Json(reply)) + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (addr, seen) + } + + fn op() -> ClusterRequest { + ClusterRequest::ShardMapMutation(Box::new(ShardMapOp::DropSpace { + db: "db".into(), + rp: None, + measurement: None, + })) + } + + /// The follower's request reaches the leader unchanged, and the + /// leader's answer comes back as-is: a refusal is a response the caller + /// checks, not an error; an error status is an error. + #[tokio::test] + async fn propose_follows_forward_to_leader() { + let refused = ClusterResponse { + ok: false, + message: Some("stale epoch".into()), + }; + let (addr, seen) = stub_leader(axum::http::StatusCode::OK, refused.clone()).await; + let rpc = HttpPeerRpc::with_client(reqwest::Client::new()); + + let resp = forward(&rpc, &addr, &op()).await.unwrap(); + assert!(!resp.ok); + assert_eq!(resp.message.as_deref(), Some("stale epoch")); + assert_eq!( + seen.lock().unwrap().as_slice(), + [serde_json::to_value(op()).unwrap()] + ); + + let (down, _) = stub_leader(axum::http::StatusCode::SERVICE_UNAVAILABLE, refused).await; + let err = forward(&rpc, &down, &op()).await.unwrap_err().to_string(); + assert!(err.contains("503"), "{err}"); + } + + #[test] + fn resolve_leader_addr_prefers_forward_node() { + let forward = ForwardToLeader { + leader_id: Some(1), + leader_node: Some(BasicNode::new("hyperbytedb-0:8086")), + }; + + let addr = resolve_leader_addr_with_lookup(&forward, None, |_| None, |_| None) + .expect("leader addr"); + + assert_eq!(addr, "hyperbytedb-0:8086"); + } + + #[test] + fn resolve_leader_addr_falls_back_to_raft_membership() { + let forward = ForwardToLeader { + leader_id: Some(1), + leader_node: None, + }; + + let addr = resolve_leader_addr_with_lookup( + &forward, + None, + |id| (id == 1).then(|| "hyperbytedb-0:8086".into()), + |_| None, + ) + .expect("leader addr"); + + assert_eq!(addr, "hyperbytedb-0:8086"); + } + + #[test] + fn resolve_leader_addr_falls_back_to_cluster_membership() { + let forward = ForwardToLeader { + leader_id: Some(2), + leader_node: None, + }; + + let addr = resolve_leader_addr_with_lookup( + &forward, + None, + |_| None, + |id| (id == 2).then(|| "hyperbytedb-1:8086".into()), + ) + .expect("leader addr"); + + assert_eq!(addr, "hyperbytedb-1:8086"); + } + + #[test] + fn resolve_leader_addr_uses_metrics_leader_when_forward_id_missing() { + let forward = ForwardToLeader { + leader_id: None, + leader_node: None, + }; + + let addr = resolve_leader_addr_with_lookup( + &forward, + Some(1), + |id| (id == 1).then(|| "hyperbytedb-0:8086".into()), + |_| None, + ) + .expect("leader addr"); + + assert_eq!(addr, "hyperbytedb-0:8086"); + } + + #[test] + fn resolve_leader_addr_errors_when_unresolvable() { + let forward = ForwardToLeader { + leader_id: Some(99), + leader_node: None, + }; + + let err = resolve_leader_addr_with_lookup(&forward, None, |_| None, |_| None) + .expect_err("expected error"); + + assert!(err.to_string().contains("leader 99 has no known address")); + } +} diff --git a/hyperbytedb/src/adapters/cluster/raft/mod.rs b/hyperbytedb/src/adapters/cluster/raft/mod.rs index 3c88f8e..7e592d0 100644 --- a/hyperbytedb/src/adapters/cluster/raft/mod.rs +++ b/hyperbytedb/src/adapters/cluster/raft/mod.rs @@ -1,3 +1,4 @@ +pub mod consensus; pub mod log_store; pub mod network; pub mod state_machine; diff --git a/hyperbytedb/src/application/cluster/drain.rs b/hyperbytedb/src/application/cluster/drain.rs index c537391..ebe73be 100644 --- a/hyperbytedb/src/application/cluster/drain.rs +++ b/hyperbytedb/src/application/cluster/drain.rs @@ -3,12 +3,12 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use crate::adapters::cluster::raft::HyperbytedbRaft; use crate::application::shard_routing::ShardRoutingContext; use crate::application::shard_transfer::run_region_transfer; use crate::domain::cluster::membership::{NodeState, SharedMembership}; use crate::domain::sharding::{MeasurementKey, ShardMapOp}; use crate::error::HyperbytedbError; +use crate::ports::consensus::ConsensusPort; use crate::ports::flush::FlushPort; use crate::ports::metadata::MetadataPort; use crate::ports::points_sink::PointsSinkPort; @@ -31,7 +31,7 @@ pub struct DrainService { metadata: Option>, points_sink: Option>, query_port: Option>, - raft: Option, + consensus: Option>, max_points_per_request: usize, /// Set once [`DrainService::drain`] has run to completion (or determined /// there was nothing to do because the node was already in a terminal @@ -61,7 +61,7 @@ impl DrainService { metadata: None, points_sink: None, query_port: None, - raft: None, + consensus: None, max_points_per_request: 0, drained: AtomicBool::new(false), } @@ -74,14 +74,14 @@ impl DrainService { metadata: Arc, points_sink: Arc, query_port: Arc, - raft: HyperbytedbRaft, + consensus: Arc, max_points_per_request: usize, ) -> Self { self.shard_routing = Some(ctx); self.metadata = Some(metadata); self.points_sink = Some(points_sink); self.query_port = Some(query_port); - self.raft = Some(raft); + self.consensus = Some(consensus); self.max_points_per_request = max_points_per_request; self } @@ -197,8 +197,8 @@ impl DrainService { .metadata .as_ref() .ok_or_else(|| HyperbytedbError::ClusterUnavailable("no metadata".into()))?; - let raft = self - .raft + let consensus = self + .consensus .as_ref() .ok_or_else(|| HyperbytedbError::ClusterUnavailable("no raft".into()))?; @@ -240,7 +240,8 @@ impl DrainService { new_primary, epoch: region.epoch, }; - raft.client_write(ClusterRequest::ShardMapMutation(Box::new(tp))) + consensus + .propose(ClusterRequest::ShardMapMutation(Box::new(tp))) .await .map_err(|e| HyperbytedbError::ShardMap(e.to_string().into()))?; } diff --git a/hyperbytedb/src/application/peer_query_service.rs b/hyperbytedb/src/application/peer_query_service.rs index 179d804..545c7d5 100644 --- a/hyperbytedb/src/application/peer_query_service.rs +++ b/hyperbytedb/src/application/peer_query_service.rs @@ -1,26 +1,21 @@ use async_trait::async_trait; use std::sync::Arc; -use openraft::BasicNode; -use openraft::error::ForwardToLeader; - -use crate::adapters::cluster::raft::HyperbytedbRaft; use crate::application::materialized_view_service::def_from_statement; use crate::application::predicate_sql::build_predicate_sql; use crate::application::query_service::check_authorization; use crate::application::shard_metadata_scatter::scatter_measurement_meta; use crate::application::shard_routing::ShardRoutingContext; -use crate::domain::cluster::membership::ClusterMembership; -use crate::domain::cluster::raft_request::{ClusterRequest, ClusterResponse}; +use crate::domain::cluster::raft_request::ClusterRequest; use crate::domain::cluster::types::MutationRequest; use crate::domain::database::{RetentionPolicy, retention_policy_from_create}; use crate::domain::materialized_view::MaterializedViewDef; use crate::domain::measurement::MeasurementMeta; use crate::domain::query_result::{QueryResponse, StatementResult}; use crate::error::HyperbytedbError; +use crate::ports::consensus::ConsensusPort; use crate::ports::metadata::ContinuousQueryDef; use crate::ports::metadata::MetadataPort; -use crate::ports::peer_http::PeerHttp; use crate::ports::query::QueryService; use crate::ports::replication::ReplicationPort; use crate::timeseriesql::ast::{ @@ -37,11 +32,8 @@ pub struct PeerQueryService { inner: Arc, metadata: Arc, replication_port: Arc, - raft: std::sync::OnceLock, - membership: std::sync::OnceLock>>, + consensus: std::sync::OnceLock>, shard_routing: std::sync::OnceLock>, - /// For schema mutations forwarded to the Raft leader. - raft_client: reqwest::Client, } impl PeerQueryService { @@ -49,19 +41,12 @@ impl PeerQueryService { inner: Arc, metadata: Arc, replication_port: Arc, - peer_http: &dyn PeerHttp, ) -> Self { Self { - raft_client: peer_http.build( - peer_http - .client_builder() - .timeout(std::time::Duration::from_secs(10)), - ), inner, metadata, replication_port, - raft: std::sync::OnceLock::new(), - membership: std::sync::OnceLock::new(), + consensus: std::sync::OnceLock::new(), shard_routing: std::sync::OnceLock::new(), } } @@ -72,28 +57,28 @@ impl PeerQueryService { let _ = self.shard_routing.set(ctx); } - /// Set the Raft instance for consensus-based schema mutation replication. - /// Called after Raft initialization completes. - pub fn set_raft(&self, raft: HyperbytedbRaft) { - if self.raft.set(raft).is_err() { - tracing::error!("PeerQueryService::set_raft called more than once; ignoring duplicate"); - } - } - - /// Shared cluster membership for resolving leader addresses during Raft - /// forward when openraft omits `leader_node` from `ForwardToLeader`. - pub fn set_membership(&self, membership: Arc>) { - if self.membership.set(membership).is_err() { + /// Route schema mutations through consensus. Called after Raft + /// initialization completes. + pub fn set_consensus(&self, consensus: Arc) { + if self.consensus.set(consensus).is_err() { tracing::error!( - "PeerQueryService::set_membership called more than once; ignoring duplicate" + "PeerQueryService::set_consensus called more than once; ignoring duplicate" ); } } async fn replicate_mutation(&self, req: MutationRequest) -> Result<(), HyperbytedbError> { - if let Some(raft) = self.raft.get() { - let cluster_req = ClusterRequest::SchemaMutation(Box::new(req)); - self.client_write_with_forward(raft, cluster_req).await + if let Some(consensus) = self.consensus.get() { + let resp = consensus + .propose(ClusterRequest::SchemaMutation(Box::new(req))) + .await?; + if resp.ok { + return Ok(()); + } + Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + resp.message + .unwrap_or_else(|| "raft schema mutation apply failed".to_string()), + ))) } else { self.replication_port .clone() @@ -119,70 +104,6 @@ impl PeerQueryService { ) } - async fn client_write_with_forward( - &self, - raft: &HyperbytedbRaft, - req: ClusterRequest, - ) -> Result<(), HyperbytedbError> { - const MAX_ATTEMPTS: u32 = 3; - for attempt in 0..MAX_ATTEMPTS { - match raft.client_write(req.clone()).await { - Ok(resp) => { - if resp.data.ok { - return Ok(()); - } - return Err(HyperbytedbError::Internal(crate::error::ChainedError::new( - resp.data - .message - .unwrap_or_else(|| "raft schema mutation apply failed".to_string()), - ))); - } - Err(e) => { - if let Some(forward) = e.forward_to_leader::() { - tracing::debug!( - leader_id = ?forward.leader_id, - leader_addr = ?forward - .leader_node - .as_ref() - .map(|node| node.addr.as_str()), - attempt, - "forwarding schema mutation to raft leader" - ); - match forward_client_write( - &self.raft_client, - forward, - raft, - self.membership.get(), - &req, - ) - .await - { - Ok(()) => return Ok(()), - Err(forward_err) => { - tracing::warn!( - error = %forward_err, - attempt, - "forward to raft leader failed; retrying" - ); - if attempt + 1 >= MAX_ATTEMPTS { - return Err(forward_err); - } - continue; - } - } - } - tracing::error!(error = %e, "failed to replicate mutation via raft"); - return Err(HyperbytedbError::Internal( - format!("raft replication failed: {e}").into(), - )); - } - } - } - Err(HyperbytedbError::Internal(crate::error::ChainedError::new( - "raft replication failed after retries", - ))) - } - async fn execute_raft_mutation( &self, db: &str, @@ -226,113 +147,6 @@ impl PeerQueryService { } } -async fn resolve_forward_leader_addr( - forward: &ForwardToLeader, - raft: &HyperbytedbRaft, - membership: Option<&Arc>>, -) -> Result { - let metrics = raft.metrics().borrow().clone(); - let membership_addrs = if let Some(membership) = membership { - let m = membership.read().await; - m.nodes - .values() - .map(|node| (node.node_id, node.addr.clone())) - .collect::>() - } else { - Vec::new() - }; - - resolve_leader_addr_with_lookup( - forward, - metrics.current_leader, - |leader_id| { - metrics - .membership_config - .membership() - .get_node(&leader_id) - .map(|node| node.addr.clone()) - }, - |leader_id| { - membership_addrs - .iter() - .find(|(id, _)| *id == leader_id) - .map(|(_, addr)| addr.clone()) - }, - ) -} - -fn resolve_leader_addr_with_lookup( - forward: &ForwardToLeader, - fallback_leader_id: Option, - raft_lookup: impl Fn(u64) -> Option, - membership_lookup: impl Fn(u64) -> Option, -) -> Result { - if let Some(node) = forward.leader_node.as_ref() { - return Ok(node.addr.clone()); - } - - let leader_id = forward.leader_id.or(fallback_leader_id); - let leader_id = leader_id.ok_or_else(|| { - HyperbytedbError::Internal(crate::error::ChainedError::new( - "raft replication failed: forward to leader but leader id is unknown", - )) - })?; - - raft_lookup(leader_id) - .or_else(|| membership_lookup(leader_id)) - .ok_or_else(|| { - HyperbytedbError::Internal( - format!("raft replication failed: leader {leader_id} has no known address").into(), - ) - }) -} - -async fn forward_client_write( - client: &reqwest::Client, - forward: &ForwardToLeader, - raft: &HyperbytedbRaft, - membership: Option<&Arc>>, - req: &ClusterRequest, -) -> Result<(), HyperbytedbError> { - let leader_addr = resolve_forward_leader_addr(forward, raft, membership).await?; - - let url = format!("http://{leader_addr}/cluster/raft/client-write"); - let resp = client.post(&url).json(req).send().await.map_err(|e| { - HyperbytedbError::Internal( - format!("raft replication failed: forward to leader {url}: {e}").into(), - ) - })?; - - let status = resp.status(); - let body_text = resp.text().await.map_err(|e| { - HyperbytedbError::Internal( - format!("raft replication failed: read leader response: {e}").into(), - ) - })?; - - if !status.is_success() { - return Err(HyperbytedbError::Internal( - format!("raft replication failed: leader returned HTTP {status}: {body_text}").into(), - )); - } - - let body: ClusterResponse = serde_json::from_str(&body_text).map_err(|e| { - HyperbytedbError::Internal( - format!("raft replication failed: invalid leader response: {e}: {body_text}").into(), - ) - })?; - - if body.ok { - Ok(()) - } else { - Err(HyperbytedbError::Internal( - body.message - .unwrap_or_else(|| "raft leader rejected schema mutation".into()) - .into(), - )) - } -} - fn extract_mv_source_rp(mv: &CreateMaterializedViewStatement) -> Option { mv.query.from.first().and_then(|s| match s { MeasurementSource::Concrete(m) => m.retention_policy.clone(), @@ -667,7 +481,7 @@ impl QueryService for PeerQueryService { } let mut results = Vec::new(); - let use_raft = self.raft.get().is_some(); + let use_raft = self.consensus.get().is_some(); for (i, stmt) in stmts.into_iter().enumerate() { let statement_id = i as u32; let stmt_query = stmt_queries.get(i).map(String::as_str).unwrap_or(query); @@ -1407,13 +1221,18 @@ mod tests { 30, sink, )); - let peer = PeerQueryService::new( - base.clone(), - metadata.clone(), - Arc::new(NoReplication), - &crate::adapters::cluster::cluster_auth::ClusterAuth::new(None).unwrap(), - ); - peer.set_raft(raft); + let peer = PeerQueryService::new(base.clone(), metadata.clone(), Arc::new(NoReplication)); + peer.set_consensus(Arc::new( + crate::adapters::cluster::raft::consensus::RaftConsensus::new( + raft, + Arc::new( + crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ), + ), + None, + ), + )); let resp = peer .execute_query( @@ -1448,84 +1267,4 @@ mod tests { assert_eq!(rows("week").await, 0, "the DELETE named ?rp=week"); assert_eq!(rows("autogen").await, 1, "autogen must be untouched"); } - - #[test] - fn resolve_leader_addr_prefers_forward_node() { - let forward = ForwardToLeader { - leader_id: Some(1), - leader_node: Some(BasicNode::new("hyperbytedb-0:8086")), - }; - - let addr = resolve_leader_addr_with_lookup(&forward, None, |_| None, |_| None) - .expect("leader addr"); - - assert_eq!(addr, "hyperbytedb-0:8086"); - } - - #[test] - fn resolve_leader_addr_falls_back_to_raft_membership() { - let forward = ForwardToLeader { - leader_id: Some(1), - leader_node: None, - }; - - let addr = resolve_leader_addr_with_lookup( - &forward, - None, - |id| (id == 1).then(|| "hyperbytedb-0:8086".into()), - |_| None, - ) - .expect("leader addr"); - - assert_eq!(addr, "hyperbytedb-0:8086"); - } - - #[test] - fn resolve_leader_addr_falls_back_to_cluster_membership() { - let forward = ForwardToLeader { - leader_id: Some(2), - leader_node: None, - }; - - let addr = resolve_leader_addr_with_lookup( - &forward, - None, - |_| None, - |id| (id == 2).then(|| "hyperbytedb-1:8086".into()), - ) - .expect("leader addr"); - - assert_eq!(addr, "hyperbytedb-1:8086"); - } - - #[test] - fn resolve_leader_addr_uses_metrics_leader_when_forward_id_missing() { - let forward = ForwardToLeader { - leader_id: None, - leader_node: None, - }; - - let addr = resolve_leader_addr_with_lookup( - &forward, - Some(1), - |id| (id == 1).then(|| "hyperbytedb-0:8086".into()), - |_| None, - ) - .expect("leader addr"); - - assert_eq!(addr, "hyperbytedb-0:8086"); - } - - #[test] - fn resolve_leader_addr_errors_when_unresolvable() { - let forward = ForwardToLeader { - leader_id: Some(99), - leader_node: None, - }; - - let err = resolve_leader_addr_with_lookup(&forward, None, |_| None, |_| None) - .expect_err("expected error"); - - assert!(err.to_string().contains("leader 99 has no known address")); - } } diff --git a/hyperbytedb/src/application/query_service.rs b/hyperbytedb/src/application/query_service.rs index 11047d8..f752cf3 100644 --- a/hyperbytedb/src/application/query_service.rs +++ b/hyperbytedb/src/application/query_service.rs @@ -171,7 +171,7 @@ impl QueryServiceImpl { // Raft-less sharding: this node's map is the placement authority. (_, None) => ctx.shard_map.apply_op(op).await.map(|_| ()), (_, Some(addr)) => crate::application::shard_routing::propose_shard_map_op_via_raft( - ctx.peer_client.http_client(), + ctx.peer_rpc.as_ref(), &addr, op, ) diff --git a/hyperbytedb/src/application/shard_routing.rs b/hyperbytedb/src/application/shard_routing.rs index 655a78b..43b5bd7 100644 --- a/hyperbytedb/src/application/shard_routing.rs +++ b/hyperbytedb/src/application/shard_routing.rs @@ -197,7 +197,7 @@ pub async fn ensure_measurement_bootstrapped( if is_raft_leader { let op = build_bootstrap_op(ctx, db, rp, measurement).await?; if let Some(addr) = leader_addr { - propose_shard_map_op_via_raft(ctx.peer_client.http_client(), addr, op).await?; + propose_shard_map_op_via_raft(ctx.peer_rpc.as_ref(), addr, op).await?; } else { // No leader address resolves: this is raft-less sharding (async // replication without OpenRaft), where this node's shard map IS @@ -947,30 +947,22 @@ pub async fn region_replication_targets( active_region_peer_targets(region, ctx.node_id, &membership) } -/// Propose `op` to the Raft leader at `leader_addr`, over this node's pooled -/// peer `client` (it carries the cluster credential). +/// Propose `op` to the Raft leader at `leader_addr`. pub async fn propose_shard_map_op_via_raft( - client: &reqwest::Client, + rpc: &dyn crate::ports::peer_rpc::PeerRpc, leader_addr: &str, op: ShardMapOp, ) -> Result { - let url = format!("http://{leader_addr}/cluster/raft/client-write"); + use crate::domain::cluster::peer_rpc::PeerRpcError; let req = ClusterRequest::ShardMapMutation(Box::new(op)); - let resp = client - .post(&url) - .json(&req) - .timeout(Duration::from_secs(5)) - .send() + rpc.raft_client_write(leader_addr, &req, Duration::from_secs(5)) .await - .map_err(|e| HyperbytedbError::PeerUnreachable(e.to_string()))?; - if !resp.status().is_success() { - return Err(HyperbytedbError::ShardMap( - format!("raft client_write failed: {}", resp.status()).into(), - )); - } - resp.json() - .await - .map_err(|e| HyperbytedbError::ShardMap(e.to_string().into())) + .map_err(|e| match e { + PeerRpcError::Status { code, .. } => { + HyperbytedbError::ShardMap(format!("raft client_write failed: {code}").into()) + } + PeerRpcError::Unreachable(e) => HyperbytedbError::PeerUnreachable(e), + }) } /// Per-region outcome counts for one sharded delete fan-out. diff --git a/hyperbytedb/src/application/shard_scheduler.rs b/hyperbytedb/src/application/shard_scheduler.rs index 24d4890..63d40fb 100644 --- a/hyperbytedb/src/application/shard_scheduler.rs +++ b/hyperbytedb/src/application/shard_scheduler.rs @@ -5,7 +5,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use metrics::counter; use tokio::sync::watch; -use crate::adapters::cluster::raft::HyperbytedbRaft; use crate::application::runtime_settings as rs; use crate::application::shard_peer_resolution::is_active_peer; use crate::application::shard_routing::effective_replication_factor; @@ -22,6 +21,7 @@ use crate::domain::sharding::{ MeasurementKey, ShardMap, ShardMapOp, ShardRegion, ShardRehomeRequest, }; use crate::error::HyperbytedbError; +use crate::ports::consensus::ConsensusPort; use crate::ports::metadata::MetadataPort; use crate::ports::peer_rpc::{PeerRpc, Peers}; use crate::ports::query::QueryPort; @@ -253,7 +253,7 @@ pub struct ShardScheduler { runtime_settings: Arc, shard_map: Arc, membership: SharedMembership, - raft: HyperbytedbRaft, + consensus: Arc, peers: Option, metadata: Arc, wal: Arc, @@ -346,7 +346,7 @@ impl ShardScheduler { pub fn new( shard_map: Arc, membership: SharedMembership, - raft: HyperbytedbRaft, + consensus: Arc, peers: Option, metadata: Arc, wal: Arc, @@ -363,7 +363,7 @@ impl ShardScheduler { runtime_settings, shard_map, membership, - raft, + consensus, peers, metadata, wal, @@ -854,8 +854,7 @@ impl ShardScheduler { if self.test_force_leader { return true; } - let metrics = self.raft.metrics().borrow().clone(); - metrics.current_leader == Some(self.node_id) + self.consensus.is_leader(self.node_id) } /// Tick on `interval` while leader. Placement moves run in a background @@ -2495,17 +2494,16 @@ impl ShardScheduler { use crate::domain::cluster::raft_request::ClusterRequest; let req = ClusterRequest::ShardMapMutation(Box::new(op)); let resp = self - .raft - .client_write(req) + .consensus + .propose(req) .await .map_err(|e| HyperbytedbError::ShardMap(e.to_string().into()))?; // Committed is not applied: the apply can refuse the op (a stale // epoch, say), and that was read as success. A refused Merge was // counted and retried every tick for 20 hours on n3, 72,541 times. - if !resp.data.ok { + if !resp.ok { return Err(HyperbytedbError::ShardMap( - resp.data - .message + resp.message .unwrap_or_else(|| "shard map op refused".into()) .into(), )); @@ -2920,6 +2918,22 @@ mod tests { } use super::*; + + fn test_consensus( + raft: crate::adapters::cluster::raft::HyperbytedbRaft, + ) -> Arc { + Arc::new( + crate::adapters::cluster::raft::consensus::RaftConsensus::new( + raft, + Arc::new( + crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ), + ), + None, + ), + ) + } use crate::adapters::sharding::rocksdb_shard_map::RocksDbShardMap; use crate::domain::cluster::membership::{ClusterMembership, NodeInfo, new_shared}; use crate::domain::sharding::ShardEpoch; @@ -3252,7 +3266,7 @@ mod tests { let scheduler = ShardScheduler::new( shard_map, bootstrap.membership.clone(), - raft, + test_consensus(raft), None, metadata, wal, @@ -3401,7 +3415,7 @@ mod tests { let scheduler = ShardScheduler::new( shard_map, bootstrap.membership.clone(), - raft, + test_consensus(raft), None, metadata, wal, @@ -3542,7 +3556,7 @@ mod tests { let scheduler = ShardScheduler::new( shard_map, bootstrap.membership.clone(), - raft, + test_consensus(raft), None, metadata, wal, @@ -3996,7 +4010,7 @@ mod tests { let scheduler = ShardScheduler::new( shard_map, bootstrap.membership.clone(), - raft, + test_consensus(raft), Some(crate::adapters::cluster::http_peer_rpc::peers_for( peer_client, )), @@ -4482,7 +4496,7 @@ mod tests { let scheduler = ShardScheduler::new( shard_map.clone(), bootstrap.membership.clone(), - raft, + test_consensus(raft), None, metadata, wal, diff --git a/hyperbytedb/src/bootstrap.rs b/hyperbytedb/src/bootstrap.rs index 3cebbb6..af87d7d 100644 --- a/hyperbytedb/src/bootstrap.rs +++ b/hyperbytedb/src/bootstrap.rs @@ -524,7 +524,6 @@ pub async fn build_services(config: &HyperbytedbConfig) -> anyhow::Result Result; +} diff --git a/hyperbytedb/src/ports/mod.rs b/hyperbytedb/src/ports/mod.rs index ee175ca..fe559c6 100644 --- a/hyperbytedb/src/ports/mod.rs +++ b/hyperbytedb/src/ports/mod.rs @@ -1,4 +1,5 @@ pub mod auth; +pub mod consensus; pub mod flush; pub mod forward_queue; pub mod ingestion; diff --git a/hyperbytedb/src/ports/peer_rpc.rs b/hyperbytedb/src/ports/peer_rpc.rs index 11b6f28..9f8e3a7 100644 --- a/hyperbytedb/src/ports/peer_rpc.rs +++ b/hyperbytedb/src/ports/peer_rpc.rs @@ -136,6 +136,14 @@ pub trait PeerRpc: Send + Sync { timeout: Duration, ) -> Result; + /// `POST /cluster/raft/client-write`: propose `req` on the Raft leader. + async fn raft_client_write( + &self, + addr: &str, + req: &crate::domain::cluster::raft_request::ClusterRequest, + timeout: Duration, + ) -> Result; + /// `GET /health`, falling back to `GET /ping` when `/health` gets no /// answer; each request is bounded by `timeout`. async fn probe( diff --git a/hyperbytedb/src/server.rs b/hyperbytedb/src/server.rs index e0731a1..ec3027c 100644 --- a/hyperbytedb/src/server.rs +++ b/hyperbytedb/src/server.rs @@ -186,12 +186,29 @@ async fn serve_inner( None => None, }; - // Wire Raft to PeerQueryService for consensus-based schema replication - if let (Some(raft), Some(pqs)) = (&raft_instance, &peer_query_service_ref) { - pqs.set_raft(raft.clone()); - if let Some(m) = &membership { - pqs.set_membership(m.clone()); - } + // One proposer for schema mutations, the scheduler and drain. Forwards + // to the leader go over a dedicated client with the cluster credential. + let consensus: Option> = + raft_instance.as_ref().map(|raft| { + use crate::ports::peer_http::PeerHttp; + let auth = &app_state.cluster_auth; + let rpc: crate::ports::peer_rpc::SharedPeerRpc = Arc::new( + crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + auth.build(auth.client_builder()), + ), + ); + Arc::new( + crate::adapters::cluster::raft::consensus::RaftConsensus::new( + raft.clone(), + rpc, + membership.clone(), + ), + ) as Arc + }); + + // Wire consensus to PeerQueryService for schema replication + if let (Some(consensus), Some(pqs)) = (&consensus, &peer_query_service_ref) { + pqs.set_consensus(consensus.clone()); if let Some(ctx) = &app_state.shard_routing { pqs.set_shard_routing(ctx.clone()); } @@ -231,13 +248,13 @@ async fn serve_inner( // Shard scheduler (leader-only PD-lite) and periodic region heartbeats. let shard_scheduler_handle = if config.sharding.enabled { - if let (Some(raft), Some(map), Some(membership)) = - (&raft_instance, &rocks_shard_map, &membership) + if let (Some(consensus), Some(map), Some(membership)) = + (&consensus, &rocks_shard_map, &membership) { let scheduler = Arc::new(crate::application::shard_scheduler::ShardScheduler::new( map.clone(), membership.clone(), - raft.clone(), + consensus.clone(), peer_client .clone() .map(crate::adapters::cluster::http_peer_rpc::peers_for), @@ -283,7 +300,7 @@ async fn serve_inner( app_state.metadata.clone(), app_state.points_sink.clone(), app_state.query_port.clone(), - raft.clone(), + consensus.clone(), config.server.max_points_per_request, ), )); @@ -305,7 +322,8 @@ async fn serve_inner( .unwrap_or_else(|| { Arc::new(crate::application::runtime::RegionWriteStats::new()) }); - let raft = raft.clone(); + let leadership: Arc = + consensus.clone(); let m = membership.clone(); let hb_interval = Duration::from_secs(config.sharding.heartbeat_interval_secs.max(1)); @@ -320,7 +338,7 @@ async fn serve_inner( )), m, map, - Arc::new(raft), + leadership, meta, query_port, region_write_stats, diff --git a/hyperbytedb/tests/common/sharding_cluster.rs b/hyperbytedb/tests/common/sharding_cluster.rs index 2fbeafe..30d88ef 100644 --- a/hyperbytedb/tests/common/sharding_cluster.rs +++ b/hyperbytedb/tests/common/sharding_cluster.rs @@ -258,12 +258,7 @@ pub async fn start_sharded_node( ingestion_svc.spawn_forward_drainer(); let ingestion: Arc = ingestion_svc; - let peer_query = PeerQueryService::new( - base_query, - metadata.clone(), - peer_client.clone(), - peer_client.cluster_auth(), - ); + let peer_query = PeerQueryService::new(base_query, metadata.clone(), peer_client.clone()); peer_query.set_shard_routing(shard_routing.clone()); let query: Arc = Arc::new(peer_query); @@ -275,7 +270,17 @@ pub async fn start_sharded_node( hyperbytedb::application::shard_scheduler::ShardScheduler::new( shard_map.clone(), shared_membership.clone(), - raft, + Arc::new( + hyperbytedb::adapters::cluster::raft::consensus::RaftConsensus::new( + raft, + Arc::new( + hyperbytedb::adapters::cluster::http_peer_rpc::HttpPeerRpc::new( + peer_client.clone(), + ), + ), + Some(shared_membership.clone()), + ), + ), Some(hyperbytedb::adapters::cluster::http_peer_rpc::peers_for( peer_client.clone(), )), diff --git a/hyperbytedb/tests/raft_integration.rs b/hyperbytedb/tests/raft_integration.rs index e89cdd3..71b4509 100644 --- a/hyperbytedb/tests/raft_integration.rs +++ b/hyperbytedb/tests/raft_integration.rs @@ -119,13 +119,9 @@ async fn start_cluster_node_with_listener( IngestCardinalityLimits::default(), )); - let query_service: Arc = - Arc::new(PeerQueryService::new( - base_query_service, - metadata.clone(), - peer_client.clone(), - peer_client.cluster_auth(), - )); + let query_service: Arc = Arc::new( + PeerQueryService::new(base_query_service, metadata.clone(), peer_client.clone()), + ); let app_state = Arc::new(AppState { ingestion: ingestion_service, diff --git a/hyperbytedb/tests/security_auth.rs b/hyperbytedb/tests/security_auth.rs index a57bfaf..9681415 100644 --- a/hyperbytedb/tests/security_auth.rs +++ b/hyperbytedb/tests/security_auth.rs @@ -134,13 +134,9 @@ async fn start_auth_cluster_node_with( IngestCardinalityLimits::default(), )); - let query_service: Arc = - Arc::new(PeerQueryService::new( - base_query_service, - metadata.clone(), - peer_client.clone(), - peer_client.cluster_auth(), - )); + let query_service: Arc = Arc::new( + PeerQueryService::new(base_query_service, metadata.clone(), peer_client.clone()), + ); let app_state = Arc::new(AppState { ingestion: ingestion_service, diff --git a/hyperbytedb/tests/sharding_failover_integration.rs b/hyperbytedb/tests/sharding_failover_integration.rs index c64e65c..ea061cd 100644 --- a/hyperbytedb/tests/sharding_failover_integration.rs +++ b/hyperbytedb/tests/sharding_failover_integration.rs @@ -117,7 +117,17 @@ async fn primary_failover_updates_shard_map_after_unhealthy_timeout() { let scheduler = Arc::new(ShardScheduler::new( shard_map.clone(), bootstrap.membership.clone(), - raft.clone(), + Arc::new( + hyperbytedb::adapters::cluster::raft::consensus::RaftConsensus::new( + raft.clone(), + Arc::new( + hyperbytedb::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( + reqwest::Client::new(), + ), + ), + Some(bootstrap.membership.clone()), + ), + ), None, metadata, wal, diff --git a/hyperbytedb/tests/sync_quorum_integration.rs b/hyperbytedb/tests/sync_quorum_integration.rs index f5a45f8..ee5093d 100644 --- a/hyperbytedb/tests/sync_quorum_integration.rs +++ b/hyperbytedb/tests/sync_quorum_integration.rs @@ -160,13 +160,9 @@ async fn start_node_on( replication, )); - let query: Arc = - Arc::new(PeerQueryService::new( - base_query, - metadata.clone(), - peer_client.clone(), - peer_client.cluster_auth(), - )); + let query: Arc = Arc::new( + PeerQueryService::new(base_query, metadata.clone(), peer_client.clone()), + ); let app_state = Arc::new(AppState { ingestion, From ecff54258824172ac3399b5b7d651bdc3c7eb598 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Tue, 10 Nov 2026 15:23:28 +0000 Subject: [PATCH 7/8] Application holds no HTTP client or Raft handle --- .../src/adapters/cluster/cluster_auth.rs | 2 +- .../cluster/hinted_handoff_watcher.rs | 2 +- hyperbytedb/src/adapters/cluster/mod.rs | 1 + .../src/adapters/cluster/peer_client.rs | 2 +- .../{ports => adapters/cluster}/peer_http.rs | 0 .../src/adapters/cluster/raft/network.rs | 2 +- .../src/adapters/cluster/sync_client.rs | 2 +- hyperbytedb/src/adapters/http/cluster.rs | 2 +- .../src/adapters/http/shard_handlers.rs | 4 ++-- hyperbytedb/src/application/cluster/drain.rs | 2 +- hyperbytedb/src/application/query_service.rs | 2 +- .../src/application/raft_leader_callbacks.rs | 21 +++++++++---------- hyperbytedb/src/application/shard_routing.rs | 17 +++++++-------- hyperbytedb/src/bootstrap.rs | 1 - hyperbytedb/src/composition/leader_monitor.rs | 2 +- hyperbytedb/src/composition/raft_formation.rs | 2 +- hyperbytedb/src/ports/mod.rs | 1 - hyperbytedb/src/server.rs | 8 +++---- hyperbytedb/tests/common/sharding_cluster.rs | 1 - hyperbytedb/tests/security_auth.rs | 2 +- .../tests/sharding_cluster_integration.rs | 7 ++++++- 21 files changed, 41 insertions(+), 42 deletions(-) rename hyperbytedb/src/{ports => adapters/cluster}/peer_http.rs (100%) diff --git a/hyperbytedb/src/adapters/cluster/cluster_auth.rs b/hyperbytedb/src/adapters/cluster/cluster_auth.rs index 9b34581..5542401 100644 --- a/hyperbytedb/src/adapters/cluster/cluster_auth.rs +++ b/hyperbytedb/src/adapters/cluster/cluster_auth.rs @@ -12,8 +12,8 @@ use std::sync::Arc; use reqwest::header::{HeaderMap, HeaderValue}; +use crate::adapters::cluster::peer_http::PeerHttp; use crate::error::HyperbytedbError; -use crate::ports::peer_http::PeerHttp; pub const HEADER: &str = "x-hyperbytedb-cluster-token"; diff --git a/hyperbytedb/src/adapters/cluster/hinted_handoff_watcher.rs b/hyperbytedb/src/adapters/cluster/hinted_handoff_watcher.rs index 9cf301b..0c81aa0 100644 --- a/hyperbytedb/src/adapters/cluster/hinted_handoff_watcher.rs +++ b/hyperbytedb/src/adapters/cluster/hinted_handoff_watcher.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use std::time::Duration; use crate::adapters::cluster::peer_client::PeerClient; +use crate::adapters::cluster::peer_http::PeerHttp; use crate::domain::cluster::membership::{NodeState, SharedMembership}; -use crate::ports::peer_http::PeerHttp; /// Polls membership state periodically: when a previously-disconnected /// peer transitions to Active, drain the hinted handoff queue for that peer diff --git a/hyperbytedb/src/adapters/cluster/mod.rs b/hyperbytedb/src/adapters/cluster/mod.rs index 3ed0211..0433b45 100644 --- a/hyperbytedb/src/adapters/cluster/mod.rs +++ b/hyperbytedb/src/adapters/cluster/mod.rs @@ -5,6 +5,7 @@ pub mod hinted_handoff; pub mod hinted_handoff_watcher; pub mod http_peer_rpc; pub mod peer_client; +pub mod peer_http; pub mod raft; pub mod replication_log; pub mod sync_client; diff --git a/hyperbytedb/src/adapters/cluster/peer_client.rs b/hyperbytedb/src/adapters/cluster/peer_client.rs index eea9c13..a864fc4 100644 --- a/hyperbytedb/src/adapters/cluster/peer_client.rs +++ b/hyperbytedb/src/adapters/cluster/peer_client.rs @@ -7,6 +7,7 @@ use tokio::sync::{Semaphore, oneshot}; use crate::adapters::cluster::cluster_auth::ClusterAuth; use crate::adapters::cluster::hinted_handoff::HintedHandoff; +use crate::adapters::cluster::peer_http::PeerHttp; use crate::adapters::cluster::replication_log::ReplicationLog; use crate::domain::cluster::membership::{ClusterMembership, SharedMembership}; use crate::domain::cluster::replication_wire::{ @@ -19,7 +20,6 @@ use crate::domain::cluster::types::{ MutationReplicateRequest, MutationReplicateResponse, MutationRequest, }; use crate::error::HyperbytedbError; -use crate::ports::peer_http::PeerHttp; use crate::ports::replication::{OutboundReplicationBatch, ReplicationPort}; pub struct PeerClient { diff --git a/hyperbytedb/src/ports/peer_http.rs b/hyperbytedb/src/adapters/cluster/peer_http.rs similarity index 100% rename from hyperbytedb/src/ports/peer_http.rs rename to hyperbytedb/src/adapters/cluster/peer_http.rs diff --git a/hyperbytedb/src/adapters/cluster/raft/network.rs b/hyperbytedb/src/adapters/cluster/raft/network.rs index d50ef4c..f6b6c87 100644 --- a/hyperbytedb/src/adapters/cluster/raft/network.rs +++ b/hyperbytedb/src/adapters/cluster/raft/network.rs @@ -25,7 +25,7 @@ impl Network { control_port: u16, cluster_auth: &crate::adapters::cluster::cluster_auth::ClusterAuth, ) -> Self { - use crate::ports::peer_http::PeerHttp; + use crate::adapters::cluster::peer_http::PeerHttp; let default_timeout = Duration::from_secs(default_timeout_secs); Self { // `Client::builder().build()` only fails if TLS init is broken, diff --git a/hyperbytedb/src/adapters/cluster/sync_client.rs b/hyperbytedb/src/adapters/cluster/sync_client.rs index 4400523..95f52ec 100644 --- a/hyperbytedb/src/adapters/cluster/sync_client.rs +++ b/hyperbytedb/src/adapters/cluster/sync_client.rs @@ -1,5 +1,5 @@ use crate::adapters::cluster::cluster_auth::ClusterAuth; -use crate::ports::peer_http::PeerHttp; +use crate::adapters::cluster::peer_http::PeerHttp; use std::sync::Arc; use std::time::Duration; diff --git a/hyperbytedb/src/adapters/http/cluster.rs b/hyperbytedb/src/adapters/http/cluster.rs index d49663b..0a30432 100644 --- a/hyperbytedb/src/adapters/http/cluster.rs +++ b/hyperbytedb/src/adapters/http/cluster.rs @@ -394,7 +394,7 @@ async fn fetch_target_raft( cluster_auth: &crate::adapters::cluster::cluster_auth::ClusterAuth, addr: &str, ) -> Option { - use crate::ports::peer_http::PeerHttp; + use crate::adapters::cluster::peer_http::PeerHttp; let client = cluster_auth .client_builder() .timeout(std::time::Duration::from_secs(3)) diff --git a/hyperbytedb/src/adapters/http/shard_handlers.rs b/hyperbytedb/src/adapters/http/shard_handlers.rs index 4b84ee8..c238b50 100644 --- a/hyperbytedb/src/adapters/http/shard_handlers.rs +++ b/hyperbytedb/src/adapters/http/shard_handlers.rs @@ -1004,10 +1004,10 @@ pub async fn handle_mv_handoff( State(state): State>, Json(req): Json, ) -> impl IntoResponse { - let Some(ctx) = state.shard_routing.as_ref() else { + let (Some(_), Some(pc)) = (state.shard_routing.as_ref(), state.peer_client.as_ref()) else { return sharding_disabled(); }; - let client = ctx.peer_client.http_client().clone(); + let client = pc.http_client().clone(); let url = format!("http://{}/internal/mv/handoff-chunk", req.to_addr); let result = crate::application::mv_handoff::hand_off_rollups( state.node_id, diff --git a/hyperbytedb/src/application/cluster/drain.rs b/hyperbytedb/src/application/cluster/drain.rs index ebe73be..c029574 100644 --- a/hyperbytedb/src/application/cluster/drain.rs +++ b/hyperbytedb/src/application/cluster/drain.rs @@ -255,7 +255,7 @@ impl DrainService { /// anyway, so the wait on them always ran its full 90 s whenever this node /// had written anything -- every restart of a replicated node. Writes are /// settled at shutdown instead: each is acked by its peers, or hinted for - /// them (`PeerClient::wait_for_inflight`). + /// them (the peer client's in-flight wait at shutdown). async fn wait_for_replication_acks(&self) -> Result<(), HyperbytedbError> { let local_mutation_seq = self.replication_log.last_mutation_seq(); let max_wait = Duration::from_secs(90); diff --git a/hyperbytedb/src/application/query_service.rs b/hyperbytedb/src/application/query_service.rs index f752cf3..a1a21c7 100644 --- a/hyperbytedb/src/application/query_service.rs +++ b/hyperbytedb/src/application/query_service.rs @@ -2849,7 +2849,7 @@ async fn execute_sharded_measurement_query( let mut region_results = Vec::with_capacity(selected_regions.len()); if let Some(meta) = materialized_meta_owned.as_ref() { let holders = { - let m = ctx.peer_client.membership().read().await; + let m = ctx.membership.read().await; materialized_partial_holders(ctx.node_id, &m) }; let futures: Vec<_> = holders diff --git a/hyperbytedb/src/application/raft_leader_callbacks.rs b/hyperbytedb/src/application/raft_leader_callbacks.rs index 825116b..b9c2f40 100644 --- a/hyperbytedb/src/application/raft_leader_callbacks.rs +++ b/hyperbytedb/src/application/raft_leader_callbacks.rs @@ -4,12 +4,12 @@ use std::sync::Arc; use parking_lot::RwLock; -use crate::adapters::cluster::raft::HyperbytedbRaft; use crate::domain::cluster::membership::SharedMembership; +use crate::ports::leadership::LeadershipPort; /// Thread-safe leader callbacks wired after Raft starts in runtime. pub struct RaftLeaderCallbacks { - raft: RwLock>, + leadership: RwLock>>, leader_addr: RwLock>, membership: Option, node_id: u64, @@ -28,7 +28,7 @@ impl RaftLeaderCallbacks { self_addr: Option, ) -> Self { Self { - raft: RwLock::new(None), + leadership: RwLock::new(None), leader_addr: RwLock::new(None), membership, node_id, @@ -36,25 +36,24 @@ impl RaftLeaderCallbacks { } } - pub fn set_raft(&self, raft: HyperbytedbRaft) { - *self.raft.write() = Some(raft); + pub fn set_leadership(&self, leadership: Arc) { + *self.leadership.write() = Some(leadership); self.refresh_leader_addr(); } pub fn is_leader(&self) -> bool { - self.raft + self.leadership .read() .as_ref() - .map(|r| r.metrics().borrow().current_leader == Some(self.node_id)) - .unwrap_or(false) + .is_some_and(|l| l.is_leader(self.node_id)) } pub fn refresh_leader_addr(&self) { - let guard = self.raft.read(); - let Some(raft) = guard.as_ref() else { + let guard = self.leadership.read(); + let Some(leadership) = guard.as_ref() else { return; }; - let Some(leader_id) = raft.metrics().borrow().current_leader else { + let Some(leader_id) = leadership.current_leader() else { return; }; if leader_id == self.node_id diff --git a/hyperbytedb/src/application/shard_routing.rs b/hyperbytedb/src/application/shard_routing.rs index 43b5bd7..d9b467f 100644 --- a/hyperbytedb/src/application/shard_routing.rs +++ b/hyperbytedb/src/application/shard_routing.rs @@ -7,7 +7,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use metrics::{counter, histogram}; -use crate::adapters::cluster::peer_client::PeerClient; use crate::application::runtime::RegionWriteStats; use crate::application::shard_peer_resolution::{ RegionTargetRole, ScatterKind, active_region_peer_targets, is_serving_node, peer_addr, @@ -30,8 +29,7 @@ pub struct ShardRoutingContext { pub location_cache: Arc, pub config: ShardingConfig, pub node_id: u64, - pub peer_client: Arc, - /// Calls to peers (replaces direct HTTP through `peer_client`). + /// Calls to peers. pub peer_rpc: crate::ports::peer_rpc::SharedPeerRpc, /// Cluster membership, for resolving node addresses. pub membership: crate::domain::cluster::membership::SharedMembership, @@ -93,7 +91,7 @@ where F: FnMut(u64, &str, Duration) -> Fut, Fut: Future>, { - let membership = ctx.peer_client.membership().read().await; + let membership = ctx.membership.read().await; let mut candidates = resolve_region_peers(region, ctx.node_id, &membership, role); candidates.retain(|id| *id != ctx.node_id); @@ -339,7 +337,7 @@ pub async fn build_bootstrap_op( /// Every active node (and this one), sorted: the candidates a new /// measurement's regions are placed over. async fn bootstrap_candidates(ctx: &ShardRoutingContext) -> Vec { - let membership = ctx.peer_client.membership().read().await; + let membership = ctx.membership.read().await; placeable_members(&membership, ctx.node_id) } @@ -476,7 +474,7 @@ async fn select_bootstrap_peers( rp: &str, measurement: &str, ) -> Result, HyperbytedbError> { - let membership = ctx.peer_client.membership().read().await; + let membership = ctx.membership.read().await; let peers = placeable_members(&membership, ctx.node_id); drop(membership); let member_count = peers.len(); @@ -604,7 +602,7 @@ pub async fn forward_shard_writes_by_node( }; let (by_node, unrouted, addrs) = { - let membership = ctx.peer_client.membership().read().await; + let membership = ctx.membership.read().await; let (by_node, unrouted) = group_forwards_by_primary(®ions, ctx.node_id, &membership); let addrs: HashMap = by_node .keys() @@ -943,7 +941,7 @@ pub async fn region_replication_targets( ctx: &ShardRoutingContext, region: &ShardRegion, ) -> Vec { - let membership = ctx.peer_client.membership().read().await; + let membership = ctx.membership.read().await; active_region_peer_targets(region, ctx.node_id, &membership) } @@ -1033,7 +1031,7 @@ pub async fn scatter_delete_to_regions( continue; } - let membership = ctx.peer_client.membership().read().await; + let membership = ctx.membership.read().await; if !is_serving_node(&membership, region.primary) { tracing::warn!( region_id = region.region_id, @@ -1230,7 +1228,6 @@ mod scatter_tests { peer_client.clone(), )), membership: peer_client.membership().clone(), - peer_client, region_write_stats: Arc::new(RegionWriteStats::new()), forward_coalescer: Default::default(), runtime_settings: Default::default(), diff --git a/hyperbytedb/src/bootstrap.rs b/hyperbytedb/src/bootstrap.rs index af87d7d..bde0b57 100644 --- a/hyperbytedb/src/bootstrap.rs +++ b/hyperbytedb/src/bootstrap.rs @@ -362,7 +362,6 @@ pub async fn build_services(config: &HyperbytedbConfig) -> anyhow::Result> = raft_instance.as_ref().map(|raft| { - use crate::ports::peer_http::PeerHttp; + use crate::adapters::cluster::peer_http::PeerHttp; let auth = &app_state.cluster_auth; let rpc: crate::ports::peer_rpc::SharedPeerRpc = Arc::new( crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client( @@ -242,8 +242,8 @@ async fn serve_inner( // Set Raft instance on AppState before building the router app_state.raft = raft_instance.clone(); - if let (Some(raft), Some(callbacks)) = (&raft_instance, &raft_leader_callbacks) { - callbacks.set_raft(raft.clone()); + if let (Some(consensus), Some(callbacks)) = (&consensus, &raft_leader_callbacks) { + callbacks.set_leadership(consensus.clone()); } // Shard scheduler (leader-only PD-lite) and periodic region heartbeats. diff --git a/hyperbytedb/tests/common/sharding_cluster.rs b/hyperbytedb/tests/common/sharding_cluster.rs index 30d88ef..ecb079a 100644 --- a/hyperbytedb/tests/common/sharding_cluster.rs +++ b/hyperbytedb/tests/common/sharding_cluster.rs @@ -173,7 +173,6 @@ pub async fn start_sharded_node( location_cache: location_cache.clone(), config: opts.sharding.clone(), node_id, - peer_client: peer_client.clone(), peer_rpc: Arc::new( hyperbytedb::adapters::cluster::http_peer_rpc::HttpPeerRpc::new(peer_client.clone()), ), diff --git a/hyperbytedb/tests/security_auth.rs b/hyperbytedb/tests/security_auth.rs index 9681415..03e99ed 100644 --- a/hyperbytedb/tests/security_auth.rs +++ b/hyperbytedb/tests/security_auth.rs @@ -466,7 +466,7 @@ async fn grant_revoke_controls_query_access() { #[serial(chdb)] async fn the_cluster_token_opens_internal_routes_only() { use hyperbytedb::adapters::cluster::cluster_auth::{self, ClusterAuth}; - use hyperbytedb::ports::peer_http::PeerHttp; + use hyperbytedb::adapters::cluster::peer_http::PeerHttp; let dir = tempfile::tempdir().unwrap(); let auth = ClusterAuth::new(Some("cluster-s3cret")).unwrap(); diff --git a/hyperbytedb/tests/sharding_cluster_integration.rs b/hyperbytedb/tests/sharding_cluster_integration.rs index 3902b6c..51e1b58 100644 --- a/hyperbytedb/tests/sharding_cluster_integration.rs +++ b/hyperbytedb/tests/sharding_cluster_integration.rs @@ -3295,7 +3295,12 @@ async fn series_cardinality_spans_regions() { } assert_eq!(sides.len(), 2, "the series must span both regions"); let resp = write_line(&client, &nodes[0].url, "cdb", &lines.join("\n")).await; - assert!(resp.status().is_success(), "{}", resp.status()); + let status = resp.status(); + assert!( + status.is_success(), + "{status}: {}", + resp.text().await.unwrap_or_default() + ); for n in &nodes { flush_node(n).await; } From f5260d043492abd257822fec2d53081aa9b84851 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Tue, 10 Nov 2026 15:47:14 +0000 Subject: [PATCH 8/8] Test the consensus forward retry loop --- .../src/adapters/cluster/raft/consensus.rs | 157 +++++++++++++++--- 1 file changed, 131 insertions(+), 26 deletions(-) diff --git a/hyperbytedb/src/adapters/cluster/raft/consensus.rs b/hyperbytedb/src/adapters/cluster/raft/consensus.rs index 14727e5..3d25754 100644 --- a/hyperbytedb/src/adapters/cluster/raft/consensus.rs +++ b/hyperbytedb/src/adapters/cluster/raft/consensus.rs @@ -85,39 +85,78 @@ impl LeadershipPort for RaftConsensus { #[async_trait::async_trait] impl ConsensusPort for RaftConsensus { async fn propose(&self, req: ClusterRequest) -> Result { - for attempt in 0..MAX_ATTEMPTS { - let e = match self.raft.client_write(req.clone()).await { - Ok(resp) => return Ok(resp.data), - Err(e) => e, - }; - let Some(fwd) = e.forward_to_leader::() else { + let local = || async { + match self.raft.client_write(req.clone()).await { + Ok(resp) => LocalWrite::Done(resp.data), + Err(e) => match e.forward_to_leader::() { + Some(fwd) => LocalWrite::Forward(fwd.clone()), + None => LocalWrite::Failed(e.to_string()), + }, + } + }; + propose_via(self.rpc.as_ref(), &req, local, |fwd| async move { + self.leader_addr(&fwd).await + }) + .await + } +} + +/// What proposing on this node's own Raft gave. +pub(crate) enum LocalWrite { + /// Committed here. + Done(ClusterResponse), + /// Not the leader: forward to the one this names. + Forward(ForwardToLeader), + /// Failed for another reason. + Failed(String), +} + +/// Propose with `local`; when it says to forward, resolve the leader with +/// `leader_addr` and forward over `rpc`, at most `MAX_ATTEMPTS` times in all. +pub(crate) async fn propose_via( + rpc: &dyn PeerRpc, + req: &ClusterRequest, + local: L, + leader_addr: A, +) -> Result +where + L: Fn() -> LF, + LF: std::future::Future, + A: Fn(ForwardToLeader) -> AF, + AF: std::future::Future>, +{ + for attempt in 0..MAX_ATTEMPTS { + let fwd = match local().await { + LocalWrite::Done(resp) => return Ok(resp), + LocalWrite::Failed(e) => { tracing::error!(error = %e, "raft proposal failed"); return Err(HyperbytedbError::Internal( format!("raft replication failed: {e}").into(), )); - }; - tracing::debug!( - leader_id = ?fwd.leader_id, - leader_addr = ?fwd.leader_node.as_ref().map(|n| n.addr.as_str()), - attempt, - "forwarding proposal to raft leader" - ); - let result = match self.leader_addr(fwd).await { - Ok(addr) => forward(self.rpc.as_ref(), &addr, &req).await, - Err(e) => Err(e), - }; - match result { - Ok(resp) => return Ok(resp), - Err(e) if attempt + 1 >= MAX_ATTEMPTS => return Err(e), - Err(e) => { - tracing::warn!(error = %e, attempt, "forward to raft leader failed; retrying"); - } + } + LocalWrite::Forward(fwd) => fwd, + }; + tracing::debug!( + leader_id = ?fwd.leader_id, + leader_addr = ?fwd.leader_node.as_ref().map(|n| n.addr.as_str()), + attempt, + "forwarding proposal to raft leader" + ); + let result = match leader_addr(fwd).await { + Ok(addr) => forward(rpc, &addr, req).await, + Err(e) => Err(e), + }; + match result { + Ok(resp) => return Ok(resp), + Err(e) if attempt + 1 >= MAX_ATTEMPTS => return Err(e), + Err(e) => { + tracing::warn!(error = %e, attempt, "forward to raft leader failed; retrying"); } } - Err(HyperbytedbError::Internal(crate::error::ChainedError::new( - "raft replication failed after retries", - ))) } + Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "raft replication failed after retries", + ))) } /// Send `req` to the leader at `leader_addr` and return its answer. @@ -191,6 +230,72 @@ mod tests { (addr, seen) } + /// A leader that answers 503 to its first `fail_first` proposals and + /// accepts the rest; returns its address and a request counter. + async fn flaky_leader(fail_first: usize) -> (String, Arc>) { + let count = Arc::new(Mutex::new(0usize)); + let c = count.clone(); + let app = axum::Router::new().route( + "/cluster/raft/client-write", + axum::routing::post(move || { + let c = c.clone(); + async move { + let n = { + let mut g = c.lock().unwrap(); + *g += 1; + *g + }; + let status = if n <= fail_first { + axum::http::StatusCode::SERVICE_UNAVAILABLE + } else { + axum::http::StatusCode::OK + }; + (status, axum::Json(ClusterResponse::success())) + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (addr, count) + } + + /// A follower keeps forwarding through a leader that fails, at most + /// `MAX_ATTEMPTS` times, and re-resolves the leader on each attempt. + #[tokio::test] + async fn propose_retries_forwards_up_to_the_cap() { + let rpc = HttpPeerRpc::with_client(reqwest::Client::new()); + let not_leader = || async { + LocalWrite::Forward(ForwardToLeader { + leader_id: Some(2), + leader_node: None, + }) + }; + + let (down, hits) = flaky_leader(usize::MAX).await; + let resolved = Mutex::new(0usize); + let err = propose_via(&rpc, &op(), not_leader, |_| { + *resolved.lock().unwrap() += 1; + let a = down.clone(); + async move { Ok(a) } + }) + .await + .unwrap_err(); + assert!(err.to_string().contains("503"), "{err}"); + assert_eq!(*hits.lock().unwrap(), MAX_ATTEMPTS as usize); + assert_eq!(*resolved.lock().unwrap(), MAX_ATTEMPTS as usize); + + let (recovers, hits) = flaky_leader(1).await; + let resp = propose_via(&rpc, &op(), not_leader, |_| { + let a = recovers.clone(); + async move { Ok(a) } + }) + .await + .unwrap(); + assert!(resp.ok); + assert_eq!(*hits.lock().unwrap(), 2); + } + fn op() -> ClusterRequest { ClusterRequest::ShardMapMutation(Box::new(ShardMapOp::DropSpace { db: "db".into(),