From 44fbe451590aa6b42f1f9662a4ad61245adbca0b Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Fri, 20 Nov 2026 05:06:24 +0000 Subject: [PATCH 1/6] Drain fails a primary handoff the apply refused consensus.propose returns Ok for a committed op that the apply refused (a stale epoch). Drain ignored ClusterResponse.ok, so a refused TransferPrimary counted as handed off. Check it, as the shard scheduler does. --- hyperbytedb/src/application/cluster/drain.rs | 66 ++++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/hyperbytedb/src/application/cluster/drain.rs b/hyperbytedb/src/application/cluster/drain.rs index 55061b7..bee1393 100644 --- a/hyperbytedb/src/application/cluster/drain.rs +++ b/hyperbytedb/src/application/cluster/drain.rs @@ -233,17 +233,13 @@ impl DrainService { .await?; } - use crate::domain::cluster::raft_request::ClusterRequest; let tp = ShardMapOp::TransferPrimary { key: key.clone(), region_id: region.region_id, new_primary, epoch: region.epoch, }; - consensus - .propose(ClusterRequest::ShardMapMutation(Box::new(tp))) - .await - .map_err(|e| HyperbytedbError::ShardMap(e.to_string().into()))?; + propose_handoff(consensus.as_ref(), tp).await?; } Ok(()) } @@ -294,6 +290,27 @@ impl DrainService { } /// The measurements materialized views write into. +/// Propose one primary handoff. A committed op the apply refused (a stale +/// epoch) moved nothing, so it is an error, as in the shard scheduler. +async fn propose_handoff( + consensus: &dyn ConsensusPort, + op: ShardMapOp, +) -> Result<(), HyperbytedbError> { + use crate::domain::cluster::raft_request::ClusterRequest; + let resp = consensus + .propose(ClusterRequest::ShardMapMutation(Box::new(op))) + .await + .map_err(|e| HyperbytedbError::ShardMap(e.to_string().into()))?; + if !resp.ok { + return Err(HyperbytedbError::ShardMap( + resp.message + .unwrap_or_else(|| "primary handoff refused".to_string()) + .into(), + )); + } + Ok(()) +} + async fn view_destinations( metadata: &dyn MetadataPort, ) -> Result, HyperbytedbError> { @@ -399,6 +416,45 @@ mod tests { ); } + /// A consensus whose apply refuses every op, as a stale epoch does. + struct RefusingConsensus; + + impl crate::ports::leadership::LeadershipPort for RefusingConsensus { + fn current_leader(&self) -> Option { + Some(1) + } + } + + #[async_trait] + impl ConsensusPort for RefusingConsensus { + async fn propose( + &self, + _req: crate::domain::cluster::raft_request::ClusterRequest, + ) -> Result + { + Ok(crate::domain::cluster::raft_request::ClusterResponse { + ok: false, + message: Some("stale epoch".into()), + }) + } + } + + /// A TransferPrimary the apply refused moved nothing: the handoff fails + /// instead of counting the region as handed off. + #[tokio::test] + async fn a_refused_primary_handoff_is_an_error() { + let op = ShardMapOp::TransferPrimary { + key: MeasurementKey::new("db", "autogen", "cpu"), + region_id: 1, + new_primary: 2, + epoch: Default::default(), + }; + let err = propose_handoff(&RefusingConsensus, op) + .await + .expect_err("a refused handoff must fail"); + assert!(err.to_string().contains("stale epoch"), "{err}"); + } + fn make_node(id: u64, state: NodeState) -> NodeInfo { NodeInfo { node_id: id, From b308938592a29da93d40edc1db71cc4760c5b8a9 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Fri, 20 Nov 2026 05:08:08 +0000 Subject: [PATCH 2/6] A reopened WAL reports its truncated entries flushed flushed_through started at 0 on open, while last_sequence is restored from disk. Only the flush truncates, so every entry below the oldest one left was flushed. With the mark at 0, every DELETE and DROP SERIES on a restarted node that had nothing left to flush waited the full 15 s for it, inside the Raft apply on replicated nodes. --- hyperbytedb/src/adapters/wal/rocksdb_wal.rs | 28 ++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/hyperbytedb/src/adapters/wal/rocksdb_wal.rs b/hyperbytedb/src/adapters/wal/rocksdb_wal.rs index 56a4ebf..e410a16 100644 --- a/hyperbytedb/src/adapters/wal/rocksdb_wal.rs +++ b/hyperbytedb/src/adapters/wal/rocksdb_wal.rs @@ -243,7 +243,9 @@ impl RocksDbWal { Ok(Self { db, seq: Arc::new(AtomicU64::new(seq)), - flushed_through: AtomicU64::new(0), + // Only the flush truncates, so every entry below the oldest one + // left was flushed. + flushed_through: AtomicU64::new(first.saturating_sub(1)), truncated_through: Arc::new(AtomicU64::new(first)), range_deletes_unflushed: Arc::new(AtomicU64::new(0)), sizes: Arc::new(std::sync::Mutex::new(std::collections::VecDeque::new())), @@ -800,6 +802,30 @@ mod tests { } } + /// Only the flush truncates the WAL, so every entry below the oldest one + /// left was flushed. A reopened WAL says so: a DELETE on a restarted + /// node waits for `flushed_through` to reach `last_sequence`, and with + /// nothing left to flush it waited its full 15 s every time. + #[tokio::test] + async fn a_reopened_wal_reports_truncated_entries_flushed() { + let dir = TempDir::new().unwrap(); + { + let wal = RocksDbWal::open(dir.path()).unwrap(); + for _ in 0..5 { + wal.append(cpu_entry(1)).await.unwrap(); + } + wal.truncate_before(4).await.unwrap(); + } + let wal = RocksDbWal::open(dir.path()).unwrap(); + assert_eq!(wal.flushed_through(), 3); + // All flushed and truncated: caught up with the last sequence. + wal.truncate_before(6).await.unwrap(); + drop(wal); + let wal = RocksDbWal::open(dir.path()).unwrap(); + assert_eq!(wal.flushed_through(), wal.last_sequence().await.unwrap()); + assert_eq!(wal.flushed_through(), 5); + } + fn memtable_counts(wal: &RocksDbWal) -> (u64, u64) { let cf = wal.db.cf_handle(WAL_CF).unwrap(); let get = |p: &str| wal.db.property_int_value_cf(&cf, p).unwrap().unwrap(); From af9fc153abb0fd7569e903fcb228278d1e06291a Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Fri, 20 Nov 2026 05:10:47 +0000 Subject: [PATCH 3/6] DROP SERIES checks its condition before dropping series drop_series wiped the measurement's series metadata, then built the delete predicate, which refuses a field or an unknown tag. A refused DROP SERIES returned an error and left SHOW SERIES empty. Build every predicate first. DROP SERIES with no FROM deleted rows only on a sharded node; on a single node it dropped the series and kept every row, which tombstones no longer hide. It now deletes them everywhere. --- .../query_service/statements/delete.rs | 11 ++-- hyperbytedb/tests/compat/ddl_tests.rs | 59 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/hyperbytedb/src/application/query_service/statements/delete.rs b/hyperbytedb/src/application/query_service/statements/delete.rs index e2dc1ff..caab6f5 100644 --- a/hyperbytedb/src/application/query_service/statements/delete.rs +++ b/hyperbytedb/src/application/query_service/statements/delete.rs @@ -100,11 +100,11 @@ pub(in crate::application::query_service) async fn drop_series( String::new() }; if measurement.is_some() || !predicate_sql.is_empty() { - svc.metadata - .delete_series_matching(target_db, &rp, measurement.as_deref(), &predicate_sql) - .await?; // The series' rows are deleted; points written to them later // stay. A tombstone used to hide those too. + // + // The predicates are built (and a field or unknown tag refused) + // before anything is dropped: an error must leave the series. let mut targets: Vec<(String, String)> = Vec::new(); if let Some(ref meas) = measurement { let pred = crate::application::predicate_sql::build_delete_predicate( @@ -116,7 +116,7 @@ pub(in crate::application::query_service) async fn drop_series( ) .await?; targets.push((meas.clone(), pred)); - } else if svc.shard_routing.is_some() { + } else { // Every measurement that has the condition's tags. for meas in svc.metadata.list_measurements(target_db).await? { if let Ok(pred) = crate::application::predicate_sql::build_delete_predicate( @@ -132,6 +132,9 @@ pub(in crate::application::query_service) async fn drop_series( } } } + svc.metadata + .delete_series_matching(target_db, &rp, measurement.as_deref(), &predicate_sql) + .await?; for (meas, pred) in targets { if let Some(ctx) = svc.shard_routing.as_ref() { scatter_delete_to_regions( diff --git a/hyperbytedb/tests/compat/ddl_tests.rs b/hyperbytedb/tests/compat/ddl_tests.rs index 4ca90e6..85e6207 100644 --- a/hyperbytedb/tests/compat/ddl_tests.rs +++ b/hyperbytedb/tests/compat/ddl_tests.rs @@ -483,6 +483,65 @@ async fn drop_series_keeps_points_written_after_it() { assert_eq!(values(&ctx, "SELECT value FROM cpu").await, vec![2.0, 3.0]); } +/// A refused DROP SERIES changes nothing. It used to wipe every series of +/// the measurement before the condition was checked, so SHOW SERIES came +/// back empty after an error. +#[tokio::test] +#[serial(chdb)] +async fn a_refused_drop_series_keeps_the_series() { + let Some(ctx) = chdb_context() else { return }; + ctx.metadata.create_database("testdb").await.unwrap(); + ctx.write_and_flush( + "testdb", + "cpu,host=a value=1.0 1000000000\ncpu,host=b value=2.0 2000000000", + ) + .await + .unwrap(); + for q in [ + "DROP SERIES FROM cpu WHERE value > 0", + "DROP SERIES FROM cpu WHERE nosuchtag = 'x'", + ] { + let failed = match ctx.query("testdb", q).await { + Err(_) => true, + Ok(resp) => resp.results[0].error.is_some(), + }; + assert!(failed, "{q} must be refused"); + let series = ctx + .metadata + .list_series("testdb", "autogen", "cpu") + .await + .unwrap(); + assert_eq!(series.len(), 2, "{q} dropped series"); + } +} + +/// DROP SERIES with no FROM deletes the matching rows of every measurement +/// that has the tag, on a single node as in a cluster. +#[tokio::test] +#[serial(chdb)] +async fn drop_series_without_from_deletes_the_rows() { + let Some(ctx) = chdb_context() else { return }; + ctx.metadata.create_database("testdb").await.unwrap(); + ctx.write_and_flush( + "testdb", + "cpu,host=a value=1.0 1000000000\ncpu,host=b value=2.0 2000000000\n\ + mem,host=a value=3.0 1000000000", + ) + .await + .unwrap(); + let resp = ctx + .query("testdb", r#"DROP SERIES WHERE "host" = 'a'"#) + .await + .unwrap(); + assert!( + resp.results[0].error.is_none(), + "{:?}", + resp.results[0].error + ); + assert_eq!(values(&ctx, "SELECT value FROM cpu").await, vec![2.0]); + assert!(values(&ctx, "SELECT value FROM mem").await.is_empty()); +} + #[tokio::test] #[serial(chdb)] async fn drop_measurement_clears_its_tombstones() { From 4dede2dc38cedc2e0f6db2794177c9e15c390ae8 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Fri, 20 Nov 2026 05:13:08 +0000 Subject: [PATCH 4/6] Sharded DELETE: one request per primary, timed to the flush wait A peer deletes a DELETE's rows from its whole table after waiting up to 15 s for its WAL to flush. The coordinator sent one request per region on the 5 s peer timeout: a peer leading K regions ran K full-table deletes, and under steady ingest the coordinator timed out and reported a failed delete that the peer then carried out. Send each primary one request, with the flush wait plus the peer timeout to answer in. --- hyperbytedb/src/application/predicate_sql.rs | 2 +- hyperbytedb/src/application/shard_routing.rs | 81 +++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/hyperbytedb/src/application/predicate_sql.rs b/hyperbytedb/src/application/predicate_sql.rs index e144c76..eaef164 100644 --- a/hyperbytedb/src/application/predicate_sql.rs +++ b/hyperbytedb/src/application/predicate_sql.rs @@ -194,7 +194,7 @@ pub async fn delete_rows( /// Longest wait for the flush service to write the WAL's current entries: /// past one flush interval (10 s by default), well inside the query /// timeout (30 s by default). -const DELETE_FLUSH_WAIT: std::time::Duration = std::time::Duration::from_secs(15); +pub(crate) const DELETE_FLUSH_WAIT: std::time::Duration = std::time::Duration::from_secs(15); async fn wait_for_flush(wal: &dyn crate::ports::wal::WalPort) -> Result<(), HyperbytedbError> { let target = wal.last_sequence().await?; diff --git a/hyperbytedb/src/application/shard_routing.rs b/hyperbytedb/src/application/shard_routing.rs index ab96fde..bc222ed 100644 --- a/hyperbytedb/src/application/shard_routing.rs +++ b/hyperbytedb/src/application/shard_routing.rs @@ -999,6 +999,12 @@ pub async fn scatter_delete_to_regions( let mut outcome = ShardDeleteOutcome::default(); // One delete removes the rows of every local region. let mut deleted_locally = false; + // Likewise on a peer: it deletes from its whole table, so each primary + // gets one request however many regions it leads. + let mut deleted_on: HashSet = HashSet::new(); + // The peer first waits for its WAL to flush, then runs the mutation. + let peer_timeout = Duration::from_millis(ctx.config.scatter_peer_timeout_ms.max(1)); + let timeout = crate::application::predicate_sql::DELETE_FLUSH_WAIT + peer_timeout; for region in &space.regions { if region.primary == ctx.node_id { @@ -1030,6 +1036,11 @@ pub async fn scatter_delete_to_regions( outcome.applied += 1; continue; } + if deleted_on.contains(®ion.primary) { + counter!("hyperbytedb_shard_delete_applied_total").increment(1); + outcome.applied += 1; + continue; + } let membership = ctx.membership.read().await; if !is_serving_node(&membership, region.primary) { @@ -1065,7 +1076,6 @@ pub async fn scatter_delete_to_regions( Some(predicate.to_string()) }, }; - let timeout = Duration::from_millis(ctx.config.scatter_peer_timeout_ms.max(1)); use crate::domain::cluster::peer_rpc::PeerRpcError; match ctx.peer_rpc.shard_delete(&addr, &req, timeout).await { Err(PeerRpcError::Status { code: 409, .. }) => { @@ -1099,6 +1109,7 @@ pub async fn scatter_delete_to_regions( ); } Ok(()) => { + deleted_on.insert(region.primary); outcome.applied += 1; } } @@ -1363,6 +1374,74 @@ mod scatter_tests { new_shared(m) } + /// A peer deletes a DELETE's rows from its whole table, after waiting + /// for its WAL to flush (up to 15 s), so: one request per primary, not + /// per region, and a timeout that covers the wait. Each region used to + /// get its own request on the 5 s peer timeout: the coordinator reported + /// a failed delete that the peer then carried out. + #[tokio::test] + async fn a_delete_reaches_each_primary_once_and_waits_for_it() { + use crate::domain::sharding::{MeasurementKey, ShardMapOp}; + let hits = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let h = { + let hits = hits.clone(); + let app = Router::new().route( + "/internal/shard/delete", + post(move || { + let hits = hits.clone(); + async move { + hits.fetch_add(1, Ordering::SeqCst); + // Longer than the peer timeout, as a WAL flush wait is. + tokio::time::sleep(Duration::from_millis(400)).await; + StatusCode::OK + } + }), + ); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }) + }; + let config = ShardingConfig { + scatter_peer_timeout_ms: 100, + ..Default::default() + }; + let ctx = test_ctx(membership_with(&[(2, &addr)]), config, 1); + let key = MeasurementKey::new("db", "autogen", "cpu"); + let region = |id: u64, start: u64, end: u64| ShardRegion { + region_id: id, + start, + end, + peers: vec![2], + primary: 2, + ..sample_region() + }; + ctx.shard_map + .apply_op(ShardMapOp::BootstrapMeasurementPresplit { + key, + regions: vec![region(1, 0, 1 << 63), region(2, 1 << 63, u64::MAX)], + }) + .await + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let metadata = + crate::adapters::metadata::rocksdb_meta::RocksDbMetadata::open(dir.path()).unwrap(); + + let outcome = scatter_delete_to_regions( + &ctx, + &metadata, + None, + "db", + "autogen", + "cpu", + "`series_id` = 1", + ) + .await + .expect("the delete completes"); + assert_eq!(hits.load(Ordering::SeqCst), 1, "one request per primary"); + assert_eq!(outcome.applied, 2); + h.abort(); + } + #[tokio::test] async fn scatter_invokes_callback_with_peer_addrs() { let (addr1, h1) = spawn_mock_peer(StatusCode::INTERNAL_SERVER_ERROR, "fail").await; From d4da0688e1b42c18a1aa8d4672dedfbb79354b5f Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Fri, 20 Nov 2026 05:14:26 +0000 Subject: [PATCH 5/6] Match chDB error codes exactly in delete fallbacks is_unknown_table matched "Code: 60", which codes 600-609 also start with, and a delete that failed with one reported success. is_unknown_column matched "Code: 47" (470-479), and a replica stored such a failed delete as a tombstone. Match "Code: 60." and "Code: 47.". --- .../src/adapters/chdb/native_adapter.rs | 14 +++++++++++++- .../src/application/schema_mutation_apply.rs | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/hyperbytedb/src/adapters/chdb/native_adapter.rs b/hyperbytedb/src/adapters/chdb/native_adapter.rs index cafd910..f62d6e5 100644 --- a/hyperbytedb/src/adapters/chdb/native_adapter.rs +++ b/hyperbytedb/src/adapters/chdb/native_adapter.rs @@ -141,7 +141,7 @@ fn table_schema_from_measurement_meta( /// A chDB error for a table that does not exist. fn is_unknown_table(msg: &str) -> bool { - msg.contains("UNKNOWN_TABLE") || msg.contains("Code: 60") || msg.contains("doesn't exist") + msg.contains("UNKNOWN_TABLE") || msg.contains("Code: 60.") || msg.contains("doesn't exist") } fn tag_column_kind(distinct_values: usize, threshold: usize) -> ColumnKind { @@ -2688,6 +2688,18 @@ mod tests { use arrow::array::Float64Array; use std::collections::BTreeMap; + /// A delete that fails for another reason must not read as "no table, + /// nothing to delete": codes 600-609 start with "Code: 60" too. + #[test] + fn only_code_60_is_an_unknown_table() { + assert!(is_unknown_table( + "Code: 60. DB::Exception: Unknown table expression identifier 'x'. (UNKNOWN_TABLE)" + )); + assert!(!is_unknown_table( + "Code: 603. DB::Exception: Cannot open file. (CANNOT_OPEN_FILE)" + )); + } + fn make_point(ts: i64, tags: &[(&str, &str)], fields: &[(&str, FieldValue)]) -> Point { let mut tag_map = BTreeMap::new(); for (k, v) in tags { diff --git a/hyperbytedb/src/application/schema_mutation_apply.rs b/hyperbytedb/src/application/schema_mutation_apply.rs index ad7145a..930a2d6 100644 --- a/hyperbytedb/src/application/schema_mutation_apply.rs +++ b/hyperbytedb/src/application/schema_mutation_apply.rs @@ -165,7 +165,7 @@ pub(crate) fn is_unknown_column(msg: &str) -> bool { msg.contains("UNKNOWN_IDENTIFIER") || msg.contains("Unknown identifier") || msg.contains("Missing columns") - || msg.contains("Code: 47") + || msg.contains("Code: 47.") } async fn apply_schema_mutation_inner( @@ -401,6 +401,23 @@ pub async fn revoke_privilege( } } +#[cfg(test)] +mod unknown_column_tests { + use super::is_unknown_column; + + /// Only code 47 is a missing column; 470-479 are other failures and a + /// replica must report them, not store the delete as a tombstone. + #[test] + fn only_code_47_is_an_unknown_column() { + assert!(is_unknown_column( + "Code: 47. DB::Exception: Unknown expression identifier `host`. (UNKNOWN_IDENTIFIER)" + )); + assert!(!is_unknown_column( + "Code: 473. DB::Exception: Possible deadlock avoided. (DEADLOCK_AVOIDED)" + )); + } +} + #[cfg(test)] mod drop_space_tests { use super::drop_space_for; From 1b80236e582062429e8dbd18c24a80da5a9c29a4 Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Fri, 20 Nov 2026 05:15:04 +0000 Subject: [PATCH 6/6] Docs name CatchUpService; tidy doc attributes The deep-dive and developer guides still pointed at adapters/cluster/sync_client.rs, which became CatchUpService in application/cluster/catch_up.rs. Also drop a repeated #[doc(hidden)] and a repeated doc line, and move apply_shard_map_op's doc comment from `mod apply;` onto the function. --- docs/deep-dive/README.md | 2 +- docs/deep-dive/deep-dive-clustering.md | 12 ++++++------ docs/deep-dive/deep-dive-self-repair.md | 2 +- docs/developer-guide/architecture.md | 2 +- docs/developer-guide/internals/core-modules.md | 2 +- docs/developer-guide/system-architecture.md | 2 +- hyperbytedb/src/adapters/chdb/native_adapter.rs | 1 - hyperbytedb/src/application/shard_scheduler.rs | 2 -- hyperbytedb/src/domain/sharding/ops/mod.rs | 2 +- 9 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/deep-dive/README.md b/docs/deep-dive/README.md index bcdcd3a..ad9d62b 100644 --- a/docs/deep-dive/README.md +++ b/docs/deep-dive/README.md @@ -8,6 +8,6 @@ Subsystem walkthroughs with pointers to source files. When a deep dive disagrees | [Read path](deep-dive-read-path.md) | TimeseriesQL → ClickHouse SQL → chDB | `timeseriesql/`, `application/query_service.rs`, `adapters/chdb/` | | [Clustering](deep-dive-clustering.md) | Replication, Raft, sync, hinted handoff, **series sharding** | `domain/cluster/`, `application/cluster/`, `application/shard_*.rs`, `adapters/cluster/` | | [Compaction](deep-dive-compaction.md) | MergeTree background merges | chDB / ClickHouse engine | -| [Self-repair](deep-dive-self-repair.md) | Peer convergence via replication and sync | `adapters/cluster/sync_client.rs`, `replication_log.rs` | +| [Self-repair](deep-dive-self-repair.md) | Peer convergence via replication and sync | `application/cluster/catch_up.rs`, `replication_log.rs` | Replication peer sends use exponential backoff starting at 1s, capped at 30s, up to `replication_max_retries` attempts. diff --git a/docs/deep-dive/deep-dive-clustering.md b/docs/deep-dive/deep-dive-clustering.md index d2c485b..2d729f5 100644 --- a/docs/deep-dive/deep-dive-clustering.md +++ b/docs/deep-dive/deep-dive-clustering.md @@ -145,10 +145,10 @@ Called during application startup when `cluster.enabled = true`. ### `run_startup_sync(config, metadata, wal, data_dir)` 1. Set node state to `Syncing`. -2. Build a `SyncClient`. +2. Build a `CatchUpService`. 3. Determine if this is a new node (no existing data) or reconnect: - - **New node:** Call `sync_client.join_and_sync()`. - - **Existing node:** Call `sync_client.reconnect_sync()`. + - **New node:** Call `catch_up.join_and_sync()`. + - **Existing node:** Call `catch_up.reconnect_sync()`. 4. Retry up to 5 times on failure. 5. On success: set state to `Active`. 6. On failure after retries: set state to `Active` with `needs_sync = true` (allows the node to operate but flags it for later re-sync). @@ -157,7 +157,7 @@ Called during application startup when `cluster.enabled = true`. ## 4. Startup Sync -**File:** `src/adapters/cluster/sync_client.rs` +**File:** `src/application/cluster/catch_up.rs` (`CatchUpService`; it reaches peers through the `PeerRpc` port, `src/adapters/cluster/http_peer_rpc.rs`) ### `join_and_sync()` -- Full join flow for new nodes @@ -178,7 +178,7 @@ Called during application startup when `cluster.enabled = true`. ## 5. Reconnect Sync -**File:** `src/adapters/cluster/sync_client.rs` +**File:** `src/application/cluster/catch_up.rs` (`CatchUpService`; it reaches peers through the `PeerRpc` port, `src/adapters/cluster/http_peer_rpc.rs`) ### `reconnect_sync()` -- For existing nodes reconnecting after downtime @@ -457,7 +457,7 @@ This ensures peers that are catching up can still read needed WAL entries. Peers stay aligned through three mechanisms: 1. **Write replication** — after local WAL append, the coordinator fans out line protocol to peers (`ReplicationPort` / `PeerClient`). -2. **Startup and reconnect sync** — `SyncClient` exchanges metadata snapshots and WAL tail entries via `/internal/sync/{manifest,metadata,wal}`. +2. **Startup and reconnect sync** — `CatchUpService` exchanges metadata snapshots and WAL tail entries via `/internal/sync/{manifest,metadata,wal}`. 3. **Local flush** — each peer replays its WAL (including replicated entries) into local MergeTree tables via `PointsSinkPort`. There is no separate file-repair loop. chDB storage on each node is derived from the shared WAL + metadata contract. diff --git a/docs/deep-dive/deep-dive-self-repair.md b/docs/deep-dive/deep-dive-self-repair.md index 605379e..3bfabc7 100644 --- a/docs/deep-dive/deep-dive-self-repair.md +++ b/docs/deep-dive/deep-dive-self-repair.md @@ -3,7 +3,7 @@ In clustered deployments, replicas converge through: 1. **Write replication** — line protocol fan-out after each local WAL append. -2. **Metadata + WAL sync** — startup and reconnect paths in `SyncClient` (`/internal/sync/manifest`, `/internal/sync/metadata`, `/internal/sync/wal`). +2. **Metadata + WAL sync** — startup and reconnect paths in `CatchUpService` (`/internal/sync/manifest`, `/internal/sync/metadata`, `/internal/sync/wal`). 3. **Local flush** — each node replays WAL entries (including replicated writes) into its own MergeTree tables. Peer ack tracking in `ReplicationLog` informs safe WAL truncation so lagging nodes can catch up. diff --git a/docs/developer-guide/architecture.md b/docs/developer-guide/architecture.md index 4a496d6..8fe4606 100644 --- a/docs/developer-guide/architecture.md +++ b/docs/developer-guide/architecture.md @@ -68,7 +68,7 @@ graph TB RocksWAL["RocksDB WAL"] RocksMeta["RocksDB Metadata"] ChDBAdapter["chDB query + native sink"] - ClusterIO["cluster/ peer_client, sync_client, raft"] + ClusterIO["cluster/ peer_client, http_peer_rpc, raft"] end InboundAdapters --> App diff --git a/docs/developer-guide/internals/core-modules.md b/docs/developer-guide/internals/core-modules.md index 0b5c4a5..e7293e8 100644 --- a/docs/developer-guide/internals/core-modules.md +++ b/docs/developer-guide/internals/core-modules.md @@ -120,7 +120,7 @@ Axum handlers, router (`AppState`), middleware, auth, internal cluster/Raft endp | File / dir | Description | |------------|-------------| | `peer_client.rs` | Outbound replication HTTP client. Implements `ReplicationPort`. | -| `sync_client.rs` | Join/reconnect sync: metadata + WAL catch-up. | +| `http_peer_rpc.rs` | `PeerRpc` over HTTP. Join/reconnect sync itself is `CatchUpService` in `application/cluster/catch_up.rs`. | | `replication_log.rs` | RocksDB-backed per-peer WAL/mutation ack tracking. | | `hinted_handoff.rs` | Per-peer queued hints for unreachable peers. | | `transport.rs` | Shared HTTP transport abstraction for cluster I/O. | diff --git a/docs/developer-guide/system-architecture.md b/docs/developer-guide/system-architecture.md index 80e7ecc..3a57ba4 100644 --- a/docs/developer-guide/system-architecture.md +++ b/docs/developer-guide/system-architecture.md @@ -152,7 +152,7 @@ src/ ├── http/ Axum handlers + internal cluster endpoints ├── chdb/ QueryPort + PointsSinkPort (native adapter) ├── wal/, metadata/, auth.rs - └── cluster/ peer_client, sync_client, replication_log, raft/ + └── cluster/ peer_client, http_peer_rpc, replication_log, raft/ ``` --- diff --git a/hyperbytedb/src/adapters/chdb/native_adapter.rs b/hyperbytedb/src/adapters/chdb/native_adapter.rs index f62d6e5..960f76e 100644 --- a/hyperbytedb/src/adapters/chdb/native_adapter.rs +++ b/hyperbytedb/src/adapters/chdb/native_adapter.rs @@ -480,7 +480,6 @@ impl ChdbNativeAdapter { /// /// Test support. #[doc(hidden)] - #[doc(hidden)] pub fn engine_probe_count(&self) -> u64 { self.engine_probes .load(std::sync::atomic::Ordering::Relaxed) diff --git a/hyperbytedb/src/application/shard_scheduler.rs b/hyperbytedb/src/application/shard_scheduler.rs index c785bc2..2c89272 100644 --- a/hyperbytedb/src/application/shard_scheduler.rs +++ b/hyperbytedb/src/application/shard_scheduler.rs @@ -766,8 +766,6 @@ impl ShardScheduler { /// The series count this scheduler holds for `(region_id, node_id)`, or /// `None` if it never recorded one. Test support for the heartbeat /// handler, which has no other externally visible effect. - /// - /// Test support. #[doc(hidden)] pub async fn recorded_heartbeat_series(&self, region_id: u64, node_id: u64) -> Option { self.heartbeats diff --git a/hyperbytedb/src/domain/sharding/ops/mod.rs b/hyperbytedb/src/domain/sharding/ops/mod.rs index 3d307b0..3c47491 100644 --- a/hyperbytedb/src/domain/sharding/ops/mod.rs +++ b/hyperbytedb/src/domain/sharding/ops/mod.rs @@ -235,9 +235,9 @@ fn sort_space_regions(space: &mut super::types::MeasurementShardSpace) { space.regions.sort_by_key(|r| r.start); } -/// Apply `op` to an in-memory map snapshot (used by Raft apply and unit tests). mod apply; +/// Apply `op` to an in-memory map snapshot (used by Raft apply and unit tests). pub fn apply_shard_map_op( map: &mut super::types::ShardMap, op: ShardMapOp,