diff --git a/hyperbytedb/src/application/schema_mutation_apply.rs b/hyperbytedb/src/application/schema_mutation_apply.rs index 930a2d6..d255e85 100644 --- a/hyperbytedb/src/application/schema_mutation_apply.rs +++ b/hyperbytedb/src/application/schema_mutation_apply.rs @@ -160,6 +160,46 @@ async fn delete_replicated_rows( .map(|_| ()) } +/// Apply a replicated `DROP SERIES` with no `FROM`: delete, from every +/// measurement of `rp`, the rows of the series `tag_condition` selects. +/// `tag_condition` is the condition over tag columns that the coordinator +/// sends; a measurement without one of its tags has no matching series and +/// is skipped. +async fn delete_series_rows_everywhere( + metadata: &dyn MetadataPort, + sink: &dyn PointsSinkPort, + wal: Option<&Arc>, + db: &str, + rp: &str, + tag_condition: &str, +) -> Result<(), HyperbytedbError> { + // The first delete waits for the WAL to flush; that covers every + // measurement, so the rest do not wait again inside the Raft apply. + let mut wal = wal; + for meas in metadata.list_measurements(db).await? { + let series = crate::domain::chdb_naming::quoted_series_table_name(db, rp, &meas); + let predicate = format!( + "`series_id` IN (SELECT `series_id` FROM {} WHERE {tag_condition})", + series.as_str() + ); + let result = crate::application::predicate_sql::delete_rows( + wal.map(|w| w.as_ref()), + sink, + db, + rp, + &meas, + &predicate, + ) + .await; + wal = None; + match result { + Err(e) if is_unknown_column(&e.to_string()) => {} + other => other?, + } + } + Ok(()) +} + /// A chDB error for a column the table does not have. pub(crate) fn is_unknown_column(msg: &str) -> bool { msg.contains("UNKNOWN_IDENTIFIER") @@ -286,10 +326,20 @@ async fn apply_schema_mutation_inner( measurement, predicate_sql, } => { + let sink = points_sink.or_else(|| mv_service.map(|mv| mv.points_sink())); if let Some(meas) = measurement.as_deref() { - let sink = points_sink.or_else(|| mv_service.map(|mv| mv.points_sink())); delete_replicated_rows(metadata, sink, wal, &database, &rp, meas, &predicate_sql) .await?; + } else if let Some(sink) = sink.filter(|_| !predicate_sql.is_empty()) { + delete_series_rows_everywhere( + metadata.as_ref(), + sink.as_ref(), + wal, + &database, + &rp, + &predicate_sql, + ) + .await?; } metadata .delete_series_matching(&database, &rp, measurement.as_deref(), &predicate_sql) diff --git a/hyperbytedb/tests/compat/cluster_replication_tests.rs b/hyperbytedb/tests/compat/cluster_replication_tests.rs index 7432ce2..ae21fba 100644 --- a/hyperbytedb/tests/compat/cluster_replication_tests.rs +++ b/hyperbytedb/tests/compat/cluster_replication_tests.rs @@ -169,3 +169,173 @@ async fn every_cluster_mutation_replicates_as_before() { std::fs::read_to_string(&fixture).expect("fixture missing; run with UPDATE_GOLDEN=1"); assert_eq!(got, want, "replicated mutations changed"); } + +/// Number of rows `q` returns on `ctx`. +async fn row_count(ctx: &TestContext, q: &str) -> usize { + let resp = ctx.query("testdb", q).await.unwrap(); + let stmt = &resp.results[0]; + assert!(stmt.error.is_none(), "{q}: {:?}", stmt.error); + stmt.series.iter().flatten().map(|s| s.values.len()).sum() +} + +/// A replica applying a replicated DROP SERIES with no FROM deletes the +/// matching rows of every measurement that has the tag, as the +/// coordinator does on its own node. +#[tokio::test] +#[serial(chdb)] +async fn a_replica_applies_drop_series_without_from_to_its_rows() { + let Ok(replica) = TestContext::new() else { + eprintln!("skipping: chDB not available"); + return; + }; + let rows = "cpu,host=a value=1.0 1000000000\ncpu,host=b value=2.0 2000000000\n\ + mem,host=a value=3.0 1000000000\ndisk,dev=x value=4.0 1000000000"; + replica.metadata.create_database("testdb").await.unwrap(); + replica.write_and_flush("testdb", rows).await.unwrap(); + + // The coordinator: build the mutation it replicates. + let coord = TestContext::new_no_chdb().unwrap(); + coord.metadata.create_database("testdb").await.unwrap(); + coord.write_and_flush("testdb", rows).await.unwrap(); + let replication = Arc::new(RecordingReplication::default()); + let peer = PeerQueryService::new( + coord.query_service.clone(), + coord.metadata.clone(), + replication.clone(), + ); + let resp = peer + .execute_query( + "testdb", + r#"DROP SERIES WHERE "host" = 'a'"#, + None, + None, + None, + ) + .await + .unwrap(); + assert!( + resp.results[0].error.is_none(), + "{:?}", + resp.results[0].error + ); + let mutation = replication + .seen + .lock() + .pop() + .expect("DROP SERIES replicates"); + + hyperbytedb::application::schema_mutation_apply::apply_schema_mutation( + hyperbytedb::application::schema_mutation_apply::SchemaMutationDeps { + metadata: &replica.metadata, + mv_service: None, + points_sink: Some(&replica.points_sink), + wal: Some(&replica.wal), + shard_routing: None, + runtime_settings: None, + }, + mutation, + ) + .await + .unwrap(); + + assert_eq!(row_count(&replica, "SELECT value FROM cpu").await, 1); + assert_eq!(row_count(&replica, "SELECT value FROM mem").await, 0); + assert_eq!(row_count(&replica, "SELECT value FROM disk").await, 1); +} + +/// A WAL that counts how often a delete asks where it ends (once per flush +/// wait) and otherwise passes through. +struct CountingWal { + inner: Arc, + last_sequence_calls: std::sync::atomic::AtomicUsize, +} + +#[async_trait] +impl hyperbytedb::ports::wal::WalPort for CountingWal { + async fn append( + &self, + entry: hyperbytedb::domain::wal::WalEntry, + ) -> Result { + self.inner.append(entry).await + } + async fn read_from( + &self, + sequence: u64, + ) -> Result, HyperbytedbError> { + self.inner.read_from(sequence).await + } + async fn read_range( + &self, + from: u64, + max_entries: usize, + ) -> Result, HyperbytedbError> { + self.inner.read_range(from, max_entries).await + } + async fn truncate_before(&self, sequence: u64) -> Result<(), HyperbytedbError> { + self.inner.truncate_before(sequence).await + } + async fn last_sequence(&self) -> Result { + self.last_sequence_calls + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.inner.last_sequence().await + } + fn flushed_through(&self) -> u64 { + self.inner.flushed_through() + } + fn set_flushed_through(&self, sequence: u64) { + self.inner.set_flushed_through(sequence) + } + fn flushed_margin_start(&self, through: u64, max_bytes: u64) -> u64 { + self.inner.flushed_margin_start(through, max_bytes) + } +} + +/// The replicated DROP SERIES with no FROM runs inside the Raft apply: it +/// waits for the WAL to flush once, not once per measurement. +#[tokio::test] +#[serial(chdb)] +async fn a_replica_drop_series_without_from_waits_for_the_flush_once() { + let Ok(replica) = TestContext::new() else { + eprintln!("skipping: chDB not available"); + return; + }; + replica.metadata.create_database("testdb").await.unwrap(); + replica + .write_and_flush( + "testdb", + "cpu,host=a value=1.0 1000000000\nmem,host=a value=3.0 1000000000\n\ + disk,host=a value=4.0 1000000000", + ) + .await + .unwrap(); + let counting = Arc::new(CountingWal { + inner: replica.wal.clone(), + last_sequence_calls: Default::default(), + }); + let wal: Arc = counting.clone(); + + hyperbytedb::application::schema_mutation_apply::apply_schema_mutation( + hyperbytedb::application::schema_mutation_apply::SchemaMutationDeps { + metadata: &replica.metadata, + mv_service: None, + points_sink: Some(&replica.points_sink), + wal: Some(&wal), + shard_routing: None, + runtime_settings: None, + }, + MutationRequest::DropSeries { + database: "testdb".into(), + rp: "autogen".into(), + measurement: None, + predicate_sql: "`host` = 'a'".into(), + }, + ) + .await + .unwrap(); + + let calls = counting + .last_sequence_calls + .load(std::sync::atomic::Ordering::Relaxed); + assert_eq!(calls, 1, "flush waits"); + assert_eq!(row_count(&replica, "SELECT value FROM cpu").await, 0); +} diff --git a/hyperbytedb/tests/compat/main.rs b/hyperbytedb/tests/compat/main.rs index 5a9a265..19b0830 100644 --- a/hyperbytedb/tests/compat/main.rs +++ b/hyperbytedb/tests/compat/main.rs @@ -90,6 +90,7 @@ pub struct TestContext { pub query_service: Arc, pub ingestion: IngestionServiceImpl, pub flush_service: Arc, + pub points_sink: Arc, _tmpdir: tempfile::TempDir, } @@ -131,7 +132,7 @@ impl TestContext { let ingestion = IngestionServiceImpl::new(wal.clone(), metadata.clone(), 100_000, 10_000, 0); - let flush_service = Arc::new(FlushServiceImpl::new(wal.clone(), 0, points_sink)); + let flush_service = Arc::new(FlushServiceImpl::new(wal.clone(), 0, points_sink.clone())); Ok(Self { wal, @@ -140,6 +141,7 @@ impl TestContext { query_service, ingestion, flush_service, + points_sink, _tmpdir: tmpdir, }) } @@ -174,7 +176,7 @@ impl TestContext { let ingestion = IngestionServiceImpl::new(wal.clone(), metadata.clone(), 100_000, 10_000, 0); - let flush_service = Arc::new(FlushServiceImpl::new(wal.clone(), 0, points_sink)); + let flush_service = Arc::new(FlushServiceImpl::new(wal.clone(), 0, points_sink.clone())); Ok(Self { wal, @@ -183,6 +185,7 @@ impl TestContext { query_service, ingestion, flush_service, + points_sink, _tmpdir: tmpdir, }) }