diff --git a/hyperbytedb/src/application/peer_query_service.rs b/hyperbytedb/src/application/peer_query_service.rs index 545c7d5..34016f4 100644 --- a/hyperbytedb/src/application/peer_query_service.rs +++ b/hyperbytedb/src/application/peer_query_service.rs @@ -432,6 +432,30 @@ async fn mutation_request_from_statement( } } +/// The database a cluster statement runs against locally: a retention +/// policy statement's own `ON `, otherwise the request's. +fn local_db<'a>(stmt: &'a Statement, db: &'a str) -> &'a str { + match stmt { + Statement::CreateRetentionPolicyStmt { db, .. } + | Statement::DropRetentionPolicyStmt { db, .. } + | Statement::AlterRetentionPolicyStmt { db, .. } => db, + _ => db, + } +} + +/// Whether failing to build `stmt`'s mutation (after it applied locally) is +/// reported on the statement rather than failing the whole request: a bad +/// continuous query or view definition, or a password that cannot be hashed. +fn build_error_is_statement_error(stmt: &Statement) -> bool { + matches!( + stmt, + Statement::CreateContinuousQuery(_) + | Statement::CreateMaterializedView(_) + | Statement::CreateUser { .. } + | Statement::SetPassword { .. } + ) +} + fn is_cluster_mutation(stmt: &Statement) -> bool { matches!( stmt, @@ -493,47 +517,10 @@ impl QueryService for PeerQueryService { continue; } let result = match stmt { - Statement::CreateDatabase(ref stmt) => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::CreateDatabase { - name: stmt.name.clone(), - rp: retention_policy_from_create(stmt), - }) - .await?; - } - result - } - Statement::DropDatabase(ref name) => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::DropDatabase(name.clone())) - .await?; - } - result - } - Statement::Delete(_) => { - let result = self + ref stmt if is_cluster_mutation(stmt) => { + let mut result = self .execute_inner_statement( - db, + local_db(stmt, db), stmt_query, epoch, retention_policy, @@ -542,445 +529,26 @@ impl QueryService for PeerQueryService { ) .await; if result.error.is_none() { - let req = mutation_request_from_statement( - &stmt, + match mutation_request_from_statement( + stmt, db, retention_policy, &self.metadata, self.shard_routing.get(), ) - .await?; - self.replicate_mutation(req).await?; - } - result - } - Statement::CreateContinuousQuery(ref cq) => { - let mut result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let def = match ContinuousQueryDef::from_create(cq) { - Ok(def) => def, - Err(e) => { - result.error = Some(e.to_string()); - results.push(result); - continue; - } - }; - self.replicate_mutation(MutationRequest::CreateContinuousQuery { - database: cq.database.clone(), - name: cq.name.clone(), - definition: def, - }) - .await?; - } - result - } - Statement::DropContinuousQuery { - ref name, - db: ref cq_db, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let target_db = if cq_db.is_empty() { db } else { cq_db }; - self.replicate_mutation(MutationRequest::DropContinuousQuery { - database: target_db.to_string(), - name: name.clone(), - }) - .await?; - } - result - } - Statement::CreateMaterializedView(ref mv) => { - let mut result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let source_rp = match resolve_mv_source_rp(&self.metadata, mv).await { - Ok(rp) => rp, - Err(e) => { + .await + { + Ok(req) => self.replicate_mutation(req).await?, + // Applied here but not replicable: the statement + // reports it, the other statements still run. + Err(e) if build_error_is_statement_error(stmt) => { result.error = Some(e.to_string()); - results.push(result); - continue; - } - }; - let dest_rp = match resolve_mv_dest_rp(&self.metadata, mv).await { - Ok(rp) => rp, - Err(e) => { - result.error = Some(e.to_string()); - results.push(result); - continue; - } - }; - if let Ok(def) = def_from_statement(mv, &source_rp, &dest_rp) { - let source_meta = - mv_source_meta(&self.metadata, self.shard_routing.get(), &def) - .await?; - self.replicate_mutation(MutationRequest::CreateMaterializedView { - database: mv.database.clone(), - name: mv.name.clone(), - definition: def, - source_meta, - }) - .await?; - } - } - result - } - Statement::DropMaterializedView { - ref name, - db: ref mv_db, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let target_db = if mv_db.is_empty() { db } else { mv_db }; - self.replicate_mutation(MutationRequest::DropMaterializedView { - database: target_db.to_string(), - name: name.clone(), - }) - .await?; - } - result - } - Statement::CreateRetentionPolicyStmt { - ref name, - ref db, - ref duration, - replication, - ref shard_duration, - is_default, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let dur = duration - .as_ref() - .map(|d| std::time::Duration::from_nanos(d.to_nanos() as u64)); - let shard_dur = shard_duration - .as_ref() - .map(|d| std::time::Duration::from_nanos(d.to_nanos() as u64)) - .unwrap_or_else(|| { - crate::domain::database::derive_shard_group_duration(dur) - }); - let rp = RetentionPolicy { - name: name.clone(), - duration: dur, - shard_group_duration: shard_dur, - replication_factor: replication, - is_default, - }; - self.replicate_mutation(MutationRequest::CreateRetentionPolicy { - db: db.clone(), - rp, - }) - .await?; - } - result - } - Statement::DropRetentionPolicyStmt { ref name, ref db } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::DropRetentionPolicy { - db: db.clone(), - name: name.clone(), - }) - .await?; - } - result - } - Statement::CreateUser { - ref username, - ref password, - admin, - } => { - let mut result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - match hash_password_for_replication(password) { - Ok(password_hash) => { - self.replicate_mutation(MutationRequest::CreateUser { - username: username.clone(), - password_hash, - admin, - }) - .await?; } - Err(e) => result.error = Some(e.to_string()), + Err(e) => return Err(e), } } result } - Statement::DropUser(ref username) => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::DropUser(username.clone())) - .await?; - } - result - } - Statement::SetPassword { - ref username, - ref password, - } => { - let mut result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - match hash_password_for_replication(password) { - Ok(password_hash) => { - self.replicate_mutation(MutationRequest::SetPassword { - username: username.clone(), - password_hash, - }) - .await?; - } - Err(e) => result.error = Some(e.to_string()), - } - } - result - } - Statement::AlterRetentionPolicyStmt { - ref name, - ref db, - duration, - replication, - ref shard_duration, - is_default, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let change = RetentionPolicyChange { - duration: duration.as_ref().map(|d| { - if d.value == 0 && d.unit == DurationUnit::Second { - None - } else { - Some(d.clone()) - } - }), - replication, - shard_duration: shard_duration.clone(), - is_default, - }; - self.replicate_mutation(MutationRequest::AlterRetentionPolicy { - db: db.clone(), - name: name.clone(), - change, - }) - .await?; - } - result - } - Statement::DropSeries(_) => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let req = mutation_request_from_statement( - &stmt, - db, - retention_policy, - &self.metadata, - self.shard_routing.get(), - ) - .await?; - self.replicate_mutation(req).await?; - } - result - } - Statement::DropMeasurement { - ref name, - rp: ref stmt_rp, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - let dm_rp = if let Some(rp) = stmt_rp { - rp.clone() - } else { - self.metadata.get_default_rp(db).await? - }; - self.replicate_mutation(MutationRequest::DropMeasurement { - database: db.to_string(), - rp: dm_rp, - name: name.clone(), - }) - .await?; - } - result - } - Statement::Grant { - ref username, - ref database, - privilege, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::Grant { - username: username.clone(), - database: database.clone(), - privilege, - }) - .await?; - } - result - } - Statement::Revoke { - ref username, - ref database, - privilege, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::Revoke { - username: username.clone(), - database: database.clone(), - privilege, - }) - .await?; - } - result - } - Statement::SetSetting { - ref name, - ref value, - } => { - let result = self - .execute_inner_statement( - db, - stmt_query, - epoch, - retention_policy, - caller, - statement_id, - ) - .await; - if result.error.is_none() { - self.replicate_mutation(MutationRequest::SetSetting { - name: name.clone(), - value: crate::application::runtime_settings::validate( - name, - value.clone(), - )?, - }) - .await?; - } - result - } _ => { self.execute_inner_statement( db, diff --git a/hyperbytedb/src/application/query_service.rs b/hyperbytedb/src/application/query_service/mod.rs similarity index 70% rename from hyperbytedb/src/application/query_service.rs rename to hyperbytedb/src/application/query_service/mod.rs index 3a5d3f3..1056c9f 100644 --- a/hyperbytedb/src/application/query_service.rs +++ b/hyperbytedb/src/application/query_service/mod.rs @@ -1,3 +1,5 @@ +mod statements; + use async_trait::async_trait; use std::collections::HashMap; use std::sync::Arc; @@ -738,374 +740,19 @@ async fn execute_statement( match stmt { Statement::ShowDatabases => { - let dbs = svc.metadata.list_databases().await?; - let columns = vec!["name".to_string()]; - let values: Vec> = dbs - .iter() - .map(|d| vec![serde_json::Value::String(d.name.clone())]) - .collect(); - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "databases".to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_catalog::show_databases(svc, statement_id).await } Statement::ShowMeasurements(s) => { - let db = s.database.as_deref().unwrap_or(db); - if db.is_empty() { - return Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: Some("database is required".to_string()), - }); - } - let rp = resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp) - .await?; - let names = list_measurements_for_rp(svc, db, &rp).await?; - let filtered = filter_names_by_selector(&names, s.measurement_filter.as_ref()); - let columns = vec!["name".to_string()]; - let values: Vec> = filtered - .iter() - .map(|n| vec![serde_json::Value::String(n.clone())]) - .collect(); - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "measurements".to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_schema::show_measurements(svc, db, s, query_rp, statement_id).await } Statement::ShowTagKeys(s) => { - let db = s.database.as_deref().unwrap_or(db); - let measurement = s.from.as_ref().and_then(|m| m.name_str()); - let default_rp = - resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp) - .await?; - let keys = if let (Some(ctx), Some(m), Some(name)) = ( - svc.shard_routing.as_ref(), - s.from.as_ref(), - s.from.as_ref().and_then(|m| m.name_str()), - ) { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = if m.retention_policy.is_some() { - resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await? - } else { - default_rp.clone() - }; - scatter_tag_keys(ctx, svc.metadata.as_ref(), query_db, &rp, name).await? - } else if let Some(m) = s.from.as_ref() { - if let Some(name) = m.name_str() { - if m.retention_policy.is_some() { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await?; - tag_keys_for_measurement(svc, query_db, &rp, name).await? - } else { - svc.metadata - .list_tag_keys(db, &default_rp, Some(name)) - .await? - } - } else { - svc.metadata - .list_tag_keys(db, &default_rp, measurement) - .await? - } - } else { - svc.metadata - .list_tag_keys(db, &default_rp, measurement) - .await? - }; - let columns = vec!["tagKey".to_string()]; - let values: Vec> = keys - .iter() - .map(|k| vec![serde_json::Value::String(k.clone())]) - .collect(); - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: measurement.unwrap_or("").to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_schema::show_tag_keys(svc, db, s, query_rp, statement_id).await } Statement::ShowTagValues(s) => { - let db = s.database.as_deref().unwrap_or(db); - let measurement = s.from.as_ref().and_then(|m| m.name_str()); - let default_rp = - resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp) - .await?; - - let all_tag_keys = if let (Some(ctx), Some(m), Some(name)) = - (svc.shard_routing.as_ref(), s.from.as_ref(), measurement) - { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = if m.retention_policy.is_some() { - resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await? - } else { - default_rp.clone() - }; - scatter_tag_keys(ctx, svc.metadata.as_ref(), query_db, &rp, name).await? - } else if let (Some(m), Some(name)) = (s.from.as_ref(), measurement) { - if m.retention_policy.is_some() { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await?; - tag_keys_for_measurement(svc, query_db, &rp, name).await? - } else { - svc.metadata - .list_tag_keys(db, &default_rp, Some(name)) - .await? - } - } else { - svc.metadata - .list_tag_keys(db, &default_rp, measurement) - .await? - }; - - let matching_keys: Vec = - filter_names_by_selector(&all_tag_keys, Some(&s.tag_key)); - - // With a WHERE: the values of the series that match it. It used - // to be ignored, so a Grafana variable chained on another (the - // CPUs of the chosen host) listed every value. - if let Some(cond) = &s.condition { - let predicate = crate::application::tag_condition::TagPredicate::compile(cond)?; - let measurements: Vec = match measurement { - Some(name) => vec![name.to_string()], - None => svc.metadata.list_measurements(db).await?, - }; - let (query_db, rp) = match s.from.as_ref() { - Some(m) if m.retention_policy.is_some() => { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await?; - (query_db, rp) - } - Some(m) => (m.database.as_deref().unwrap_or(db), default_rp.clone()), - None => (db, default_rp.clone()), - }; - let mut pairs = std::collections::BTreeSet::new(); - for name in &measurements { - let series = match svc.shard_routing.as_ref() { - Some(ctx) => { - crate::application::shard_metadata_scatter::scatter_series_tags( - ctx, - svc.metadata.as_ref(), - query_db, - &rp, - name, - ) - .await? - } - None => svc - .metadata - .list_series(query_db, &rp, name) - .await? - .into_iter() - .map(|(_, tags)| tags) - .collect(), - }; - for tags in series.iter().filter(|t| predicate.matches(t)) { - for key in &matching_keys { - if let Some(v) = tags.get(key) { - pairs.insert((key.clone(), v.clone())); - } - } - } - } - let values = limit_offset( - pairs - .into_iter() - .map(|(k, v)| { - vec![serde_json::Value::String(k), serde_json::Value::String(v)] - }) - .collect(), - s.limit, - s.offset, - ); - return Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: measurement.unwrap_or("").to_string(), - tags: None, - columns: vec!["key".to_string(), "value".to_string()], - values, - partial: None, - }]), - error: None, - }); - } - let default_rp_for_values = default_rp.clone(); - - let key_value_futures: Vec<_> = matching_keys - .iter() - .map(|tag_key| { - let tag_key = tag_key.clone(); - let default_rp = default_rp_for_values.clone(); - async move { - let values_list = if let (Some(ctx), Some(m), Some(name)) = - (svc.shard_routing.as_ref(), s.from.as_ref(), measurement) - { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = if m.retention_policy.is_some() { - resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await? - } else { - default_rp.clone() - }; - scatter_tag_values( - ctx, - svc.metadata.as_ref(), - query_db, - &rp, - name, - &tag_key, - ) - .await? - } else if let (Some(m), Some(name)) = (s.from.as_ref(), measurement) { - if m.retention_policy.is_some() { - let query_db = m.database.as_deref().unwrap_or(db); - let rp = resolve_retention_policy( - svc.metadata.as_ref(), - query_db, - m.retention_policy.as_deref(), - ) - .await?; - tag_values_for_measurement(svc, query_db, &rp, name, &tag_key) - .await? - } else { - svc.metadata - .list_tag_values(db, &default_rp, &tag_key, Some(name)) - .await? - } - } else { - svc.metadata - .list_tag_values(db, &default_rp, &tag_key, measurement) - .await? - }; - Ok::<_, HyperbytedbError>((tag_key, values_list)) - } - }) - .collect(); - - let mut all_values = Vec::new(); - for (tag_key, values_list) in futures::future::try_join_all(key_value_futures).await? { - for v in values_list { - all_values.push(vec![ - serde_json::Value::String(tag_key.clone()), - serde_json::Value::String(v), - ]); - } - } - - let columns = vec!["key".to_string(), "value".to_string()]; - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: measurement.unwrap_or("").to_string(), - tags: None, - columns, - values: limit_offset(all_values, s.limit, s.offset), - partial: None, - }]), - error: None, - }) + statements::show_schema::show_tag_values(svc, db, s, query_rp, statement_id).await } Statement::ShowFieldKeys(s) => { - let db = s.database.as_deref().unwrap_or(db); - let measurement = s.from.as_ref().and_then(|m| m.name_str()); - if db.is_empty() { - return Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: Some("database is required".to_string()), - }); - } - let rp = resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp) - .await?; - let mut field_values = Vec::new(); - let measurements: Vec = if let Some(m) = measurement { - vec![m.to_string()] - } else { - svc.metadata.list_measurements(db).await? - }; - for m in &measurements { - let fields = match svc.shard_routing.as_ref() { - Some(ctx) => { - crate::application::shard_metadata_scatter::scatter_field_keys( - ctx, - svc.metadata.as_ref(), - db, - &rp, - m, - ) - .await? - } - None => crate::application::shard_metadata_scatter::local_field_keys( - svc.metadata.get_measurement(db, &rp, m).await?.as_ref(), - ), - }; - for (name, typ) in fields { - field_values.push(vec![ - serde_json::Value::String(name), - serde_json::Value::String(typ), - ]); - } - } - let columns = vec!["fieldKey".to_string(), "fieldType".to_string()]; - let values: Vec> = field_values; - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: measurement.unwrap_or("").to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_schema::show_field_keys(svc, db, s, query_rp, statement_id).await } Statement::CreateDatabase(stmt) => { svc.metadata.create_database_with(stmt).await?; @@ -1116,541 +763,52 @@ async fn execute_statement( }) } Statement::DropDatabase(name) => { - crate::application::database_drop::drop_database( - &svc.metadata, - Some(&svc.mv_service), - Some(&svc.points_sink), - Some(&svc.wal), - name, - ) - .await?; - svc.drop_shard_spaces(name, None, None).await; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::drop_database(svc, name, statement_id).await } Statement::Select(select_stmt) => { - if db.is_empty() { - return Ok(StatementResult { - statement_id, - series: None, - error: Some("database is required".to_string()), - }); - } - - if select_stmt.into.is_some() { - let count = execute_select_into(svc, db, select_stmt, epoch).await?; - return Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "result".to_string(), - tags: None, - columns: vec!["time".to_string(), "written".to_string()], - values: vec![vec![ - serde_json::Value::String(chrono::Utc::now().to_rfc3339()), - serde_json::json!(count), - ]], - partial: None, - }]), - error: None, - }); - } - - let group_by_tags: Vec = select_stmt - .group_by - .as_ref() - .map(|gb| { - gb.tag_dimensions() - .into_iter() - .map(|s| s.to_string()) - .collect() - }) - .unwrap_or_default(); - - let (time_min, time_max) = - to_clickhouse::extract_time_bounds(select_stmt.condition.as_ref()); - - if select_stmt.from.is_empty() { - return Err(HyperbytedbError::QueryParse( - "SELECT requires FROM clause".to_string(), - )); - } - - let mut all_series = Vec::new(); - for source in &select_stmt.from { - let mut series = execute_select_from_source( - svc, - db, - select_stmt, - source, - time_min, - time_max, - epoch, - &group_by_tags, - query_rp, - ) - .await?; - all_series.append(&mut series); - } - - stamp_aggregate_time(select_stmt, &mut all_series, epoch, time_min); - order_columns_like_select(select_stmt, &mut all_series); - - // Apply SLIMIT/SOFFSET (series-level pagination) - if select_stmt.slimit.is_some() || select_stmt.soffset.is_some() { - let soffset = select_stmt.soffset.unwrap_or(0) as usize; - let slimit = select_stmt.slimit.unwrap_or(u64::MAX) as usize; - let len = all_series.len(); - let start = soffset.min(len); - let end = (start + slimit).min(len); - all_series = all_series[start..end].to_vec(); - } - - Ok(StatementResult { - statement_id, - series: Some(all_series), - error: None, - }) + statements::select::select(svc, db, select_stmt, epoch, query_rp, statement_id).await } Statement::ShowCardinality(s) => { execute_show_cardinality(svc, db, s, query_rp, statement_id).await } Statement::ShowSeries(s) => { - use crate::domain::series::SeriesKey; - - let query_db = s.database.as_deref().unwrap_or(db); - if query_db.is_empty() { - return Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: Some("database is required".to_string()), - }); - } - - let measurements: Vec = if let Some(ref from) = s.from { - if let Some(name) = from.name_str() { - vec![name.to_string()] - } else { - svc.metadata.list_measurements(query_db).await? - } - } else { - svc.metadata.list_measurements(query_db).await? - }; - - let rp = if let Some(ref from) = s.from - && let Some(rp_name) = from.retention_policy.as_ref() - { - resolve_retention_policy(svc.metadata.as_ref(), query_db, Some(rp_name)).await? - } else { - resolve_retention_policy_for_select(svc.metadata.as_ref(), query_db, None, query_rp) - .await? - }; - - let mut values = Vec::new(); - for meas in &measurements { - if let Some(ctx) = svc.shard_routing.as_ref() { - for key in - scatter_series_keys(ctx, svc.metadata.as_ref(), query_db, &rp, meas).await? - { - values.push(vec![serde_json::Value::String(key)]); - } - } else { - let series = svc.metadata.list_series(query_db, &rp, meas).await?; - for (_, tags) in series { - let key = SeriesKey::new(meas, &tags); - values.push(vec![serde_json::Value::String(key.to_canonical())]); - } - } - } - - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: String::new(), - tags: None, - columns: vec!["key".to_string()], - values, - partial: None, - }]), - error: None, - }) + statements::show_schema::show_series(svc, db, s, query_rp, statement_id).await } Statement::DropMeasurement { name, rp: stmt_rp } => { - if let Err(e) = svc.mv_service.drop_for_source_measurement(db, name).await { - tracing::warn!( - db = db, - measurement = name, - error = %e, - "failed to cascade-drop materialized views for source measurement" - ); - } - let rp = if let Some(inner) = stmt_rp { - inner.clone() - } else { - svc.metadata.get_default_rp(db).await? - }; - svc.metadata.delete_measurement(db, &rp, name).await?; - if let Some(ctx) = svc.shard_routing.as_ref() { - scatter_delete_to_regions(ctx, svc.metadata.as_ref(), None, db, &rp, name, "") - .await?; - } - if let Err(e) = svc.points_sink.drop_measurement(db, &rp, name).await { - tracing::warn!( - db = db, - measurement = name, - error = %e, - "failed to drop chDB native table for measurement (metadata already cleared)" - ); - } - svc.drop_shard_spaces(db, Some(&rp), Some(name)).await; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::drop_measurement(svc, db, name, stmt_rp, statement_id).await } Statement::Delete(del) => { - let del_rp = if let Some(rp) = query_rp { - rp.to_string() - } else { - svc.metadata - .get_default_rp(db) - .await - .unwrap_or_else(|_| "autogen".to_string()) - }; - // The rows are deleted, as in InfluxDB: points written after the - // DELETE stay. It used to store a tombstone that every later query - // applied, hiding later points too. - let predicate_sql = crate::application::predicate_sql::build_delete_predicate( - &svc.metadata, - db, - &del_rp, - &del.from, - del.condition.as_ref(), - ) - .await?; - - if let Some(ctx) = svc.shard_routing.as_ref() { - scatter_delete_to_regions( - ctx, - svc.metadata.as_ref(), - Some(crate::application::predicate_sql::LocalRows { - wal: Some(svc.wal.as_ref()), - sink: svc.points_sink.as_ref(), - }), - db, - &del_rp, - &del.from, - &predicate_sql, - ) - .await?; - } else { - crate::application::predicate_sql::delete_rows( - Some(svc.wal.as_ref()), - svc.points_sink.as_ref(), - db, - &del_rp, - &del.from, - &predicate_sql, - ) - .await?; - } - - tracing::debug!( - db = db, - measurement = %del.from, - predicate = %predicate_sql, - "DELETE applied" - ); - - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::delete::delete(svc, db, del, query_rp, statement_id).await } Statement::CreateContinuousQuery(cq) => { - svc.metadata - .get_database(&cq.database) - .await? - .ok_or_else(|| HyperbytedbError::DatabaseNotFound(cq.database.clone()))?; - - let def = match ContinuousQueryDef::from_create(cq) { - Ok(def) => def, - Err(e) => { - return Ok(StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }); - } - }; - - svc.metadata - .store_continuous_query(&def.database, &def.name, &def) - .await?; - - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::create_continuous_query(svc, cq, statement_id).await } Statement::ShowContinuousQueries => { - let dbs = svc.metadata.list_databases().await?; - let mut all_series = Vec::new(); - for db_entry in &dbs { - let cqs = svc.metadata.list_continuous_queries(&db_entry.name).await?; - if cqs.is_empty() { - continue; - } - let columns = vec!["name".to_string(), "query".to_string()]; - let values: Vec> = cqs - .iter() - .map(|cq| { - vec![ - serde_json::Value::String(cq.name.clone()), - serde_json::Value::String(reconstruct_cq_text(cq)), - ] - }) - .collect(); - all_series.push(SeriesResult { - name: db_entry.name.clone(), - tags: None, - columns, - values, - partial: None, - }); - } - - Ok(StatementResult { - statement_id, - series: Some(all_series), - error: None, - }) + statements::show_catalog::show_continuous_queries(svc, statement_id).await } Statement::DropContinuousQuery { name, db: cq_db } => { - let target_db = if cq_db.is_empty() { db } else { cq_db }; - svc.metadata - .get_database(target_db) - .await? - .ok_or_else(|| HyperbytedbError::DatabaseNotFound(target_db.to_string()))?; - svc.metadata.drop_continuous_query(target_db, name).await?; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::drop_continuous_query(svc, db, name, cq_db, statement_id).await } Statement::CreateMaterializedView(mv) => { - svc.metadata - .get_database(&mv.database) - .await? - .ok_or_else(|| HyperbytedbError::DatabaseNotFound(mv.database.clone()))?; - svc.mv_service.create(mv).await?; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::create_materialized_view(svc, mv, statement_id).await } Statement::ShowMaterializedViews => { - let dbs = svc.metadata.list_databases().await?; - let mut all_mvs = Vec::new(); - for db_entry in &dbs { - let mvs = svc.metadata.list_materialized_views(&db_entry.name).await?; - all_mvs.extend(mvs); - } - - let columns = vec![ - "name".to_string(), - "database".to_string(), - "query".to_string(), - "source_measurement".to_string(), - "dest_measurement".to_string(), - ]; - let values: Vec> = all_mvs - .iter() - .map(|mv| { - vec![ - serde_json::Value::String(mv.name.clone()), - serde_json::Value::String(mv.database.clone()), - serde_json::Value::String(mv.query_text.clone()), - serde_json::Value::String(mv.source_measurement.clone()), - serde_json::Value::String(mv.dest_measurement.clone()), - ] - }) - .collect(); - - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "materialized_views".to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_catalog::show_materialized_views(svc, statement_id).await } Statement::SetSetting { name, value } => { - // A single node; a cluster replicates the SET through Raft and - // every node applies it the same way. - let value = crate::application::runtime_settings::validate(name, value.clone())?; - svc.settings()? - .apply(svc.metadata.as_ref(), name, value) - .await?; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) - } - Statement::ShowSettings => { - let rows = svc.settings()?.show(svc.metadata.as_ref()).await?; - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "settings".to_string(), - tags: None, - columns: vec![ - "name".into(), - "value".into(), - "source".into(), - "config".into(), - "description".into(), - ], - values: rows - .into_iter() - .map(|r| { - vec![ - serde_json::Value::String(r.name.to_string()), - r.value - .as_ref() - .map_or(serde_json::Value::Null, |v| v.to_json()), - r.source.map_or(serde_json::Value::Null, |s| { - serde_json::Value::String(s.as_str().to_string()) - }), - serde_json::Value::String(r.config.to_string()), - serde_json::Value::String(r.description.to_string()), - ] - }) - .collect(), - partial: None, - }]), - error: None, - }) + statements::ddl::set_setting(svc, name, value, statement_id).await } + Statement::ShowSettings => statements::show_catalog::show_settings(svc, statement_id).await, Statement::DropMaterializedView { name, db: mv_db } => { - let target_db = if mv_db.is_empty() { db } else { mv_db }; - svc.metadata - .get_database(target_db) - .await? - .ok_or_else(|| HyperbytedbError::DatabaseNotFound(target_db.to_string()))?; - svc.mv_service.drop_mv(target_db, name).await?; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::drop_materialized_view(svc, db, name, mv_db, statement_id).await } Statement::ShowRetentionPolicies(rp_db) => { - let target_db = if rp_db.is_empty() { db } else { rp_db }; - if !rp_db.is_empty() { - svc.metadata - .get_database(target_db) - .await? - .ok_or_else(|| HyperbytedbError::DatabaseNotFound(target_db.to_string()))?; - } - let rps = svc.metadata.list_retention_policies(target_db).await?; - let columns = vec![ - "name".to_string(), - "duration".to_string(), - "shardGroupDuration".to_string(), - "replicaN".to_string(), - "default".to_string(), - ]; - let values: Vec> = rps - .iter() - .map(|rp| { - let dur_str = crate::domain::database::format_influx_duration(rp.duration); - let sgd_str = crate::domain::database::format_influx_duration(Some( - rp.shard_group_duration, - )); - vec![ - serde_json::Value::String(rp.name.clone()), - serde_json::Value::String(dur_str), - serde_json::Value::String(sgd_str), - serde_json::json!(rp.replication_factor), - serde_json::Value::Bool(rp.is_default), - ] - }) - .collect(); - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "".to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_catalog::show_retention_policies(svc, db, rp_db, statement_id).await } Statement::ShowGrants(username) => { - let user = svc.metadata.get_user(username).await?.ok_or_else(|| { - HyperbytedbError::QueryParse(format!("user '{username}' does not exist")) - })?; - let mut grants: Vec<_> = user.privileges.into_iter().collect(); - grants.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: String::new(), - tags: None, - columns: vec!["database".to_string(), "privilege".to_string()], - values: grants - .into_iter() - .map(|(db, p)| { - vec![ - serde_json::Value::String(db), - serde_json::Value::String(p.as_str().to_string()), - ] - }) - .collect(), - partial: None, - }]), - error: None, - }) - } - Statement::ShowUsers => { - let users = svc.metadata.list_users().await?; - let columns = vec!["user".to_string(), "admin".to_string()]; - let mut values = Vec::new(); - for u in &users { - if let Ok(Some(stored)) = svc.metadata.get_user(u).await { - values.push(vec![ - serde_json::Value::String(u.clone()), - serde_json::Value::Bool(stored.admin), - ]); - } - } - Ok(StatementResult { - statement_id, - series: Some(vec![SeriesResult { - name: "users".to_string(), - tags: None, - columns, - values, - partial: None, - }]), - error: None, - }) + statements::show_catalog::show_grants(svc, username, statement_id).await } + Statement::ShowUsers => statements::show_catalog::show_users(svc, statement_id).await, Statement::CreateRetentionPolicyStmt { name, db: rp_db, @@ -1659,61 +817,27 @@ async fn execute_statement( shard_duration, is_default, } => { - let target_db = if rp_db.is_empty() { db } else { rp_db.as_str() }; - let std_duration = duration - .as_ref() - .map(|d| std::time::Duration::from_nanos(d.to_nanos() as u64)); - let shard_dur = shard_duration - .as_ref() - .map(|d| std::time::Duration::from_nanos(d.to_nanos() as u64)) - .unwrap_or_else(|| { - crate::domain::database::derive_shard_group_duration(std_duration) - }); - let rp = crate::domain::database::RetentionPolicy { - name: name.clone(), - duration: std_duration, - shard_group_duration: shard_dur, - replication_factor: *replication, - is_default: *is_default, - }; - svc.metadata.create_retention_policy(target_db, rp).await?; - Ok(StatementResult { + statements::ddl::create_retention_policy( + svc, + db, + name, + rp_db, + duration, + replication, + shard_duration, + is_default, statement_id, - series: Some(vec![]), - error: None, - }) + ) + .await } Statement::DropRetentionPolicyStmt { name, db: rp_db } => { - let target_db = if rp_db.is_empty() { db } else { rp_db.as_str() }; - svc.metadata.drop_retention_policy(target_db, name).await?; - svc.drop_shard_spaces(target_db, Some(name), None).await; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::ddl::drop_retention_policy(svc, db, name, rp_db, statement_id).await } Statement::CreateUser { username, password, admin, - } => { - let password_hash = crate::domain::user::hash_password(password).map_err(|e| { - HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)) - })?; - crate::application::schema_mutation_apply::create_user( - svc.metadata.as_ref(), - username, - &password_hash, - *admin, - ) - .await?; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) - } + } => statements::users::create_user(svc, username, password, admin, statement_id).await, Statement::DropUser(username) => { svc.metadata.drop_user(username).await?; Ok(StatementResult { @@ -1723,20 +847,7 @@ async fn execute_statement( }) } Statement::SetPassword { username, password } => { - let password_hash = crate::domain::user::hash_password(password).map_err(|e| { - HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)) - })?; - crate::application::schema_mutation_apply::set_password( - svc.metadata.as_ref(), - username, - &password_hash, - ) - .await?; - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::users::set_password(svc, username, password, statement_id).await } Statement::AlterRetentionPolicyStmt { name, @@ -1746,185 +857,32 @@ async fn execute_statement( shard_duration, is_default, } => { - let target_db = if rp_db.is_empty() { db } else { rp_db.as_str() }; - let change = RetentionPolicyChange { - duration: duration.as_ref().map(|d| { - if d.value == 0 && d.unit == DurationUnit::Second { - None - } else { - Some(d.clone()) - } - }), - replication: *replication, - shard_duration: shard_duration.clone(), - is_default: *is_default, - }; - svc.metadata - .alter_retention_policy(target_db, name, &change) - .await?; - Ok(StatementResult { + statements::ddl::alter_retention_policy( + svc, + db, + name, + rp_db, + duration, + replication, + shard_duration, + is_default, statement_id, - series: Some(vec![]), - error: None, - }) + ) + .await } Statement::DropSeries(s) => { - let target_db = s.database.as_deref().unwrap_or(db); - let measurement = s.from.as_ref().and_then(|n| match n { - MeasurementName::Name(n) => Some(n.clone()), - MeasurementName::Regex(_) => None, - }); - let rp = if let Some(rp) = query_rp { - rp.to_string() - } else { - svc.metadata.get_default_rp(target_db).await? - }; - let predicate_sql = if let Some(ref cond) = s.condition { - let meas = measurement.as_deref().unwrap_or(""); - crate::application::predicate_sql::build_predicate_sql( - &svc.metadata, - target_db, - &rp, - meas, - cond, - ) - .await? - } else { - String::new() - }; - if measurement.is_some() || !predicate_sql.is_empty() { - svc.metadata - .delete_series_matching(target_db, &rp, measurement.as_deref(), &predicate_sql) - .await?; - // The series' rows are deleted; points written to them later - // stay. A tombstone used to hide those too. - let mut targets: Vec<(String, String)> = Vec::new(); - if let Some(ref meas) = measurement { - let pred = crate::application::predicate_sql::build_delete_predicate( - &svc.metadata, - target_db, - &rp, - meas, - s.condition.as_ref(), - ) - .await?; - targets.push((meas.clone(), pred)); - } else if svc.shard_routing.is_some() { - // Every measurement that has the condition's tags. - for meas in svc.metadata.list_measurements(target_db).await? { - if let Ok(pred) = crate::application::predicate_sql::build_delete_predicate( - &svc.metadata, - target_db, - &rp, - &meas, - s.condition.as_ref(), - ) - .await - { - targets.push((meas, pred)); - } - } - } - for (meas, pred) in targets { - if let Some(ctx) = svc.shard_routing.as_ref() { - scatter_delete_to_regions( - ctx, - svc.metadata.as_ref(), - Some(crate::application::predicate_sql::LocalRows { - wal: Some(svc.wal.as_ref()), - sink: svc.points_sink.as_ref(), - }), - target_db, - &rp, - &meas, - &pred, - ) - .await?; - } else { - crate::application::predicate_sql::delete_rows( - Some(svc.wal.as_ref()), - svc.points_sink.as_ref(), - target_db, - &rp, - &meas, - &pred, - ) - .await?; - } - } - } - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) + statements::delete::drop_series(svc, db, s, query_rp, statement_id).await } Statement::Grant { username, database, privilege, - } => { - match database { - Some(db) => { - svc.metadata - .grant_privilege(username, db, *privilege) - .await?; - } - None => { - if svc.metadata.get_user(username).await?.is_none() { - return Err(HyperbytedbError::QueryParse(format!( - "user '{username}' does not exist" - ))); - } - crate::application::schema_mutation_apply::set_admin( - svc.metadata.as_ref(), - username, - true, - ) - .await?; - } - } - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) - } + } => statements::users::grant(svc, username, database, privilege, statement_id).await, Statement::Revoke { username, database, privilege, - } => { - match database { - Some(db) => { - crate::application::schema_mutation_apply::revoke_privilege( - svc.metadata.as_ref(), - username, - db, - *privilege, - ) - .await?; - } - None => { - if svc.metadata.get_user(username).await?.is_none() { - return Err(HyperbytedbError::QueryParse(format!( - "user '{username}' does not exist" - ))); - } - crate::application::schema_mutation_apply::set_admin( - svc.metadata.as_ref(), - username, - false, - ) - .await?; - } - } - Ok(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }) - } + } => statements::users::revoke(svc, username, database, privilege, statement_id).await, } } diff --git a/hyperbytedb/src/application/query_service/statements/ddl.rs b/hyperbytedb/src/application/query_service/statements/ddl.rs new file mode 100644 index 0000000..597ca76 --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/ddl.rs @@ -0,0 +1,264 @@ +//! Create, drop and alter of measurements, retention policies and continuous queries. + +use super::super::*; + +pub(in crate::application::query_service) async fn drop_measurement( + svc: &QueryServiceImpl, + db: &str, + name: &str, + stmt_rp: &Option, + statement_id: u32, +) -> Result { + if let Err(e) = svc.mv_service.drop_for_source_measurement(db, name).await { + tracing::warn!( + db = db, + measurement = name, + error = %e, + "failed to cascade-drop materialized views for source measurement" + ); + } + let rp = if let Some(inner) = stmt_rp { + inner.clone() + } else { + svc.metadata.get_default_rp(db).await? + }; + svc.metadata.delete_measurement(db, &rp, name).await?; + if let Some(ctx) = svc.shard_routing.as_ref() { + scatter_delete_to_regions(ctx, svc.metadata.as_ref(), None, db, &rp, name, "").await?; + } + if let Err(e) = svc.points_sink.drop_measurement(db, &rp, name).await { + tracing::warn!( + db = db, + measurement = name, + error = %e, + "failed to drop chDB native table for measurement (metadata already cleared)" + ); + } + svc.drop_shard_spaces(db, Some(&rp), Some(name)).await; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn create_continuous_query( + svc: &QueryServiceImpl, + cq: &crate::timeseriesql::ast::CreateContinuousQueryStatement, + statement_id: u32, +) -> Result { + svc.metadata + .get_database(&cq.database) + .await? + .ok_or_else(|| HyperbytedbError::DatabaseNotFound(cq.database.clone()))?; + + let def = match ContinuousQueryDef::from_create(cq) { + Ok(def) => def, + Err(e) => { + return Ok(StatementResult { + statement_id, + series: None, + error: Some(e.to_string()), + }); + } + }; + + svc.metadata + .store_continuous_query(&def.database, &def.name, &def) + .await?; + + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +// `&String` and one parameter per field: the body is moved verbatim from the +// match arm. +#[allow(clippy::ptr_arg, clippy::too_many_arguments)] +pub(in crate::application::query_service) async fn create_retention_policy( + svc: &QueryServiceImpl, + db: &str, + name: &String, + rp_db: &String, + duration: &Option, + replication: &u32, + shard_duration: &Option, + is_default: &bool, + statement_id: u32, +) -> Result { + let target_db = if rp_db.is_empty() { db } else { rp_db.as_str() }; + let std_duration = duration + .as_ref() + .map(|d| std::time::Duration::from_nanos(d.to_nanos() as u64)); + let shard_dur = shard_duration + .as_ref() + .map(|d| std::time::Duration::from_nanos(d.to_nanos() as u64)) + .unwrap_or_else(|| crate::domain::database::derive_shard_group_duration(std_duration)); + let rp = crate::domain::database::RetentionPolicy { + name: name.clone(), + duration: std_duration, + shard_group_duration: shard_dur, + replication_factor: *replication, + is_default: *is_default, + }; + svc.metadata.create_retention_policy(target_db, rp).await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +// `&String` and one parameter per field: the body is moved verbatim from the +// match arm. +#[allow(clippy::ptr_arg, clippy::too_many_arguments)] +pub(in crate::application::query_service) async fn alter_retention_policy( + svc: &QueryServiceImpl, + db: &str, + name: &String, + rp_db: &String, + duration: &Option, + replication: &Option, + shard_duration: &Option, + is_default: &Option, + statement_id: u32, +) -> Result { + let target_db = if rp_db.is_empty() { db } else { rp_db.as_str() }; + let change = RetentionPolicyChange { + duration: duration.as_ref().map(|d| { + if d.value == 0 && d.unit == DurationUnit::Second { + None + } else { + Some(d.clone()) + } + }), + replication: *replication, + shard_duration: shard_duration.clone(), + is_default: *is_default, + }; + svc.metadata + .alter_retention_policy(target_db, name, &change) + .await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn drop_database( + svc: &QueryServiceImpl, + name: &str, + statement_id: u32, +) -> Result { + crate::application::database_drop::drop_database( + &svc.metadata, + Some(&svc.mv_service), + Some(&svc.points_sink), + Some(&svc.wal), + name, + ) + .await?; + svc.drop_shard_spaces(name, None, None).await; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn drop_continuous_query( + svc: &QueryServiceImpl, + db: &str, + name: &str, + cq_db: &str, + statement_id: u32, +) -> Result { + let target_db = if cq_db.is_empty() { db } else { cq_db }; + svc.metadata + .get_database(target_db) + .await? + .ok_or_else(|| HyperbytedbError::DatabaseNotFound(target_db.to_string()))?; + svc.metadata.drop_continuous_query(target_db, name).await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn create_materialized_view( + svc: &QueryServiceImpl, + mv: &crate::timeseriesql::ast::CreateMaterializedViewStatement, + statement_id: u32, +) -> Result { + svc.metadata + .get_database(&mv.database) + .await? + .ok_or_else(|| HyperbytedbError::DatabaseNotFound(mv.database.clone()))?; + svc.mv_service.create(mv).await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn set_setting( + svc: &QueryServiceImpl, + name: &str, + value: &Option, + statement_id: u32, +) -> Result { + // A single node; a cluster replicates the SET through Raft and + // every node applies it the same way. + let value = crate::application::runtime_settings::validate(name, value.clone())?; + svc.settings()? + .apply(svc.metadata.as_ref(), name, value) + .await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn drop_materialized_view( + svc: &QueryServiceImpl, + db: &str, + name: &str, + mv_db: &str, + statement_id: u32, +) -> Result { + let target_db = if mv_db.is_empty() { db } else { mv_db }; + svc.metadata + .get_database(target_db) + .await? + .ok_or_else(|| HyperbytedbError::DatabaseNotFound(target_db.to_string()))?; + svc.mv_service.drop_mv(target_db, name).await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +#[allow(clippy::ptr_arg)] // `&String`: the body is moved verbatim from the match arm. +pub(in crate::application::query_service) async fn drop_retention_policy( + svc: &QueryServiceImpl, + db: &str, + name: &str, + rp_db: &String, + statement_id: u32, +) -> Result { + let target_db = if rp_db.is_empty() { db } else { rp_db.as_str() }; + svc.metadata.drop_retention_policy(target_db, name).await?; + svc.drop_shard_spaces(target_db, Some(name), None).await; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} diff --git a/hyperbytedb/src/application/query_service/statements/delete.rs b/hyperbytedb/src/application/query_service/statements/delete.rs new file mode 100644 index 0000000..e2dc1ff --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/delete.rs @@ -0,0 +1,168 @@ +//! DELETE and DROP SERIES. + +use super::super::*; + +pub(in crate::application::query_service) async fn delete( + svc: &QueryServiceImpl, + db: &str, + del: &crate::timeseriesql::ast::DeleteStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + let del_rp = if let Some(rp) = query_rp { + rp.to_string() + } else { + svc.metadata + .get_default_rp(db) + .await + .unwrap_or_else(|_| "autogen".to_string()) + }; + // The rows are deleted, as in InfluxDB: points written after the + // DELETE stay. It used to store a tombstone that every later query + // applied, hiding later points too. + let predicate_sql = crate::application::predicate_sql::build_delete_predicate( + &svc.metadata, + db, + &del_rp, + &del.from, + del.condition.as_ref(), + ) + .await?; + + if let Some(ctx) = svc.shard_routing.as_ref() { + scatter_delete_to_regions( + ctx, + svc.metadata.as_ref(), + Some(crate::application::predicate_sql::LocalRows { + wal: Some(svc.wal.as_ref()), + sink: svc.points_sink.as_ref(), + }), + db, + &del_rp, + &del.from, + &predicate_sql, + ) + .await?; + } else { + crate::application::predicate_sql::delete_rows( + Some(svc.wal.as_ref()), + svc.points_sink.as_ref(), + db, + &del_rp, + &del.from, + &predicate_sql, + ) + .await?; + } + + tracing::debug!( + db = db, + measurement = %del.from, + predicate = %predicate_sql, + "DELETE applied" + ); + + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn drop_series( + svc: &QueryServiceImpl, + db: &str, + s: &crate::timeseriesql::ast::DropSeriesStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + let target_db = s.database.as_deref().unwrap_or(db); + let measurement = s.from.as_ref().and_then(|n| match n { + MeasurementName::Name(n) => Some(n.clone()), + MeasurementName::Regex(_) => None, + }); + let rp = if let Some(rp) = query_rp { + rp.to_string() + } else { + svc.metadata.get_default_rp(target_db).await? + }; + let predicate_sql = if let Some(ref cond) = s.condition { + let meas = measurement.as_deref().unwrap_or(""); + crate::application::predicate_sql::build_predicate_sql( + &svc.metadata, + target_db, + &rp, + meas, + cond, + ) + .await? + } else { + String::new() + }; + if measurement.is_some() || !predicate_sql.is_empty() { + svc.metadata + .delete_series_matching(target_db, &rp, measurement.as_deref(), &predicate_sql) + .await?; + // The series' rows are deleted; points written to them later + // stay. A tombstone used to hide those too. + let mut targets: Vec<(String, String)> = Vec::new(); + if let Some(ref meas) = measurement { + let pred = crate::application::predicate_sql::build_delete_predicate( + &svc.metadata, + target_db, + &rp, + meas, + s.condition.as_ref(), + ) + .await?; + targets.push((meas.clone(), pred)); + } else if svc.shard_routing.is_some() { + // Every measurement that has the condition's tags. + for meas in svc.metadata.list_measurements(target_db).await? { + if let Ok(pred) = crate::application::predicate_sql::build_delete_predicate( + &svc.metadata, + target_db, + &rp, + &meas, + s.condition.as_ref(), + ) + .await + { + targets.push((meas, pred)); + } + } + } + for (meas, pred) in targets { + if let Some(ctx) = svc.shard_routing.as_ref() { + scatter_delete_to_regions( + ctx, + svc.metadata.as_ref(), + Some(crate::application::predicate_sql::LocalRows { + wal: Some(svc.wal.as_ref()), + sink: svc.points_sink.as_ref(), + }), + target_db, + &rp, + &meas, + &pred, + ) + .await?; + } else { + crate::application::predicate_sql::delete_rows( + Some(svc.wal.as_ref()), + svc.points_sink.as_ref(), + target_db, + &rp, + &meas, + &pred, + ) + .await?; + } + } + } + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} diff --git a/hyperbytedb/src/application/query_service/statements/mod.rs b/hyperbytedb/src/application/query_service/statements/mod.rs new file mode 100644 index 0000000..7c786f0 --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/mod.rs @@ -0,0 +1,8 @@ +//! `execute_statement`'s larger arms, one function per statement kind. + +pub(super) mod ddl; +pub(super) mod delete; +pub(super) mod select; +pub(super) mod show_catalog; +pub(super) mod show_schema; +pub(super) mod users; diff --git a/hyperbytedb/src/application/query_service/statements/select.rs b/hyperbytedb/src/application/query_service/statements/select.rs new file mode 100644 index 0000000..0b09ac3 --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/select.rs @@ -0,0 +1,93 @@ +//! SELECT. + +use super::super::*; + +pub(in crate::application::query_service) async fn select( + svc: &QueryServiceImpl, + db: &str, + select_stmt: &crate::timeseriesql::ast::SelectStatement, + epoch: Option<&str>, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + if db.is_empty() { + return Ok(StatementResult { + statement_id, + series: None, + error: Some("database is required".to_string()), + }); + } + + if select_stmt.into.is_some() { + let count = execute_select_into(svc, db, select_stmt, epoch).await?; + return Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "result".to_string(), + tags: None, + columns: vec!["time".to_string(), "written".to_string()], + values: vec![vec![ + serde_json::Value::String(chrono::Utc::now().to_rfc3339()), + serde_json::json!(count), + ]], + partial: None, + }]), + error: None, + }); + } + + let group_by_tags: Vec = select_stmt + .group_by + .as_ref() + .map(|gb| { + gb.tag_dimensions() + .into_iter() + .map(|s| s.to_string()) + .collect() + }) + .unwrap_or_default(); + + let (time_min, time_max) = to_clickhouse::extract_time_bounds(select_stmt.condition.as_ref()); + + if select_stmt.from.is_empty() { + return Err(HyperbytedbError::QueryParse( + "SELECT requires FROM clause".to_string(), + )); + } + + let mut all_series = Vec::new(); + for source in &select_stmt.from { + let mut series = execute_select_from_source( + svc, + db, + select_stmt, + source, + time_min, + time_max, + epoch, + &group_by_tags, + query_rp, + ) + .await?; + all_series.append(&mut series); + } + + stamp_aggregate_time(select_stmt, &mut all_series, epoch, time_min); + order_columns_like_select(select_stmt, &mut all_series); + + // Apply SLIMIT/SOFFSET (series-level pagination) + if select_stmt.slimit.is_some() || select_stmt.soffset.is_some() { + let soffset = select_stmt.soffset.unwrap_or(0) as usize; + let slimit = select_stmt.slimit.unwrap_or(u64::MAX) as usize; + let len = all_series.len(); + let start = soffset.min(len); + let end = (start + slimit).min(len); + all_series = all_series[start..end].to_vec(); + } + + Ok(StatementResult { + statement_id, + series: Some(all_series), + error: None, + }) +} diff --git a/hyperbytedb/src/application/query_service/statements/show_catalog.rs b/hyperbytedb/src/application/query_service/statements/show_catalog.rs new file mode 100644 index 0000000..a33cdff --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/show_catalog.rs @@ -0,0 +1,255 @@ +//! SHOW statements over the catalog: databases, retention policies, users, grants, continuous queries, materialized views and settings. + +use super::super::*; + +pub(in crate::application::query_service) async fn show_continuous_queries( + svc: &QueryServiceImpl, + statement_id: u32, +) -> Result { + let dbs = svc.metadata.list_databases().await?; + let mut all_series = Vec::new(); + for db_entry in &dbs { + let cqs = svc.metadata.list_continuous_queries(&db_entry.name).await?; + if cqs.is_empty() { + continue; + } + let columns = vec!["name".to_string(), "query".to_string()]; + let values: Vec> = cqs + .iter() + .map(|cq| { + vec![ + serde_json::Value::String(cq.name.clone()), + serde_json::Value::String(reconstruct_cq_text(cq)), + ] + }) + .collect(); + all_series.push(SeriesResult { + name: db_entry.name.clone(), + tags: None, + columns, + values, + partial: None, + }); + } + + Ok(StatementResult { + statement_id, + series: Some(all_series), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_materialized_views( + svc: &QueryServiceImpl, + statement_id: u32, +) -> Result { + let dbs = svc.metadata.list_databases().await?; + let mut all_mvs = Vec::new(); + for db_entry in &dbs { + let mvs = svc.metadata.list_materialized_views(&db_entry.name).await?; + all_mvs.extend(mvs); + } + + let columns = vec![ + "name".to_string(), + "database".to_string(), + "query".to_string(), + "source_measurement".to_string(), + "dest_measurement".to_string(), + ]; + let values: Vec> = all_mvs + .iter() + .map(|mv| { + vec![ + serde_json::Value::String(mv.name.clone()), + serde_json::Value::String(mv.database.clone()), + serde_json::Value::String(mv.query_text.clone()), + serde_json::Value::String(mv.source_measurement.clone()), + serde_json::Value::String(mv.dest_measurement.clone()), + ] + }) + .collect(); + + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "materialized_views".to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_settings( + svc: &QueryServiceImpl, + statement_id: u32, +) -> Result { + let rows = svc.settings()?.show(svc.metadata.as_ref()).await?; + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "settings".to_string(), + tags: None, + columns: vec![ + "name".into(), + "value".into(), + "source".into(), + "config".into(), + "description".into(), + ], + values: rows + .into_iter() + .map(|r| { + vec![ + serde_json::Value::String(r.name.to_string()), + r.value + .as_ref() + .map_or(serde_json::Value::Null, |v| v.to_json()), + r.source.map_or(serde_json::Value::Null, |s| { + serde_json::Value::String(s.as_str().to_string()) + }), + serde_json::Value::String(r.config.to_string()), + serde_json::Value::String(r.description.to_string()), + ] + }) + .collect(), + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_retention_policies( + svc: &QueryServiceImpl, + db: &str, + rp_db: &str, + statement_id: u32, +) -> Result { + let target_db = if rp_db.is_empty() { db } else { rp_db }; + if !rp_db.is_empty() { + svc.metadata + .get_database(target_db) + .await? + .ok_or_else(|| HyperbytedbError::DatabaseNotFound(target_db.to_string()))?; + } + let rps = svc.metadata.list_retention_policies(target_db).await?; + let columns = vec![ + "name".to_string(), + "duration".to_string(), + "shardGroupDuration".to_string(), + "replicaN".to_string(), + "default".to_string(), + ]; + let values: Vec> = rps + .iter() + .map(|rp| { + let dur_str = crate::domain::database::format_influx_duration(rp.duration); + let sgd_str = + crate::domain::database::format_influx_duration(Some(rp.shard_group_duration)); + vec![ + serde_json::Value::String(rp.name.clone()), + serde_json::Value::String(dur_str), + serde_json::Value::String(sgd_str), + serde_json::json!(rp.replication_factor), + serde_json::Value::Bool(rp.is_default), + ] + }) + .collect(); + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "".to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_grants( + svc: &QueryServiceImpl, + username: &str, + statement_id: u32, +) -> Result { + let user = + svc.metadata.get_user(username).await?.ok_or_else(|| { + HyperbytedbError::QueryParse(format!("user '{username}' does not exist")) + })?; + let mut grants: Vec<_> = user.privileges.into_iter().collect(); + grants.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: String::new(), + tags: None, + columns: vec!["database".to_string(), "privilege".to_string()], + values: grants + .into_iter() + .map(|(db, p)| { + vec![ + serde_json::Value::String(db), + serde_json::Value::String(p.as_str().to_string()), + ] + }) + .collect(), + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_users( + svc: &QueryServiceImpl, + statement_id: u32, +) -> Result { + let users = svc.metadata.list_users().await?; + let columns = vec!["user".to_string(), "admin".to_string()]; + let mut values = Vec::new(); + for u in &users { + if let Ok(Some(stored)) = svc.metadata.get_user(u).await { + values.push(vec![ + serde_json::Value::String(u.clone()), + serde_json::Value::Bool(stored.admin), + ]); + } + } + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "users".to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_databases( + svc: &QueryServiceImpl, + statement_id: u32, +) -> Result { + let dbs = svc.metadata.list_databases().await?; + let columns = vec!["name".to_string()]; + let values: Vec> = dbs + .iter() + .map(|d| vec![serde_json::Value::String(d.name.clone())]) + .collect(); + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "databases".to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} diff --git a/hyperbytedb/src/application/query_service/statements/show_schema.rs b/hyperbytedb/src/application/query_service/statements/show_schema.rs new file mode 100644 index 0000000..21f9caf --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/show_schema.rs @@ -0,0 +1,431 @@ +//! SHOW statements over a database's measurement schema. + +use super::super::*; + +pub(in crate::application::query_service) async fn show_measurements( + svc: &QueryServiceImpl, + db: &str, + s: &crate::timeseriesql::ast::ShowMeasurementsStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + let db = s.database.as_deref().unwrap_or(db); + if db.is_empty() { + return Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: Some("database is required".to_string()), + }); + } + let rp = resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp).await?; + let names = list_measurements_for_rp(svc, db, &rp).await?; + let filtered = filter_names_by_selector(&names, s.measurement_filter.as_ref()); + let columns = vec!["name".to_string()]; + let values: Vec> = filtered + .iter() + .map(|n| vec![serde_json::Value::String(n.clone())]) + .collect(); + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: "measurements".to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_tag_keys( + svc: &QueryServiceImpl, + db: &str, + s: &crate::timeseriesql::ast::ShowTagKeysStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + let db = s.database.as_deref().unwrap_or(db); + let measurement = s.from.as_ref().and_then(|m| m.name_str()); + let default_rp = + resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp).await?; + let keys = if let (Some(ctx), Some(m), Some(name)) = ( + svc.shard_routing.as_ref(), + s.from.as_ref(), + s.from.as_ref().and_then(|m| m.name_str()), + ) { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = if m.retention_policy.is_some() { + resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await? + } else { + default_rp.clone() + }; + scatter_tag_keys(ctx, svc.metadata.as_ref(), query_db, &rp, name).await? + } else if let Some(m) = s.from.as_ref() { + if let Some(name) = m.name_str() { + if m.retention_policy.is_some() { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await?; + tag_keys_for_measurement(svc, query_db, &rp, name).await? + } else { + svc.metadata + .list_tag_keys(db, &default_rp, Some(name)) + .await? + } + } else { + svc.metadata + .list_tag_keys(db, &default_rp, measurement) + .await? + } + } else { + svc.metadata + .list_tag_keys(db, &default_rp, measurement) + .await? + }; + let columns = vec!["tagKey".to_string()]; + let values: Vec> = keys + .iter() + .map(|k| vec![serde_json::Value::String(k.clone())]) + .collect(); + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: measurement.unwrap_or("").to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_tag_values( + svc: &QueryServiceImpl, + db: &str, + s: &crate::timeseriesql::ast::ShowTagValuesStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + let db = s.database.as_deref().unwrap_or(db); + let measurement = s.from.as_ref().and_then(|m| m.name_str()); + let default_rp = + resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp).await?; + + let all_tag_keys = if let (Some(ctx), Some(m), Some(name)) = + (svc.shard_routing.as_ref(), s.from.as_ref(), measurement) + { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = if m.retention_policy.is_some() { + resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await? + } else { + default_rp.clone() + }; + scatter_tag_keys(ctx, svc.metadata.as_ref(), query_db, &rp, name).await? + } else if let (Some(m), Some(name)) = (s.from.as_ref(), measurement) { + if m.retention_policy.is_some() { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await?; + tag_keys_for_measurement(svc, query_db, &rp, name).await? + } else { + svc.metadata + .list_tag_keys(db, &default_rp, Some(name)) + .await? + } + } else { + svc.metadata + .list_tag_keys(db, &default_rp, measurement) + .await? + }; + + let matching_keys: Vec = filter_names_by_selector(&all_tag_keys, Some(&s.tag_key)); + + // With a WHERE: the values of the series that match it. It used + // to be ignored, so a Grafana variable chained on another (the + // CPUs of the chosen host) listed every value. + if let Some(cond) = &s.condition { + let predicate = crate::application::tag_condition::TagPredicate::compile(cond)?; + let measurements: Vec = match measurement { + Some(name) => vec![name.to_string()], + None => svc.metadata.list_measurements(db).await?, + }; + let (query_db, rp) = match s.from.as_ref() { + Some(m) if m.retention_policy.is_some() => { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await?; + (query_db, rp) + } + Some(m) => (m.database.as_deref().unwrap_or(db), default_rp.clone()), + None => (db, default_rp.clone()), + }; + let mut pairs = std::collections::BTreeSet::new(); + for name in &measurements { + let series = match svc.shard_routing.as_ref() { + Some(ctx) => { + crate::application::shard_metadata_scatter::scatter_series_tags( + ctx, + svc.metadata.as_ref(), + query_db, + &rp, + name, + ) + .await? + } + None => svc + .metadata + .list_series(query_db, &rp, name) + .await? + .into_iter() + .map(|(_, tags)| tags) + .collect(), + }; + for tags in series.iter().filter(|t| predicate.matches(t)) { + for key in &matching_keys { + if let Some(v) = tags.get(key) { + pairs.insert((key.clone(), v.clone())); + } + } + } + } + let values = limit_offset( + pairs + .into_iter() + .map(|(k, v)| vec![serde_json::Value::String(k), serde_json::Value::String(v)]) + .collect(), + s.limit, + s.offset, + ); + return Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: measurement.unwrap_or("").to_string(), + tags: None, + columns: vec!["key".to_string(), "value".to_string()], + values, + partial: None, + }]), + error: None, + }); + } + let default_rp_for_values = default_rp.clone(); + + let key_value_futures: Vec<_> = matching_keys + .iter() + .map(|tag_key| { + let tag_key = tag_key.clone(); + let default_rp = default_rp_for_values.clone(); + async move { + let values_list = if let (Some(ctx), Some(m), Some(name)) = + (svc.shard_routing.as_ref(), s.from.as_ref(), measurement) + { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = if m.retention_policy.is_some() { + resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await? + } else { + default_rp.clone() + }; + scatter_tag_values(ctx, svc.metadata.as_ref(), query_db, &rp, name, &tag_key) + .await? + } else if let (Some(m), Some(name)) = (s.from.as_ref(), measurement) { + if m.retention_policy.is_some() { + let query_db = m.database.as_deref().unwrap_or(db); + let rp = resolve_retention_policy( + svc.metadata.as_ref(), + query_db, + m.retention_policy.as_deref(), + ) + .await?; + tag_values_for_measurement(svc, query_db, &rp, name, &tag_key).await? + } else { + svc.metadata + .list_tag_values(db, &default_rp, &tag_key, Some(name)) + .await? + } + } else { + svc.metadata + .list_tag_values(db, &default_rp, &tag_key, measurement) + .await? + }; + Ok::<_, HyperbytedbError>((tag_key, values_list)) + } + }) + .collect(); + + let mut all_values = Vec::new(); + for (tag_key, values_list) in futures::future::try_join_all(key_value_futures).await? { + for v in values_list { + all_values.push(vec![ + serde_json::Value::String(tag_key.clone()), + serde_json::Value::String(v), + ]); + } + } + + let columns = vec!["key".to_string(), "value".to_string()]; + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: measurement.unwrap_or("").to_string(), + tags: None, + columns, + values: limit_offset(all_values, s.limit, s.offset), + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_field_keys( + svc: &QueryServiceImpl, + db: &str, + s: &crate::timeseriesql::ast::ShowFieldKeysStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + let db = s.database.as_deref().unwrap_or(db); + let measurement = s.from.as_ref().and_then(|m| m.name_str()); + if db.is_empty() { + return Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: Some("database is required".to_string()), + }); + } + let rp = resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp).await?; + let mut field_values = Vec::new(); + let measurements: Vec = if let Some(m) = measurement { + vec![m.to_string()] + } else { + svc.metadata.list_measurements(db).await? + }; + for m in &measurements { + let fields = match svc.shard_routing.as_ref() { + Some(ctx) => { + crate::application::shard_metadata_scatter::scatter_field_keys( + ctx, + svc.metadata.as_ref(), + db, + &rp, + m, + ) + .await? + } + None => crate::application::shard_metadata_scatter::local_field_keys( + svc.metadata.get_measurement(db, &rp, m).await?.as_ref(), + ), + }; + for (name, typ) in fields { + field_values.push(vec![ + serde_json::Value::String(name), + serde_json::Value::String(typ), + ]); + } + } + let columns = vec!["fieldKey".to_string(), "fieldType".to_string()]; + let values: Vec> = field_values; + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: measurement.unwrap_or("").to_string(), + tags: None, + columns, + values, + partial: None, + }]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn show_series( + svc: &QueryServiceImpl, + db: &str, + s: &crate::timeseriesql::ast::ShowSeriesStatement, + query_rp: Option<&str>, + statement_id: u32, +) -> Result { + use crate::domain::series::SeriesKey; + + let query_db = s.database.as_deref().unwrap_or(db); + if query_db.is_empty() { + return Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: Some("database is required".to_string()), + }); + } + + let measurements: Vec = if let Some(ref from) = s.from { + if let Some(name) = from.name_str() { + vec![name.to_string()] + } else { + svc.metadata.list_measurements(query_db).await? + } + } else { + svc.metadata.list_measurements(query_db).await? + }; + + let rp = if let Some(ref from) = s.from + && let Some(rp_name) = from.retention_policy.as_ref() + { + resolve_retention_policy(svc.metadata.as_ref(), query_db, Some(rp_name)).await? + } else { + resolve_retention_policy_for_select(svc.metadata.as_ref(), query_db, None, query_rp).await? + }; + + let mut values = Vec::new(); + for meas in &measurements { + if let Some(ctx) = svc.shard_routing.as_ref() { + for key in scatter_series_keys(ctx, svc.metadata.as_ref(), query_db, &rp, meas).await? { + values.push(vec![serde_json::Value::String(key)]); + } + } else { + let series = svc.metadata.list_series(query_db, &rp, meas).await?; + for (_, tags) in series { + let key = SeriesKey::new(meas, &tags); + values.push(vec![serde_json::Value::String(key.to_canonical())]); + } + } + } + + Ok(StatementResult { + statement_id, + series: Some(vec![SeriesResult { + name: String::new(), + tags: None, + columns: vec!["key".to_string()], + values, + partial: None, + }]), + error: None, + }) +} diff --git a/hyperbytedb/src/application/query_service/statements/users.rs b/hyperbytedb/src/application/query_service/statements/users.rs new file mode 100644 index 0000000..51a4bf7 --- /dev/null +++ b/hyperbytedb/src/application/query_service/statements/users.rs @@ -0,0 +1,119 @@ +//! User and privilege statements. + +use super::super::*; + +pub(in crate::application::query_service) async fn create_user( + svc: &QueryServiceImpl, + username: &str, + password: &str, + admin: &bool, + statement_id: u32, +) -> Result { + let password_hash = crate::domain::user::hash_password(password) + .map_err(|e| HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)))?; + crate::application::schema_mutation_apply::create_user( + svc.metadata.as_ref(), + username, + &password_hash, + *admin, + ) + .await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn grant( + svc: &QueryServiceImpl, + username: &str, + database: &Option, + privilege: &crate::domain::user::DatabasePrivilege, + statement_id: u32, +) -> Result { + match database { + Some(db) => { + svc.metadata + .grant_privilege(username, db, *privilege) + .await?; + } + None => { + if svc.metadata.get_user(username).await?.is_none() { + return Err(HyperbytedbError::QueryParse(format!( + "user '{username}' does not exist" + ))); + } + crate::application::schema_mutation_apply::set_admin( + svc.metadata.as_ref(), + username, + true, + ) + .await?; + } + } + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn revoke( + svc: &QueryServiceImpl, + username: &str, + database: &Option, + privilege: &crate::domain::user::DatabasePrivilege, + statement_id: u32, +) -> Result { + match database { + Some(db) => { + crate::application::schema_mutation_apply::revoke_privilege( + svc.metadata.as_ref(), + username, + db, + *privilege, + ) + .await?; + } + None => { + if svc.metadata.get_user(username).await?.is_none() { + return Err(HyperbytedbError::QueryParse(format!( + "user '{username}' does not exist" + ))); + } + crate::application::schema_mutation_apply::set_admin( + svc.metadata.as_ref(), + username, + false, + ) + .await?; + } + } + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} + +pub(in crate::application::query_service) async fn set_password( + svc: &QueryServiceImpl, + username: &str, + password: &str, + statement_id: u32, +) -> Result { + let password_hash = crate::domain::user::hash_password(password) + .map_err(|e| HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)))?; + crate::application::schema_mutation_apply::set_password( + svc.metadata.as_ref(), + username, + &password_hash, + ) + .await?; + Ok(StatementResult { + statement_id, + series: Some(vec![]), + error: None, + }) +} diff --git a/hyperbytedb/src/domain/sharding/ops/apply.rs b/hyperbytedb/src/domain/sharding/ops/apply.rs new file mode 100644 index 0000000..74cd07b --- /dev/null +++ b/hyperbytedb/src/domain/sharding/ops/apply.rs @@ -0,0 +1,464 @@ +//! One function per [`ShardMapOp`]: what `apply_shard_map_op` does for it, +//! after bumping `map_version`. + +use super::*; +use crate::domain::sharding::types::ShardMap; + +pub(super) fn apply_bootstrap_measurement_presplit( + map: &mut ShardMap, + key: MeasurementKey, + regions: Vec, +) -> Result<(), ShardMapApplyError> { + if map.spaces.contains_key(&key) { + return Err(ShardMapApplyError::SpaceExists { + db: key.db.clone(), + rp: key.rp.clone(), + measurement: key.measurement.clone(), + }); + } + let mut ids = std::collections::HashSet::new(); + for region in ®ions { + if !ids.insert(region.region_id) + || map.spaces.values().any(|space| { + space + .regions + .iter() + .any(|r| r.region_id == region.region_id) + }) + { + return Err(ShardMapApplyError::DuplicateRegionId(region.region_id)); + } + } + if let Some(max) = regions.iter().map(|r| r.region_id).max() { + map.next_region_id = map.next_region_id.max(max.saturating_add(1)); + } + let mut space = super::types::MeasurementShardSpace { + key: key.clone(), + regions, + }; + sort_space_regions(&mut space); + space.validate().map_err(ShardMapApplyError::Invalid)?; + map.spaces.insert(key, space); + Ok(()) +} + +pub(super) fn apply_bootstrap_measurement( + map: &mut ShardMap, + key: MeasurementKey, + region: ShardRegion, +) -> Result<(), ShardMapApplyError> { + if map.spaces.contains_key(&key) { + return Err(ShardMapApplyError::SpaceExists { + db: key.db.clone(), + rp: key.rp.clone(), + measurement: key.measurement.clone(), + }); + } + if map.spaces.values().any(|space| { + space + .regions + .iter() + .any(|r| r.region_id == region.region_id) + }) { + return Err(ShardMapApplyError::DuplicateRegionId(region.region_id)); + } + map.next_region_id = map.next_region_id.max(region.region_id.saturating_add(1)); + let mut space = super::types::MeasurementShardSpace { + key: key.clone(), + regions: vec![region], + }; + sort_space_regions(&mut space); + space.validate().map_err(ShardMapApplyError::Invalid)?; + map.spaces.insert(key, space); + Ok(()) +} + +pub(super) fn apply_split( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + split_key: u64, + epoch: ShardEpoch, + mut left: ShardRegion, + mut right: ShardRegion, +) -> Result<(), ShardMapApplyError> { + // Allocate region_id at apply time so concurrent split proposals cannot + // collide on a stale proposer-chosen id. + let allocated = map.next_region_id; + if map + .spaces + .values() + .any(|s| s.regions.iter().any(|r| r.region_id == allocated)) + { + return Err(ShardMapApplyError::DuplicateRegionId(allocated)); + } + right.region_id = allocated; + map.next_region_id = allocated.saturating_add(1); + + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let idx = space + .regions + .iter() + .position(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + let old = &space.regions[idx]; + if old.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("Split")); + } + if split_key <= old.start || split_key >= old.end { + return Err(ShardMapApplyError::Invalid(format!( + "split_key {split_key} not inside region [{}, {})", + old.start, old.end + ))); + } + if left.end != split_key || right.start != split_key { + return Err(ShardMapApplyError::Invalid( + "split regions must meet at split_key".into(), + )); + } + // Apply-time normalization: flag every child whose primary differs + // from the parent's (a metadata-only split never does). Nothing + // clears the flag any more: the reconciliation that re-pushed + // and verified flagged children was removed. `transfer_first_seen` propagates from the + // parent so re-splitting a flagged region cannot reset the age + // gate. Predicates are deterministic from (op, prior map), so all + // replicas normalize identically. + let inherited_first_seen = old.transfer_first_seen; + for child in [&mut left, &mut right] { + if child.primary != old.primary { + child.transfer_verified = Some(false); + child.transfer_first_seen = inherited_first_seen.or(child.transfer_first_seen); + } + } + space.regions[idx] = left; + space.regions.insert(idx + 1, right); + sort_space_regions(space); + space.validate().map_err(ShardMapApplyError::Invalid)?; + map.validate_global_region_ids() + .map_err(ShardMapApplyError::Invalid)?; + Ok(()) +} + +pub(super) fn apply_drop_space( + map: &mut ShardMap, + db: String, + rp: Option, + measurement: Option, +) -> Result<(), ShardMapApplyError> { + map.spaces + .retain(|key, _| !ShardMapOp::drop_matches(&db, &rp, &measurement, key)); + Ok(()) +} + +pub(super) fn apply_merge( + map: &mut ShardMap, + key: MeasurementKey, + left_region_id: u64, + right_region_id: u64, + epoch: ShardEpoch, + right_epoch: Option, + merged: ShardRegion, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let li = space + .regions + .iter() + .position(|r| r.region_id == left_region_id) + .ok_or(ShardMapApplyError::UnknownLeftRegion(left_region_id))?; + let left_epoch = space.regions[li].epoch; + if left_epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("Merge")); + } + let ri = space + .regions + .iter() + .position(|r| r.region_id == right_region_id) + .ok_or(ShardMapApplyError::UnknownRightRegion(right_region_id))?; + if ri != li + 1 { + return Err(ShardMapApplyError::Invalid( + "merge regions must be adjacent".into(), + )); + } + // The right half's data was staged onto the merged owner based on + // its state at proposal time; a concurrently changed right region + // (MovePeer/TransferPrimary) invalidates that staging. + if space.regions[ri].epoch != right_epoch.unwrap_or(epoch) { + return Err(ShardMapApplyError::StaleEpoch("Merge right region")); + } + space.regions.remove(ri); + space.regions[li] = merged; + sort_space_regions(space); + space.validate().map_err(ShardMapApplyError::Invalid)?; + Ok(()) +} + +pub(super) fn apply_move_peer( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + from_peer: u64, + to_peer: u64, + epoch: ShardEpoch, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("MovePeer")); + } + if !region.peers.contains(&from_peer) { + return Err(ShardMapApplyError::Invalid(format!( + "peer {from_peer} not in region" + ))); + } + if region.staging_peers.contains(&to_peer) { + return Err(ShardMapApplyError::Invalid(format!( + "cannot move peer to learner {to_peer}" + ))); + } + region.peers.retain(|p| *p != from_peer); + region.staging_peers.retain(|p| *p != from_peer); + if !region.peers.contains(&to_peer) { + region.peers.push(to_peer); + } + region.epoch = epoch.bump_conf_ver(); + if region.primary == from_peer { + region.primary = to_peer; + } + sort_space_regions(space); + Ok(()) +} + +pub(super) fn apply_add_peer( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + to_peer: u64, + epoch: ShardEpoch, + staging: bool, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("AddPeer")); + } + if region.transfer_outstanding() { + return Err(ShardMapApplyError::Invalid( + "cannot add peer while transfer debt is outstanding".into(), + )); + } + if region.peers.contains(&to_peer) { + return Err(ShardMapApplyError::Invalid(format!( + "peer {to_peer} already in region" + ))); + } + region.peers.push(to_peer); + if staging && !region.staging_peers.contains(&to_peer) { + region.staging_peers.push(to_peer); + } + region.epoch = epoch.bump_conf_ver(); + sort_space_regions(space); + Ok(()) +} + +pub(super) fn apply_promote_peer( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + peer: u64, + epoch: ShardEpoch, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("PromotePeer")); + } + if !region.peers.contains(&peer) { + return Err(ShardMapApplyError::Invalid(format!( + "cannot promote {peer}: not a member of region {region_id}" + ))); + } + // Idempotent: a retry after a leadership change must not wedge a + // learner that is already promoted. + region.staging_peers.retain(|p| *p != peer); + region.epoch = epoch.bump_conf_ver(); + sort_space_regions(space); + Ok(()) +} + +pub(super) fn apply_remove_peer( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + from_peer: u64, + epoch: ShardEpoch, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("RemovePeer")); + } + if region.transfer_outstanding() { + return Err(ShardMapApplyError::Invalid( + "cannot remove peer while transfer debt is outstanding".into(), + )); + } + if !region.peers.contains(&from_peer) { + return Err(ShardMapApplyError::Invalid(format!( + "peer {from_peer} not in region" + ))); + } + if region.primary == from_peer { + return Err(ShardMapApplyError::Invalid(format!( + "cannot remove peer {from_peer}: it is the region primary" + ))); + } + if region.peers.len() <= 1 { + return Err(ShardMapApplyError::Invalid( + "cannot remove the last peer of a region".into(), + )); + } + let remaining_serving = region + .peers + .iter() + .filter(|p| **p != from_peer && region.is_serving_peer(**p)) + .count(); + if remaining_serving == 0 { + return Err(ShardMapApplyError::Invalid( + "cannot remove peer: no serving replica would remain".into(), + )); + } + region.peers.retain(|p| *p != from_peer); + // Drop the learner marker too, or a peer re-added later inherits a + // stale "not ready" flag that nothing will ever clear. + region.staging_peers.retain(|p| *p != from_peer); + region.epoch = epoch.bump_conf_ver(); + sort_space_regions(space); + Ok(()) +} + +pub(super) fn apply_transfer_primary( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + new_primary: u64, + epoch: ShardEpoch, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("TransferPrimary")); + } + // A learner holds only a prefix of the range until it is promoted. + // Making it primary is the data-less-primary bug: PrimaryRead then + // resolves to NOTHING, because resolve_region_peers refuses a + // learner, and every aggregate for the region fails with + // "no active peers" even though the region is serving and the + // node is a member. Observed exactly that way on region 64. + if region.staging_peers.contains(&new_primary) { + return Err(ShardMapApplyError::Invalid(format!( + "cannot make learner {new_primary} the primary of region {region_id}" + ))); + } + if !region.peers.contains(&new_primary) { + return Err(ShardMapApplyError::Invalid(format!( + "new primary {new_primary} not a peer" + ))); + } + region.primary = new_primary; + region.epoch = epoch.bump_conf_ver(); + sort_space_regions(space); + Ok(()) +} + +pub(super) fn apply_clear_verified( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + epoch: ShardEpoch, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("ClearVerified")); + } + region.transfer_verified = Some(true); + region.transfer_first_seen = None; + region.epoch = epoch.bump_conf_ver(); + sort_space_regions(space); + Ok(()) +} + +pub(super) fn apply_activate_region( + map: &mut ShardMap, + key: MeasurementKey, + region_id: u64, + epoch: ShardEpoch, +) -> Result<(), ShardMapApplyError> { + let space = map + .spaces + .get_mut(&key) + .ok_or(ShardMapApplyError::UnknownSpace)?; + let region = space + .regions + .iter_mut() + .find(|r| r.region_id == region_id) + .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; + if region.epoch != epoch { + return Err(ShardMapApplyError::StaleEpoch("ActivateRegion")); + } + // Idempotent rather than an error: a retried activation after a + // leadership change must not wedge the region in Staging. + region.state = RegionState::Serving; + region.staged_from = None; + region.epoch = epoch.bump_conf_ver(); + sort_space_regions(space); + Ok(()) +} diff --git a/hyperbytedb/src/domain/sharding/ops.rs b/hyperbytedb/src/domain/sharding/ops/mod.rs similarity index 75% rename from hyperbytedb/src/domain/sharding/ops.rs rename to hyperbytedb/src/domain/sharding/ops/mod.rs index fc082fb..7778801 100644 --- a/hyperbytedb/src/domain/sharding/ops.rs +++ b/hyperbytedb/src/domain/sharding/ops/mod.rs @@ -2,6 +2,8 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use super::types::{MeasurementKey, RegionState, ShardEpoch, ShardRegion}; +// For `super::types::…` paths in `apply.rs`, moved from the match arms. +use super::types; #[derive(Debug, Error)] pub enum ShardMapApplyError { @@ -234,6 +236,8 @@ fn sort_space_regions(space: &mut super::types::MeasurementShardSpace) { } /// Apply `op` to an in-memory map snapshot (used by Raft apply and unit tests). +mod apply; + pub fn apply_shard_map_op( map: &mut super::types::ShardMap, op: ShardMapOp, @@ -241,136 +245,24 @@ pub fn apply_shard_map_op( map.map_version = map.map_version.saturating_add(1); match op { ShardMapOp::BootstrapMeasurementPresplit { key, regions } => { - if map.spaces.contains_key(&key) { - return Err(ShardMapApplyError::SpaceExists { - db: key.db.clone(), - rp: key.rp.clone(), - measurement: key.measurement.clone(), - }); - } - let mut ids = std::collections::HashSet::new(); - for region in ®ions { - if !ids.insert(region.region_id) - || map.spaces.values().any(|space| { - space - .regions - .iter() - .any(|r| r.region_id == region.region_id) - }) - { - return Err(ShardMapApplyError::DuplicateRegionId(region.region_id)); - } - } - if let Some(max) = regions.iter().map(|r| r.region_id).max() { - map.next_region_id = map.next_region_id.max(max.saturating_add(1)); - } - let mut space = super::types::MeasurementShardSpace { - key: key.clone(), - regions, - }; - sort_space_regions(&mut space); - space.validate().map_err(ShardMapApplyError::Invalid)?; - map.spaces.insert(key, space); + apply::apply_bootstrap_measurement_presplit(map, key, regions) } ShardMapOp::BootstrapMeasurement { key, region } => { - if map.spaces.contains_key(&key) { - return Err(ShardMapApplyError::SpaceExists { - db: key.db.clone(), - rp: key.rp.clone(), - measurement: key.measurement.clone(), - }); - } - if map.spaces.values().any(|space| { - space - .regions - .iter() - .any(|r| r.region_id == region.region_id) - }) { - return Err(ShardMapApplyError::DuplicateRegionId(region.region_id)); - } - map.next_region_id = map.next_region_id.max(region.region_id.saturating_add(1)); - let mut space = super::types::MeasurementShardSpace { - key: key.clone(), - regions: vec![region], - }; - sort_space_regions(&mut space); - space.validate().map_err(ShardMapApplyError::Invalid)?; - map.spaces.insert(key, space); + apply::apply_bootstrap_measurement(map, key, region) } ShardMapOp::Split { key, region_id, split_key, epoch, - mut left, - mut right, - } => { - // Allocate region_id at apply time so concurrent split proposals cannot - // collide on a stale proposer-chosen id. - let allocated = map.next_region_id; - if map - .spaces - .values() - .any(|s| s.regions.iter().any(|r| r.region_id == allocated)) - { - return Err(ShardMapApplyError::DuplicateRegionId(allocated)); - } - right.region_id = allocated; - map.next_region_id = allocated.saturating_add(1); - - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let idx = space - .regions - .iter() - .position(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - let old = &space.regions[idx]; - if old.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("Split")); - } - if split_key <= old.start || split_key >= old.end { - return Err(ShardMapApplyError::Invalid(format!( - "split_key {split_key} not inside region [{}, {})", - old.start, old.end - ))); - } - if left.end != split_key || right.start != split_key { - return Err(ShardMapApplyError::Invalid( - "split regions must meet at split_key".into(), - )); - } - // Apply-time normalization: flag every child whose primary differs - // from the parent's (a metadata-only split never does). Nothing - // clears the flag any more: the reconciliation that re-pushed - // and verified flagged children was removed. `transfer_first_seen` propagates from the - // parent so re-splitting a flagged region cannot reset the age - // gate. Predicates are deterministic from (op, prior map), so all - // replicas normalize identically. - let inherited_first_seen = old.transfer_first_seen; - for child in [&mut left, &mut right] { - if child.primary != old.primary { - child.transfer_verified = Some(false); - child.transfer_first_seen = inherited_first_seen.or(child.transfer_first_seen); - } - } - space.regions[idx] = left; - space.regions.insert(idx + 1, right); - sort_space_regions(space); - space.validate().map_err(ShardMapApplyError::Invalid)?; - map.validate_global_region_ids() - .map_err(ShardMapApplyError::Invalid)?; - } + left, + right, + } => apply::apply_split(map, key, region_id, split_key, epoch, left, right), ShardMapOp::DropSpace { db, rp, measurement, - } => { - map.spaces - .retain(|key, _| !ShardMapOp::drop_matches(&db, &rp, &measurement, key)); - } + } => apply::apply_drop_space(map, db, rp, measurement), ShardMapOp::Merge { key, left_region_id, @@ -378,287 +270,58 @@ pub fn apply_shard_map_op( epoch, right_epoch, merged, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let li = space - .regions - .iter() - .position(|r| r.region_id == left_region_id) - .ok_or(ShardMapApplyError::UnknownLeftRegion(left_region_id))?; - let left_epoch = space.regions[li].epoch; - if left_epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("Merge")); - } - let ri = space - .regions - .iter() - .position(|r| r.region_id == right_region_id) - .ok_or(ShardMapApplyError::UnknownRightRegion(right_region_id))?; - if ri != li + 1 { - return Err(ShardMapApplyError::Invalid( - "merge regions must be adjacent".into(), - )); - } - // The right half's data was staged onto the merged owner based on - // its state at proposal time; a concurrently changed right region - // (MovePeer/TransferPrimary) invalidates that staging. - if space.regions[ri].epoch != right_epoch.unwrap_or(epoch) { - return Err(ShardMapApplyError::StaleEpoch("Merge right region")); - } - space.regions.remove(ri); - space.regions[li] = merged; - sort_space_regions(space); - space.validate().map_err(ShardMapApplyError::Invalid)?; - } + } => apply::apply_merge( + map, + key, + left_region_id, + right_region_id, + epoch, + right_epoch, + merged, + ), ShardMapOp::MovePeer { key, region_id, from_peer, to_peer, epoch, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("MovePeer")); - } - if !region.peers.contains(&from_peer) { - return Err(ShardMapApplyError::Invalid(format!( - "peer {from_peer} not in region" - ))); - } - if region.staging_peers.contains(&to_peer) { - return Err(ShardMapApplyError::Invalid(format!( - "cannot move peer to learner {to_peer}" - ))); - } - region.peers.retain(|p| *p != from_peer); - region.staging_peers.retain(|p| *p != from_peer); - if !region.peers.contains(&to_peer) { - region.peers.push(to_peer); - } - region.epoch = epoch.bump_conf_ver(); - if region.primary == from_peer { - region.primary = to_peer; - } - sort_space_regions(space); - } + } => apply::apply_move_peer(map, key, region_id, from_peer, to_peer, epoch), ShardMapOp::AddPeer { key, region_id, to_peer, epoch, staging, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("AddPeer")); - } - if region.transfer_outstanding() { - return Err(ShardMapApplyError::Invalid( - "cannot add peer while transfer debt is outstanding".into(), - )); - } - if region.peers.contains(&to_peer) { - return Err(ShardMapApplyError::Invalid(format!( - "peer {to_peer} already in region" - ))); - } - region.peers.push(to_peer); - if staging && !region.staging_peers.contains(&to_peer) { - region.staging_peers.push(to_peer); - } - region.epoch = epoch.bump_conf_ver(); - sort_space_regions(space); - } + } => apply::apply_add_peer(map, key, region_id, to_peer, epoch, staging), ShardMapOp::PromotePeer { key, region_id, peer, epoch, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("PromotePeer")); - } - if !region.peers.contains(&peer) { - return Err(ShardMapApplyError::Invalid(format!( - "cannot promote {peer}: not a member of region {region_id}" - ))); - } - // Idempotent: a retry after a leadership change must not wedge a - // learner that is already promoted. - region.staging_peers.retain(|p| *p != peer); - region.epoch = epoch.bump_conf_ver(); - sort_space_regions(space); - } + } => apply::apply_promote_peer(map, key, region_id, peer, epoch), ShardMapOp::RemovePeer { key, region_id, from_peer, epoch, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("RemovePeer")); - } - if region.transfer_outstanding() { - return Err(ShardMapApplyError::Invalid( - "cannot remove peer while transfer debt is outstanding".into(), - )); - } - if !region.peers.contains(&from_peer) { - return Err(ShardMapApplyError::Invalid(format!( - "peer {from_peer} not in region" - ))); - } - if region.primary == from_peer { - return Err(ShardMapApplyError::Invalid(format!( - "cannot remove peer {from_peer}: it is the region primary" - ))); - } - if region.peers.len() <= 1 { - return Err(ShardMapApplyError::Invalid( - "cannot remove the last peer of a region".into(), - )); - } - let remaining_serving = region - .peers - .iter() - .filter(|p| **p != from_peer && region.is_serving_peer(**p)) - .count(); - if remaining_serving == 0 { - return Err(ShardMapApplyError::Invalid( - "cannot remove peer: no serving replica would remain".into(), - )); - } - region.peers.retain(|p| *p != from_peer); - // Drop the learner marker too, or a peer re-added later inherits a - // stale "not ready" flag that nothing will ever clear. - region.staging_peers.retain(|p| *p != from_peer); - region.epoch = epoch.bump_conf_ver(); - sort_space_regions(space); - } + } => apply::apply_remove_peer(map, key, region_id, from_peer, epoch), ShardMapOp::TransferPrimary { key, region_id, new_primary, epoch, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("TransferPrimary")); - } - // A learner holds only a prefix of the range until it is promoted. - // Making it primary is the data-less-primary bug: PrimaryRead then - // resolves to NOTHING, because resolve_region_peers refuses a - // learner, and every aggregate for the region fails with - // "no active peers" even though the region is serving and the - // node is a member. Observed exactly that way on region 64. - if region.staging_peers.contains(&new_primary) { - return Err(ShardMapApplyError::Invalid(format!( - "cannot make learner {new_primary} the primary of region {region_id}" - ))); - } - if !region.peers.contains(&new_primary) { - return Err(ShardMapApplyError::Invalid(format!( - "new primary {new_primary} not a peer" - ))); - } - region.primary = new_primary; - region.epoch = epoch.bump_conf_ver(); - sort_space_regions(space); - } + } => apply::apply_transfer_primary(map, key, region_id, new_primary, epoch), ShardMapOp::ClearVerified { key, region_id, epoch, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("ClearVerified")); - } - region.transfer_verified = Some(true); - region.transfer_first_seen = None; - region.epoch = epoch.bump_conf_ver(); - sort_space_regions(space); - } + } => apply::apply_clear_verified(map, key, region_id, epoch), ShardMapOp::ActivateRegion { key, region_id, epoch, - } => { - let space = map - .spaces - .get_mut(&key) - .ok_or(ShardMapApplyError::UnknownSpace)?; - let region = space - .regions - .iter_mut() - .find(|r| r.region_id == region_id) - .ok_or(ShardMapApplyError::UnknownRegion(region_id))?; - if region.epoch != epoch { - return Err(ShardMapApplyError::StaleEpoch("ActivateRegion")); - } - // Idempotent rather than an error: a retried activation after a - // leadership change must not wedge the region in Staging. - region.state = RegionState::Serving; - region.staged_from = None; - region.epoch = epoch.bump_conf_ver(); - sort_space_regions(space); - } + } => apply::apply_activate_region(map, key, region_id, epoch), } - Ok(()) } impl From for crate::error::HyperbytedbError { @@ -689,6 +352,58 @@ mod tests { } } + /// A refused op still bumps `map_version`: the bump happens before the + /// op is checked, so every Raft apply moves the version. + #[test] + fn refused_op_still_bumps_version() { + let key = MeasurementKey::new("db", "autogen", "cpu"); + let split_key = u64::MAX / 2; + let mut map = ShardMap::default(); + apply_shard_map_op( + &mut map, + ShardMapOp::BootstrapMeasurement { + key: key.clone(), + region: sample_region(1, 0, u64::MAX, 1), + }, + ) + .unwrap(); + apply_shard_map_op( + &mut map, + ShardMapOp::Split { + key: key.clone(), + region_id: 1, + split_key, + epoch: ShardEpoch::default(), + left: sample_region(1, 0, split_key, 1), + right: sample_region(2, split_key, u64::MAX, 1), + }, + ) + .unwrap(); + map.map_version = 5; + let left_epoch = map.spaces[&key].regions[0].epoch; + let stale = ShardEpoch { + version: left_epoch.version + 7, + ..left_epoch + }; + let err = apply_shard_map_op( + &mut map, + ShardMapOp::Merge { + key: key.clone(), + left_region_id: 1, + right_region_id: 2, + epoch: stale, + right_epoch: None, + merged: sample_region(1, 0, u64::MAX, 1), + }, + ) + .unwrap_err(); + assert!( + matches!(err, ShardMapApplyError::StaleEpoch("Merge")), + "{err:?}" + ); + assert_eq!(map.map_version, 6); + } + #[test] fn merge_adjacent_regions() { let key = MeasurementKey::new("db", "autogen", "cpu"); diff --git a/hyperbytedb/tests/compat/cluster_replication_tests.rs b/hyperbytedb/tests/compat/cluster_replication_tests.rs new file mode 100644 index 0000000..7432ce2 --- /dev/null +++ b/hyperbytedb/tests/compat/cluster_replication_tests.rs @@ -0,0 +1,171 @@ +//! Golden record of what each cluster statement replicates (non-Raft +//! cluster): the mutation `PeerQueryService` sends for every statement +//! kind, and the statement's own result. Regenerate the fixture with +//! `UPDATE_GOLDEN=1` only when a change to it is intended. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use hyperbytedb::adapters::http::router::QueryService; +use hyperbytedb::application::peer_query_service::PeerQueryService; +use hyperbytedb::domain::cluster::types::MutationRequest; +use hyperbytedb::error::HyperbytedbError; +use hyperbytedb::ports::replication::{OutboundReplicationBatch, ReplicationPort}; +use serial_test::serial; + +use super::TestContext; + +/// Records every mutation it is asked to replicate. +#[derive(Default)] +struct RecordingReplication { + seen: parking_lot::Mutex>, +} + +#[async_trait] +impl ReplicationPort for RecordingReplication { + fn replicate_write( + self: Arc, + _batch: OutboundReplicationBatch, + ) -> Result<(), HyperbytedbError> { + Ok(()) + } + + async fn replicate_write_sync( + self: Arc, + _batch: OutboundReplicationBatch, + _required_acks: usize, + _timeout: Duration, + ) -> Result<(), HyperbytedbError> { + Ok(()) + } + + fn replicate_mutation(self: Arc, req: MutationRequest, _targets: Option>) { + self.seen.lock().push(req); + } + + async fn replicate_mutation_sync( + self: Arc, + req: MutationRequest, + ) -> Result<(), HyperbytedbError> { + self.seen.lock().push(req); + Ok(()) + } + + async fn replication_peer_count(&self, _self_node_id: u64) -> usize { + 0 + } +} + +/// One statement per replicating arm, plus the cross-database CREATE +/// RETENTION POLICY and an invalid SET. Each is run against `mydb`. +const STATEMENTS: &[&str] = &[ + "CREATE DATABASE mydb", + "CREATE DATABASE otherdb", + "CREATE RETENTION POLICY week ON mydb DURATION 7d REPLICATION 1", + "CREATE RETENTION POLICY r ON otherdb DURATION 1d REPLICATION 1", + "ALTER RETENTION POLICY week ON mydb DURATION 14d", + "CREATE USER bob WITH PASSWORD 'secret-one'", + "SET PASSWORD FOR bob = 'secret-two'", + "GRANT READ ON mydb TO bob", + "REVOKE READ ON mydb FROM bob", + "CREATE CONTINUOUS QUERY cq1 ON mydb BEGIN SELECT mean(v) INTO cpu_1m FROM cpu GROUP BY time(1m) END", + "DROP CONTINUOUS QUERY cq1 ON mydb", + r#"CREATE MATERIALIZED VIEW "mv1" ON "mydb" AS SELECT mean("v") AS "mean_v" INTO "cpu_5m" FROM "cpu" GROUP BY time(5m), *"#, + r#"DROP MATERIALIZED VIEW "mv1" ON "mydb""#, + "DELETE FROM cpu WHERE host = 'a'", + "DROP SERIES FROM cpu WHERE host = 'b'", + "DROP MEASUREMENT cpu", + "SET query_pool_size = 5", + "SET query_pool_size = 0", + "DROP USER bob", + "DROP RETENTION POLICY week ON mydb", + "DROP DATABASE otherdb", +]; + +/// Replace each `password_hash` with a placeholder, after checking it is an +/// argon2 hash and not the plain password, and each `created_at` (wall +/// clock) with another. +fn mask_hashes(v: &mut serde_json::Value) { + match v { + serde_json::Value::Object(map) => { + for (k, val) in map.iter_mut() { + if k == "password_hash" { + let h = val.as_str().expect("password_hash is a string"); + assert!(h.starts_with("$argon2"), "not an argon2 hash: {h}"); + assert!(!h.contains("secret"), "plain password replicated"); + *val = serde_json::Value::String("".into()); + } else if k == "created_at" { + *val = serde_json::Value::String("