Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions hyperbytedb/src/adapters/chdb/native_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ use parking_lot::RwLock;
use crate::adapters::chdb::catalog;
use crate::adapters::chdb::connection_pool::ChdbConnectionPool;
use crate::adapters::chdb::session::{SharedSession, execute_connection};
use crate::application::ingest_metadata::backfill_tag_metadata;
use crate::domain::chdb_naming::{
field_column_name, quote_backticks, quoted_series_table_name, quoted_table_name,
tag_column_name, unquoted_series_table_name, unquoted_table_name,
Expand Down Expand Up @@ -543,7 +542,8 @@ impl ChdbNativeAdapter {
tag_pairs.push((k.clone(), v.clone()));
}
}
backfill_tag_metadata(meta, &key.db, &key.rp, &key.measurement, tag_pairs).await
meta.backfill_tag_keys(&key.db, &key.rp, &key.measurement, tag_pairs)
.await
}

async fn ddl_mutex(&self, key: &TableKey) -> Arc<tokio::sync::Mutex<()>> {
Expand Down Expand Up @@ -1285,7 +1285,7 @@ impl ChdbNativeAdapter {
wire: &crate::domain::columnar_batch::ColumnarMsgpackBatch,
precision: Option<&str>,
) -> Result<crate::domain::prepared_wal::PreparedWalSlot, HyperbytedbError> {
use crate::application::columnar_msgpack::columnar_timestamps_ns;
use crate::domain::columnar_batch::columnar_timestamps_ns;
use crate::domain::prepared_wal::{PreparedMeasurementBatch, PreparedWalSlot};
use crate::domain::series::series_id;

Expand Down Expand Up @@ -1374,9 +1374,8 @@ impl ChdbNativeAdapter {

use crate::domain::prepared_wal::PreparedMeasurementBatch;

let mut stage = crate::application::stage_timer::StageTimer::new(
"hyperbytedb_prepare_measurement_stage_seconds",
);
let mut stage =
crate::telemetry::StageTimer::new("hyperbytedb_prepare_measurement_stage_seconds");
let ensured = self.ensure_table(key, points).await?;
stage.lap("ensure_table");
let origins = vec![origin_node_id; points.len()];
Expand Down Expand Up @@ -2340,7 +2339,7 @@ fn columnar_representative_point(
wire: &crate::domain::columnar_batch::ColumnarMsgpackBatch,
precision: Option<&str>,
) -> Result<Point, HyperbytedbError> {
use crate::application::columnar_msgpack::columnar_timestamps_ns;
use crate::domain::columnar_batch::columnar_timestamps_ns;

let ts = columnar_timestamps_ns(wire, precision)?;
let mut fields = BTreeMap::new();
Expand Down
47 changes: 47 additions & 0 deletions hyperbytedb/src/adapters/cluster/http_peer_rpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,16 @@ impl HttpPeerRpc {
Err(PeerRpcError::Status { code, body })
}

/// The body as JSON `T`; a body that does not parse is `Unreachable`.
async fn decode<T: serde::de::DeserializeOwned>(
resp: reqwest::Response,
what: &str,
) -> Result<T, PeerRpcError> {
let body = Self::body(resp).await?;
serde_json::from_slice(&body)
.map_err(|e| PeerRpcError::Unreachable(format!("parse {what}: {e}")))
}

async fn body(resp: reqwest::Response) -> Result<Bytes, PeerRpcError> {
resp.bytes()
.await
Expand Down Expand Up @@ -377,6 +387,43 @@ impl PeerRpc for HttpPeerRpc {
})
}

async fn join(
&self,
addr: &str,
req: &crate::domain::cluster::sync::JoinRequest,
timeout: Duration,
) -> Result<crate::domain::cluster::sync::JoinResponse, PeerRpcError> {
let resp = self
.post_json(addr, "/internal/membership/join", req, timeout)
.await?;
Self::decode(resp, "join response").await
}

async fn sync_metadata_snapshot(
&self,
addr: &str,
timeout: Duration,
) -> Result<crate::domain::cluster::sync::MetadataSnapshot, PeerRpcError> {
let resp = self.get(addr, "/internal/sync/metadata", timeout).await?;
Self::decode(resp, "metadata snapshot").await
}

async fn sync_wal(
&self,
addr: &str,
from_seq: u64,
max_entries: u32,
timeout: Duration,
) -> Result<crate::domain::cluster::sync::WalSyncResponse, PeerRpcError> {
let path = format!("/internal/sync/wal?from_seq={from_seq}&max_entries={max_entries}");
let resp = self.get(addr, &path, timeout).await?;
Self::decode(resp, "WAL sync response").await
}

async fn ping(&self, addr: &str, timeout: Duration) -> Result<(), PeerRpcError> {
self.get(addr, "/ping", timeout).await.map(|_| ())
}

async fn probe(&self, addr: &str, timeout: Duration) -> ProbeReply {
let get = |path: &str| {
self.client()
Expand Down
1 change: 0 additions & 1 deletion hyperbytedb/src/adapters/cluster/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,3 @@ pub mod peer_client;
pub mod peer_http;
pub mod raft;
pub mod replication_log;
pub mod sync_client;
7 changes: 0 additions & 7 deletions hyperbytedb/src/adapters/cluster/peer_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,6 @@ impl PeerClient {
self
}

/// A peer's committed `map_version`, read from `/internal/shard/map`.
pub async fn fetch_map_version(&self, peer_addr: &str) -> Result<u64, HyperbytedbError> {
let map =
super::sync_client::fetch_shard_map(self.http_client().clone(), peer_addr).await?;
Ok(map.map_version)
}

/// The client for the next peer request (round-robin over connections).
pub fn http_client(&self) -> &reqwest::Client {
let i = self
Expand Down
39 changes: 23 additions & 16 deletions hyperbytedb/src/adapters/http/peer_handlers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -909,22 +909,29 @@ pub async fn handle_sync_trigger(State(state): State<Arc<AppState>>) -> impl Int
};

tokio::spawn(async move {
let mut sync_client = crate::adapters::cluster::sync_client::SyncClient::with_points_sink(
node_id,
{
let m = membership_clone.read().await;
m.get_node(node_id)
.map(|n| n.addr.clone())
.unwrap_or_default()
},
membership_clone.clone(),
metadata.clone(),
wal.clone(),
Some(points_sink),
max_points_per_request,
fallback_peers,
&cluster_auth,
);
let mut sync_client =
crate::application::cluster::catch_up::CatchUpService::with_points_sink(
node_id,
{
let m = membership_clone.read().await;
m.get_node(node_id)
.map(|n| n.addr.clone())
.unwrap_or_default()
},
membership_clone.clone(),
metadata.clone(),
wal.clone(),
Some(points_sink),
max_points_per_request,
fallback_peers,
Arc::new(
crate::adapters::cluster::http_peer_rpc::HttpPeerRpc::with_client({
use crate::adapters::cluster::peer_http::PeerHttp;
let auth = &cluster_auth;
auth.build(auth.client_builder())
}),
),
);
sync_client = sync_client.with_runtime_settings(runtime_settings);
// Only a sharded node syncs a shard map. `state.shard_map` is a
// `DisabledShardMap` otherwise, never `None`: attaching it made a
Expand Down
6 changes: 2 additions & 4 deletions hyperbytedb/src/adapters/http/shard_handlers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -420,8 +420,7 @@ async fn apply_shard_write_batch(
}
};

let mut stage =
crate::application::stage_timer::StageTimer::new("hyperbytedb_shard_primary_stage_seconds");
let mut stage = crate::telemetry::StageTimer::new("hyperbytedb_shard_primary_stage_seconds");
// Check and parse each part; group the good ones by (db, rp).
let mut groups: Vec<ForwardGroup> = Vec::new();
for (index, req) in parts.into_iter().enumerate() {
Expand Down Expand Up @@ -879,8 +878,7 @@ pub async fn handle_shard_query(
)
.into_response()
};
let mut stage =
crate::application::stage_timer::StageTimer::new("hyperbytedb_shard_query_stage_seconds");
let mut stage = crate::telemetry::StageTimer::new("hyperbytedb_shard_query_stage_seconds");
let mut batches = match state.query_port.execute_sql_arrow(&sql).await {
Ok(b) => b,
Err(e) => return failed(e),
Expand Down
41 changes: 41 additions & 0 deletions hyperbytedb/src/adapters/metadata/rocksdb_meta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2659,6 +2659,47 @@ impl MetadataPort for RocksDbMetadata {
mod series_range_tests {
use super::*;

/// Backfill adds the keys a measurement lacks and keeps the list sorted.
#[tokio::test]
async fn backfill_adds_missing_tag_keys() {
let dir = tempfile::tempdir().unwrap();
let meta = RocksDbMetadata::open(dir.path()).unwrap();
let cpu = MeasurementMeta {
name: "cpu".into(),
tag_keys: vec!["host".into()],
..Default::default()
};
meta.register_metadata_batch("db", "autogen", &[cpu], &[])
.await
.unwrap();
meta.backfill_tag_keys("db", "autogen", "cpu", vec![("dc".into(), "x".into())])
.await
.unwrap();
let got = meta
.get_measurement("db", "autogen", "cpu")
.await
.unwrap()
.unwrap();
assert_eq!(got.tag_keys, ["dc", "host"]);
}

/// Backfill for a measurement not yet known creates it with the keys.
#[tokio::test]
async fn backfill_creates_unknown_measurement() {
let dir = tempfile::tempdir().unwrap();
let meta = RocksDbMetadata::open(dir.path()).unwrap();
meta.backfill_tag_keys("db", "autogen", "mem", vec![("host".into(), "a".into())])
.await
.unwrap();
let got = meta
.get_measurement("db", "autogen", "mem")
.await
.unwrap()
.unwrap();
assert_eq!(got.tag_keys, ["host"]);
assert!(got.field_types.is_empty());
}

/// A tag value already on disk is not new after a restart: rewriting
/// host=a after one counted it again (SHOW TAG VALUES CARDINALITY 3 for
/// a and b) and made it look new to the cardinality limit.
Expand Down
Loading