Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ time = ">=0.3.55, <0.4"
hyper-util = { version = ">=0.1.21, <0.2", features = ["client-legacy", "http1", "tokio"] }
hyper-rustls = { version = ">=0.27.10, <0.28", default-features = false, features = ["http1", "tls12", "native-tokio", "aws-lc-rs"] }
rustls = { version = ">=0.23.46, <0.24", default-features = false, features = ["std", "tls12", "aws_lc_rs"] }
# The connector trait http_transport's endpoint-naming wrapper implements. Same
# range as the copy hyper-util resolves, so cargo unifies to one.
tower-service = ">=0.3.3, <0.4"

uuid = { version = ">=1.29.2, <2", features = ["v4", "v7"] }
rustc-hash = ">=2.1.3, <3"
Expand Down
174 changes: 174 additions & 0 deletions src/clickhouse/client_http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@
//! - Table existence checks
//! - Health checks (ping)

use std::fmt;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use std::task::{Context, Poll};
use std::time::Duration;

use dfe_clickhouse::TcpClient;
Expand All @@ -36,6 +39,7 @@ use hyper_util::client::legacy::connect::HttpConnector;
use hyper_util::rt::{TokioExecutor, TokioTimer};
use rustc_hash::FxHashMap;
use serde::Deserialize;
use tower_service::Service;

use super::config::{ClickHouseConfig, Timeouts};
use super::error::ClickHouseError;
Expand Down Expand Up @@ -224,6 +228,78 @@ pub(crate) fn insert_sink(
.with_timeouts(Some(timeouts.request), Some(timeouts.insert))
}

type BoxError = Box<dyn std::error::Error + Send + Sync>;

/// A failed connect for the endpoint it was for.
///
/// The connector's own error says what failed -- a lookup, a refusal, a
/// certificate -- but not for which host, and the client's text for it is
/// `client error (Connect)`. This sits above the connector's error in the
/// source chain, so every log line built from [`error_text`] names the host.
///
/// [`error_text`]: super::error::error_text
#[derive(Debug)]
struct ConnectFailed {
endpoint: Arc<str>,
source: BoxError,
}

impl fmt::Display for ConnectFailed {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "connecting to {}", self.endpoint)
}
}

impl std::error::Error for ConnectFailed {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(self.source.as_ref())
}
}

/// A connector that fails with [`ConnectFailed`] for the endpoint it was built for.
#[derive(Clone)]
struct AtEndpoint<C> {
endpoint: Arc<str>,
inner: C,
}

impl<C, R> Service<R> for AtEndpoint<C>
where
C: Service<R>,
C::Response: Send,
C::Future: Send + 'static,
C::Error: Into<BoxError>,
{
type Response = C::Response;
type Error = BoxError;
type Future = Pin<Box<dyn Future<Output = std::result::Result<C::Response, BoxError>> + Send>>;

fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), BoxError>> {
let endpoint = &self.endpoint;
self.inner.poll_ready(cx).map_err(|source| {
ConnectFailed {
endpoint: Arc::clone(endpoint),
source: source.into(),
}
.into()
})
}

fn call(&mut self, request: R) -> Self::Future {
let connecting = self.inner.call(request);
let endpoint = Arc::clone(&self.endpoint);
Box::pin(async move {
connecting.await.map_err(|source| {
ConnectFailed {
endpoint,
source: source.into(),
}
.into()
})
})
}
}

/// The HTTP client a `clickhouse::Client` sends through.
///
/// HTTPS verifies the server's certificate and host name against the system
Expand All @@ -232,11 +308,16 @@ fn http_transport(config: &ClickHouseConfig) -> Result<clickhouse::Client> {
let mut connector = HttpConnector::new();
connector.set_keepalive(Some(TCP_KEEPALIVE));
connector.set_connect_timeout(Some(config.connect_timeout()));
let endpoint: Arc<str> = config.primary_endpoint().unwrap_or_default().into();
let mut pool = HyperClient::builder(TokioExecutor::new());
// hyper-util evicts idle connections only when the pool has a timer.
pool.pool_timer(TokioTimer::new())
.pool_idle_timeout(POOL_IDLE_TIMEOUT);
if !config.tls {
let connector = AtEndpoint {
endpoint,
inner: connector,
};
return Ok(clickhouse::Client::with_http_client(pool.build(connector)));
}
connector.enforce_http(false);
Expand All @@ -250,6 +331,10 @@ fn http_transport(config: &ClickHouseConfig) -> Result<clickhouse::Client> {
.https_only()
.enable_http1()
.wrap_connector(connector);
let https = AtEndpoint {
endpoint,
inner: https,
};
Ok(clickhouse::Client::with_http_client(pool.build(https)))
}

Expand Down Expand Up @@ -1460,4 +1545,93 @@ mod tests {
"the query gave up after {elapsed:?}, not at its 300 ms bound"
);
}

/// The text a health check against `config` fails with, bounded so a
/// resolver that never answers fails the test and does not hang it.
async fn health_check_failure(config: &ClickHouseConfig) -> String {
let client = ClickHouseQueryClient::new(config).unwrap();
tokio::time::timeout(Duration::from_secs(20), client.health_check())
.await
.expect("the health check ran past 20 s")
.expect_err("the health check reported success")
.to_string()
}

/// A TLS server whose certificate no trust store holds, so a verifying
/// client refuses it mid-handshake.
fn self_signed_tls_server() -> SocketAddr {
let certified = rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).unwrap();
let key =
rustls::pki_types::PrivatePkcs8KeyDer::from(certified.signing_key.serialize_der());
let tls = Arc::new(
rustls::ServerConfig::builder_with_provider(Arc::new(
rustls::crypto::aws_lc_rs::default_provider(),
))
.with_safe_default_protocol_versions()
.unwrap()
.with_no_client_auth()
.with_single_cert(vec![certified.cert.der().clone()], key.into())
.unwrap(),
);
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
std::thread::spawn(move || {
let (mut tcp, _) = listener.accept().unwrap();
let mut conn = rustls::ServerConnection::new(tls).unwrap();
// The client aborts on the certificate, so this ends in an error.
while conn.is_handshaking() && conn.complete_io(&mut tcp).is_ok() {}
});
addr
}

#[tokio::test]
async fn a_refused_connection_names_the_address_it_was_refused_at() {
let closed = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = closed.local_addr().unwrap();
drop(closed);

let text = health_check_failure(&http_config(addr, 2_000, 2_000)).await;

assert!(text.contains(&format!("connecting to {addr}")), "{text}");
assert!(text.contains("Connection refused"), "{text}");
assert!(!text.contains("dns error"), "{text}");
assert!(!text.contains("certificate"), "{text}");
}

#[tokio::test]
async fn a_name_that_does_not_resolve_is_reported_as_a_lookup_failure() {
// `.invalid` never resolves (RFC 6761), with or without network.
let config = ClickHouseConfig {
hosts: vec!["clickhouse.invalid:8123".to_string()],
transport: super::super::config::Transport::Http,
database: "default".to_string(),
connect_timeout_ms: 2_000,
request_timeout_ms: 2_000,
..Default::default()
};

let text = health_check_failure(&config).await;

assert!(
text.contains("connecting to clickhouse.invalid:8123"),
"{text}"
);
assert!(text.contains("dns error"), "{text}");
assert!(!text.contains("Connection refused"), "{text}");
assert!(!text.contains("certificate"), "{text}");
}

#[tokio::test]
async fn an_untrusted_certificate_is_reported_with_the_verify_error() {
let addr = self_signed_tls_server();
let mut config = http_config(addr, 2_000, 2_000);
config.tls = true;

let text = health_check_failure(&config).await;

assert!(text.contains(&format!("connecting to {addr}")), "{text}");
assert!(text.contains("invalid peer certificate"), "{text}");
assert!(!text.contains("dns error"), "{text}");
assert!(!text.contains("Connection refused"), "{text}");
}
}
55 changes: 53 additions & 2 deletions src/clickhouse/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -346,10 +346,30 @@ const ERROR_TEXT_CAP: usize = 1 << 20;
pub fn error_text(err: &clickhouse::error::Error) -> String {
capped(match ServerException::parse(err) {
Some(exception) => server_text(exception.code, &exception.message),
None => err.to_string(),
None => with_causes(err),
})
}

/// `err` and each cause beneath it, joined by `: `. The client wraps a failed
/// connect as `network error: client error (Connect)` and keeps what the
/// transport said -- a failed lookup, a refusal, a certificate fault -- only
/// as the error's source, so `to_string()` alone drops it.
fn with_causes(err: &(dyn std::error::Error + 'static)) -> String {
let mut text = err.to_string();
let mut cause = err.source();
while let Some(next) = cause {
let part = next.to_string();
// An error that spells its source into its own message (`{0}` beside
// `#[source]`) must not print it twice.
if !text.contains(&part) {
text.push_str(": ");
text.push_str(&part);
}
cause = next.source();
}
text
}

/// [`error_text`] for an error off the native protocol, which carries the
/// server's code structurally.
#[must_use]
Expand All @@ -359,7 +379,7 @@ pub fn native_error_text(err: &dfe_clickhouse::Error) -> String {
dfe_clickhouse::Error::BadResponse(body) => {
return error_text(&clickhouse::error::Error::BadResponse(body.clone()));
}
other => other.to_string(),
other => with_causes(other),
})
}

Expand Down Expand Up @@ -447,6 +467,37 @@ pub fn classify_from_message(msg: &str) -> ErrorCategory {
mod tests {
use super::*;

#[test]
fn error_text_keeps_a_cause_the_message_leaves_out_and_prints_one_it_repeats() {
// `Network` spells its source into its own message, so the chain
// must not print it twice.
let repeated = clickhouse::error::Error::Network("socket closed".into());
assert_eq!(error_text(&repeated), "network error: socket closed");

let layered = clickhouse::error::Error::Network(Box::new(std::io::Error::other("outer")));
assert_eq!(error_text(&layered), "network error: outer");

#[derive(Debug)]
struct Wrapper(std::io::Error);
impl std::fmt::Display for Wrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("client error (Connect)")
}
}
impl std::error::Error for Wrapper {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.0)
}
}
let hidden = clickhouse::error::Error::Network(Box::new(Wrapper(std::io::Error::other(
"tcp connect error",
))));
assert_eq!(
error_text(&hidden),
"network error: client error (Connect): tcp connect error"
);
}

#[test]
fn test_connection_error_is_transient() {
let err = ClickHouseError::Connection("network timeout".into());
Expand Down
Loading