From 02686a334fe6d647b3c218e4c840f6fd5b9391cc Mon Sep 17 00:00:00 2001 From: Derek Date: Sat, 21 Nov 2026 20:22:07 +1100 Subject: [PATCH] fix: use scalo config and drop refused DLQ entries --config still wins. Without it, scalo 2.14.33 reads the file DFE_LOADER_CONFIG names. scalo ignores a blank DFE_LOADER_CONFIG. The hot-reload watcher uses CommonArgs::config_path. The loader's resolve_config_path is gone. On DlqError::Refused, place_dead_letters drops the entries the error lists. The loader counts them in pipeline_dead_letters_dropped_total with reason too_large. The rest of the write counts as placed. When the broker refuses a dead letter, the batch can move on before the retry expires. Every other DLQ error keeps its retry. Both scalo minimum versions are 2.14.33 to pick up DlqError::Refused. --- Cargo.toml | 4 +- src/config/loader.rs | 20 +-- src/main.rs | 6 +- src/pipeline/orchestrator.rs | 158 ++++++++++++++++++++--- tests/integration/config_reachability.rs | 103 ++++++++++----- 5 files changed, 214 insertions(+), 77 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76123928..6c096708 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ tokio = { version = ">=1.54.1, <2", features = ["full"] } tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] } # The floor moves with `release.helm.library` in .hyperi-ci.yaml. -scalo = { version = ">=2.14.32, <3", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack", "geoip-download"] } +scalo = { version = ">=2.14.33, <3", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack", "geoip-download"] } # Steps scalo's jittered retry schedule (sink_stack backoff). Range MUST track # scalo's: BackoffBuilder only applies to the ExponentialBuilder scalo returns. @@ -105,7 +105,7 @@ test-case = ">=3.4.1, <4" reqwest = { version = ">=0.12, <0.13", default-features = false, features = ["blocking", "rustls-tls-native-roots"] } # The Kafka tests' broker image pin and TLS/SASL broker fixture. -scalo = { version = ">=2.14.32, <3", features = ["transport-kafka-test-support"] } +scalo = { version = ">=2.14.33, <3", features = ["transport-kafka-test-support"] } # The status code a raw Push is refused with, through scalo's generated gRPC # client. Same range as the copy scalo resolves, so cargo unifies to one. diff --git a/src/config/loader.rs b/src/config/loader.rs index 81d61978..2fcf2940 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -9,7 +9,7 @@ //! 3. Figment env vars, `__` nesting, either separator after the prefix //! (`DFE_LOADER__KAFKA__BROKERS` or `DFE_LOADER_KAFKA__BROKERS`) //! 4. `./.env` in the working directory, read by scalo's cascade at startup -//! 5. Config file specified by --config or `DFE_LOADER_CONFIG` +//! 5. Config file specified by --config or `DFE_LOADER_CONFIG`, resolved by scalo's `run_app` //! 6. Hard-coded defaults use std::path::Path; @@ -556,15 +556,14 @@ impl Config { /// (chart / deployment-contract form) or `DFE_LOADER_KAFKA__SASL__USERNAME` /// 4. `.env` file: the binary's `load_config` seeds scalo's cascade first, /// which reads `./.env` and no parent directory's - /// 5. Config file (YAML, specified by `--config` or auto-detected) + /// 5. Config file (YAML): `config_path` is what scalo's `run_app` resolved from + /// `--config` or `DFE_LOADER_CONFIG`, else auto-detected /// 6. Hard-coded defaults pub fn load(config_path: Option<&str>) -> Result { // 1. Start with hard-coded defaults let mut config = Config::default(); // 2. Load YAML config file (overrides defaults). - let resolved = Self::resolve_config_path(config_path); - let config_path = resolved.as_deref(); if let Some(path) = config_path { // A named file must be read: running on defaults instead hides a // missing mount, and a SIGHUP reload would swap them in live. @@ -595,19 +594,6 @@ impl Config { Ok(config) } - /// Resolve the config file: `--config`, else `DFE_LOADER_CONFIG`. - /// - /// The env var is step 5 of this module's cascade and nothing read it -- - /// scalo's `--config` arg carries no `env =`. The watcher resolves here too. - #[must_use] - pub fn resolve_config_path(cli_path: Option<&str>) -> Option { - cli_path.map(String::from).or_else(|| { - std::env::var(format!("{ENV_PREFIX}_CONFIG")) - .ok() - .filter(|p| !p.is_empty()) - }) - } - /// Register all config sections in the global config registry. /// /// Enables `/config` endpoint dump (with redaction) and change notifications. diff --git a/src/main.rs b/src/main.rs index 03d422aa..01aa254e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,9 +87,9 @@ impl ServiceApp for App { config: Self::Config, mut runtime: ServiceRuntime, ) -> impl std::future::Future> + Send { - // Same resolution Config::load() uses, so a config file named by - // DFE_LOADER_CONFIG is hot-reloaded like one named by --config. - let config_path = Config::resolve_config_path(self.common_args().config.as_deref()); + // The file run_app loaded, so one named by DFE_LOADER_CONFIG is + // hot-reloaded like one named by --config. + let config_path = self.common_args().config_path(self.env_prefix()); async move { let mut config = config; diff --git a/src/pipeline/orchestrator.rs b/src/pipeline/orchestrator.rs index fe1029b4..8bb9ec3e 100644 --- a/src/pipeline/orchestrator.rs +++ b/src/pipeline/orchestrator.rs @@ -1731,6 +1731,12 @@ impl Orchestrator { self.count_placed(&dead_letters); self.dlq_placed.extend(placed_offsets(&dead_letters)); } + Placement::Refused { dropped, placed } => { + self.drop_broker_refused(&dropped); + self.count_placed(&placed); + self.dlq_placed.extend(placed_offsets(&dropped)); + self.dlq_placed.extend(placed_offsets(&placed)); + } Placement::Closed => { warn!( rows = dead_letters.len(), @@ -2134,6 +2140,16 @@ impl Orchestrator { } } + /// Drop dead letters the DLQ's broker refused for their size once every + /// delivery report was in, so no retry can land them. + fn drop_broker_refused(&mut self, dropped: &[DlqEntry]) { + warn!( + rows = dropped.len(), + "The DLQ's broker refused dead letters for their size -- dropped, counted in pipeline_dead_letters_dropped_total and dfe_loader_rows_lost_total" + ); + self.drop_dead_letters(dropped, DROPPED_TOO_LARGE); + } + /// Drop every dead letter read from a `dlq_load` topic, and return the rest. /// /// Dead-lettering one again would feed it back to the topic it was read @@ -2257,6 +2273,11 @@ impl Orchestrator { self.count_placed(&dead_letters); self.settle_dead_letters_as(&dead_letters, DeliveryStatus::Rejected); } + Placement::Refused { dropped, placed } => { + self.drop_broker_refused(&dropped); + self.count_placed(&placed); + self.settle_dead_letters_as(&placed, DeliveryStatus::Rejected); + } Placement::Closed => { warn!( rows = dead_letters.len(), @@ -2817,6 +2838,10 @@ const DROPPED_NO_PAYLOAD: &str = "no_payload"; /// `reason` for dead letters read from a `dlq_load` topic that failed to load. const DROPPED_DLQ_LOAD: &str = "dlq_load"; +/// The `pipeline_dead_letters_dropped_total` reason for a dead letter the DLQ's +/// broker refused for its size, as scalo's pipeline labels it. +const DROPPED_TOO_LARGE: &str = "too_large"; + /// The record sequence number a gRPC offset carries: the adapter stores each /// token's `seq` as the message offset. fn seq_of(offset: i64) -> u64 { @@ -3153,10 +3178,16 @@ fn queue_rejected<'a>( } /// How handing a set of dead letters to the DLQ ended. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug)] enum Placement { /// A backend holds every one. Written, + /// The Kafka-only DLQ's broker holds `placed` and refused `dropped` for + /// their size: no retry can land those, so they are dropped. + Refused { + dropped: Vec, + placed: Vec, + }, /// The DLQ's drain has exited, so no retry can land them. Closed, /// The DLQ still refused when the retry's expiry passed, or at shutdown. @@ -3209,6 +3240,16 @@ async fn place_dead_letters( return Placement::Written; } Ok(Err(scalo::dlq::DlqError::Closed)) => return Placement::Closed, + Ok(Err(scalo::dlq::DlqError::Refused(refused))) => { + let (dropped, placed) = split_refused(dead_letters, |entry| { + refused.contains(&dlq.cap(entry.clone())) + }); + return if dropped.is_empty() { + Placement::Written + } else { + Placement::Refused { dropped, placed } + }; + } Ok(Err(e)) => e.to_string(), Err(_elapsed) => { return Placement::GaveUp( @@ -3234,6 +3275,17 @@ async fn place_dead_letters( } } +/// Split `dead_letters` into those `is_refused` names and the rest. +fn split_refused( + dead_letters: &[DlqEntry], + is_refused: impl Fn(&DlqEntry) -> bool, +) -> (Vec, Vec) { + dead_letters + .iter() + .cloned() + .partition(|entry| is_refused(entry)) +} + /// The `stage` label a DLQ write retry is counted under, as scalo's pipeline /// names it. const DLQ_STAGE: &str = "dlq"; @@ -4024,9 +4076,11 @@ mod tests { let dead_letters = rejected_entries(&payloads, &rejected_rows(rows, "server error code 117")); - assert_eq!( - place(&dlq, &dead_letters, in_30s()).await, - Placement::Written, + assert!( + matches!( + place(&dlq, &dead_letters, in_30s()).await, + Placement::Written + ), "a working DLQ takes every rejected row" ); @@ -4345,9 +4399,11 @@ mod tests { let payloads: Vec> = vec![Arc::from(&b"{\"n\":1}"[..])]; let dead_letters = rejected_entries(&payloads, &rejected_rows(1, "code 117")); - assert_eq!( - place(&dlq, &dead_letters, in_30s()).await, - Placement::Written, + assert!( + matches!( + place(&dlq, &dead_letters, in_30s()).await, + Placement::Written + ), "the gRPC DLQ refused a dead letter" ); @@ -4425,9 +4481,11 @@ mod tests { (source.topic.as_deref(), source.partition, source.offset), (Some("dfe-events"), Some(3), Some(77)) ); - assert_eq!( - place(&dlq, &dead_letters, in_30s()).await, - Placement::Written, + assert!( + matches!( + place(&dlq, &dead_letters, in_30s()).await, + Placement::Written + ), "a working DLQ takes the row" ); @@ -4454,9 +4512,11 @@ mod tests { row_json: Some(br#"{"_raw":"{\"user\":\"kaz\"}"}"#.to_vec()), }]; - assert_eq!( - place(&dlq, &rejected_entries(&payloads, &failed), in_30s()).await, - Placement::Written, + assert!( + matches!( + place(&dlq, &rejected_entries(&payloads, &failed), in_30s()).await, + Placement::Written + ), "a working DLQ takes the row" ); @@ -4541,9 +4601,11 @@ mod tests { "clickhouse_permanent_reject", b"{\"kept\":1}".to_vec(), )]; - assert_eq!( - place(&dlq, &dead_letters, in_30s()).await, - Placement::Written, + assert!( + matches!( + place(&dlq, &dead_letters, in_30s()).await, + Placement::Written + ), "a working DLQ takes rows written after an earlier loss" ); assert!( @@ -4569,10 +4631,10 @@ mod tests { }) .collect(); - assert_eq!( + assert!(matches!( place(&dlq, &dead_letters, in_30s()).await, Placement::Written - ); + )); assert_eq!(spooled_lines(&dir).len(), 3); let _ = std::fs::remove_dir_all(&dir); } @@ -4589,9 +4651,11 @@ mod tests { )]; let started = std::time::Instant::now(); - assert_eq!( - place(&dlq, &dead_letters, in_30s()).await, - Placement::Closed, + assert!( + matches!( + place(&dlq, &dead_letters, in_30s()).await, + Placement::Closed + ), "a stopped DLQ was taken as a home for the rows" ); assert!( @@ -4710,6 +4774,58 @@ mod tests { dlq.shutdown().await.expect("stop the DLQ"); } + /// `RefusedEntries` cannot be built outside scalo, so the broker's answer is + /// stood in for by the predicate `split_refused` takes. + #[test] + fn the_entries_a_refusal_names_split_from_the_rest() { + let entries: Vec = (1..=4).map(|seq| dead_letter_for(seq, 8)).collect(); + let named = |entry: &DlqEntry| { + entry + .source + .as_ref() + .and_then(|s| s.offset) + .is_some_and(|offset| offset % 2 == 0) + }; + + let (dropped, placed) = split_refused(&entries, named); + assert_eq!(dropped.len(), 2); + assert_eq!(placed.len(), 2); + assert!(dropped.iter().all(named) && !placed.iter().any(named)); + + let (dropped, placed) = split_refused(&entries, |_| false); + assert!(dropped.is_empty() && placed.len() == 4); + } + + #[test] + fn a_dead_letter_the_broker_refused_is_counted_lost_and_answered_dropped() { + for holds_answers in [false, true] { + let mut orchestrator = Orchestrator::new(Config::default()); + orchestrator.holds_answers = holds_answers; + orchestrator.acks.admit(41); + let recorder = crate::metrics::counting::CountingRecorder::default(); + let _local = metrics::set_default_local_recorder(&recorder); + + orchestrator.drop_broker_refused(&[dead_letter_for(41, 8)]); + + assert_eq!( + recorder.labelled( + crate::metrics::DEAD_LETTERS_DROPPED_TOTAL, + "reason", + "too_large" + ), + 1, + "the refused row was not counted (holds_answers {holds_answers})" + ); + assert_eq!(recorder.total(crate::metrics::RETRIES_TOTAL), 0); + let released = orchestrator.acks.take_settled(); + if holds_answers { + assert_eq!(released, vec![(DeliveryStatus::Dropped, vec![41])]); + } else { + assert_eq!(released.len(), 0); + } + } + } + /// A dead letter of the record read from `topic` at `offset`. fn dead_letter_on(topic: &str, offset: i64) -> DlqEntry { DlqEntry::new( diff --git a/tests/integration/config_reachability.rs b/tests/integration/config_reachability.rs index 58516e1e..5ca128d3 100644 --- a/tests/integration/config_reachability.rs +++ b/tests/integration/config_reachability.rs @@ -328,8 +328,7 @@ fn clickhouse_tls_fields_with_no_reader_are_rejected() { // Env-var cascade // ============================================================================ -/// Both separator forms after the prefix must reach the same key, and -/// `DFE_LOADER_CONFIG` must name the config file the cascade docs say it does. +/// Both separator forms after the prefix must reach the same key. /// /// One test because env vars are process-global. #[test] @@ -361,38 +360,6 @@ fn env_cascade_forms_reach_the_config() { unsafe { std::env::remove_var("DFE_LOADER__SCALING__WEIGHT_KAFKA_LAG") }; assert!((config.scaling.weight_kafka_lag - 0.45).abs() < f64::EPSILON); } - - // DFE_LOADER_CONFIG names the config file when --config is absent. - { - let dir = tempfile::TempDir::new().expect("tempdir"); - let path = write_config(&dir, "routing:\n default_table: from_env_named_file\n"); - unsafe { std::env::set_var("DFE_LOADER_CONFIG", &path) }; - let config = Config::load(None).expect("config loads"); - unsafe { std::env::remove_var("DFE_LOADER_CONFIG") }; - assert_eq!(config.routing.default_table, "from_env_named_file"); - - // A file it names that is not there is refused, never swapped for defaults. - let missing = dir.path().join("absent.yaml"); - unsafe { std::env::set_var("DFE_LOADER_CONFIG", &missing) }; - let refused = Config::load(None); - unsafe { std::env::remove_var("DFE_LOADER_CONFIG") }; - assert!( - refused.is_err(), - "DFE_LOADER_CONFIG named a missing file and the loader ran on defaults" - ); - } - - // --config still wins over DFE_LOADER_CONFIG. - { - let dir = tempfile::TempDir::new().expect("tempdir"); - let env_path = write_config(&dir, "routing:\n default_table: from_env\n"); - let flag_dir = tempfile::TempDir::new().expect("tempdir"); - let flag_path = write_config(&flag_dir, "routing:\n default_table: from_flag\n"); - unsafe { std::env::set_var("DFE_LOADER_CONFIG", &env_path) }; - let config = Config::load(Some(&flag_path)).expect("config loads"); - unsafe { std::env::remove_var("DFE_LOADER_CONFIG") }; - assert_eq!(config.routing.default_table, "from_flag"); - } } /// `config-check` run by the binary from `dir`, returning what it printed. @@ -410,6 +377,74 @@ fn config_check_in(dir: &std::path::Path) -> String { printed } +/// `config-check` run by the binary from `dir` with `args` and `config_env` as +/// `DFE_LOADER_CONFIG` (unset when `None`). +fn config_check_with( + dir: &std::path::Path, + args: &[&str], + config_env: Option<&str>, +) -> std::process::Output { + let mut cmd = std::process::Command::new(env!("CARGO_BIN_EXE_dfe-loader")); + cmd.args(args) + .arg("config-check") + .current_dir(dir) + .env_remove("DFE_LOADER_ROUTING_DEFAULT_DB") + .env_remove("DFE_LOADER_ROUTING_DEFAULT_TABLE"); + match config_env { + Some(value) => cmd.env("DFE_LOADER_CONFIG", value), + None => cmd.env_remove("DFE_LOADER_CONFIG"), + }; + cmd.output().expect("the binary runs") +} + +/// `DFE_LOADER_CONFIG` names the config file when `--config` is absent, a +/// missing file it names is refused, `--config` wins over it and a blank value +/// is ignored. Runs the binary, because scalo's `run_app` resolves the path. +#[test] +fn the_config_env_var_names_the_config_file() { + let work = tempfile::TempDir::new().expect("tempdir"); + let env_dir = tempfile::TempDir::new().expect("tempdir"); + let env_path = write_config(&env_dir, "routing:\n default_table: from_env_named_file\n"); + let flag_dir = tempfile::TempDir::new().expect("tempdir"); + let flag_path = write_config(&flag_dir, "routing:\n default_table: from_flag\n"); + let printed = |out: &std::process::Output| String::from_utf8_lossy(&out.stderr).into_owned(); + + let out = config_check_with(work.path(), &[], Some(&env_path)); + assert!( + out.status.success(), + "config-check failed:\n{}", + printed(&out) + ); + assert!(printed(&out).contains("from_env_named_file")); + + let missing = env_dir.path().join("absent.yaml"); + let out = config_check_with( + work.path(), + &[], + Some(missing.to_str().expect("utf-8 path")), + ); + assert!( + !out.status.success(), + "DFE_LOADER_CONFIG named a missing file and the loader ran on defaults" + ); + + let out = config_check_with(work.path(), &["--config", &flag_path], Some(&env_path)); + assert!( + out.status.success(), + "config-check failed:\n{}", + printed(&out) + ); + assert!(printed(&out).contains("from_flag")); + assert!(!printed(&out).contains("from_env_named_file")); + + let out = config_check_with(work.path(), &[], Some("")); + assert!( + out.status.success(), + "a blank value must be ignored:\n{}", + printed(&out) + ); +} + /// The binary reads the `.env` in its working directory and no other. /// /// A `.env` in a parent directory belongs to whatever project sits above, so a