From 4d49d678a8a3d93d3c7d2b26fadd25a54bd6b6c7 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 19 Nov 2026 16:18:23 +1100 Subject: [PATCH] fix(vector): rebuild on scalo 2.14.32 - Bump scalo to >=2.14.32, <3 and the Helm library chart to 2.14.32. - At rung 1, scalo still has an 18-second intake hold and a 15-second send timeout. - The return hold is 16.5 seconds. It is the intake hold minus SINK_BATCHING_MARGIN. - A FilteredDlq batch reply means one record was dead-lettered and the rest were sent. The old code released the whole batch as dropped. Vector resends the batch one record at a time. It sends the record that still gets FilteredDlq back to the pipeline. The pipeline writes it to the DLQ before the source is released. - The config schema, capability catalog and Dockerfile are unchanged after regeneration. Closes #151 --- .hyperi-ci.yaml | 2 +- Cargo.lock | 111 ++++++---------- Cargo.toml | 4 +- src/bridge.rs | 181 +++++++++++++++----------- src/config/loader.rs | 2 +- tests/integration/direct_transport.rs | 25 ++-- 6 files changed, 158 insertions(+), 167 deletions(-) diff --git a/.hyperi-ci.yaml b/.hyperi-ci.yaml index 2d2e3ad..7eb0da7 100644 --- a/.hyperi-ci.yaml +++ b/.hyperi-ci.yaml @@ -54,7 +54,7 @@ release: helm: enabled: true contract: emit - library: "2.14.30" + library: "2.14.32" container: enabled: true # Generated by `dfe-transform-vector emit-dockerfile`, see docs/internals.md. diff --git a/Cargo.lock b/Cargo.lock index 8e1bb89..0e3aefb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1516,9 +1516,9 @@ dependencies = [ [[package]] name = "metrics-exporter-opentelemetry" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500e1fa58894a8c140ee0a9282e1d9e765fc7c7853bd310d1357d2808ad8f0b0" +checksum = "de7023a611dedd9bccd24e5b287717193ca5e07acb571acf42162d77a3623c78" dependencies = [ "derive_more", "metrics", @@ -1833,9 +1833,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee" dependencies = [ "futures-core", "futures-sink", @@ -1847,41 +1847,42 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" dependencies = [ "async-trait", "bytes", "http", "opentelemetry", - "reqwest 0.12.28", + "reqwest", ] [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ "http", + "httpdate", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.12.28", + "reqwest", "thiserror", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "d576f5dddc3299341bcd85a04be17b2564844c0c9563214f02d85f2524d1afae" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1892,15 +1893,16 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.5", "thiserror", "tokio", @@ -2557,40 +2559,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "663ba70707f96e871406fe10d68128412e619b06d1d47cb91c3a4c6501176240" -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "reqwest" version = "0.13.5" @@ -2599,7 +2567,9 @@ checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ "base64 0.23.1", "bytes", + "futures-channel", "futures-core", + "futures-util", "http", "http-body", "http-body-util", @@ -2799,12 +2769,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - [[package]] name = "safe_arch" version = "1.2.0" @@ -2825,9 +2789,9 @@ dependencies = [ [[package]] name = "scalo" -version = "2.14.30" +version = "2.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee9c374451af23a6ebf90c02ce4ae33ea37fda7f397622825a3e0618988c566" +checksum = "8743b18a1ff3a93498d48bd16a5aea38130b161119062b76e802be5eb0720afa" dependencies = [ "base64 0.23.1", "bytes", @@ -2858,7 +2822,7 @@ dependencies = [ "rayon", "rdkafka", "regex", - "reqwest 0.13.5", + "reqwest", "rmpv", "schemars", "serde", @@ -3050,18 +3014,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "sha1_smol" version = "1.0.1" @@ -3592,6 +3544,17 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "tonic-types" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccbb6c31deb8e99dcdf87df19c5289b7838a600da74a8224baa671e52fe7c5ad" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3686,9 +3649,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" dependencies = [ "js-sys", "opentelemetry", @@ -3827,9 +3790,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.29.0" +version = "1.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +checksum = "3c14b15dffa2f4bd712103e857140aa04a4f2ff6e955c77d7eb8e53ee02a3b64" dependencies = [ "getrandom 0.4.3", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index bd98b5a..ef48ae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] } # `health` is REQUIRED for a truthful `/readyz`. `transport-kafka` is for its # config only: the image ships no librdkafka, so nothing may build an rdkafka # client. The floor moves with `release.helm.library` in .hyperi-ci.yaml. -scalo = { version = ">=2.14.30, <3", features = [ +scalo = { version = ">=2.14.32, <3", features = [ "cli-service", "config-reload", "config-schema", @@ -95,7 +95,7 @@ pgo-driver = [] [dev-dependencies] tempfile = ">=3.27.0, <4" tokio-test = ">=0.4.6, <0.5" -scalo = { version = ">=2.14.30, <3", features = [ +scalo = { version = ">=2.14.32, <3", features = [ "transport-kafka", "transport-kafka-test-support", ] } diff --git a/src/bridge.rs b/src/bridge.rs index 6893ac3..796bb84 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -61,12 +61,12 @@ //! The deadlines nest, outermost first, so an inner hop is always answered //! while the hop around it can still answer its own sender: //! -//! - [`INTAKE_HOLD`] (18 s): the longest the intake listener holds a push, +//! - [`intake_hold`] (18 s, the hold of the ladder's depth-1 rung): the longest the intake listener holds a push, //! under the deadline of the stage pushing in. -//! - [`RETURN_HOLD`] (16.5 s): the longest the listener Vector's sink pushes +//! - [`return_hold`] (16.5 s): the longest the listener Vector's sink pushes //! into holds a push. On direct to direct that hold sits inside the intake //! one, with room for Vector's own sink batching. -//! - [`SEND_TIMEOUT_MS`] (15 s): each send to the next stage, and each dial to +//! - [`send_timeout_ms`] (15 s, that rung's next-hop deadline): each send to the next stage, and each dial to //! Vector, gives up after this, so a hung hop is retried inside the hold. //! //! ## Pressure and memory @@ -85,12 +85,13 @@ use scalo::dlq::Dlq; use scalo::governor::UnifiedPressure; use scalo::memory::MemoryGuard; use scalo::transport::grpc::{GrpcConfig, GrpcToken, GrpcTransport}; +use scalo::transport::hold::{HoldLadder, Rung}; use scalo::transport::vector_compat::VectorCompatClient; use scalo::transport::{ AcknowledgementsConfig, DEAD_LETTER_REJECTED, DeliveryStatus, Record, SendResult, SinkConfirmation, TransportError, TransportSender, WorkBatch, }; -use scalo::worker::engine::{BatchProcessingConfig, BlockPieces}; +use scalo::worker::engine::{BatchProcessingConfig, BlockPieces, Refusals}; use scalo::worker::{AdaptiveWorkerPool, BatchEngine, EngineError}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -99,18 +100,38 @@ use crate::config::Config; use crate::metrics::count_dropped_dead_letters; use crate::{Error, Result}; +/// The room the return hold leaves inside the intake hold for Vector's +/// one-second sink batching. +pub const SINK_BATCHING_MARGIN: Duration = Duration::from_millis(1_500); + +/// A transform's rung on scalo's hold ladder: one hop behind the receiver. +#[allow(clippy::expect_used)] +fn rung() -> Rung { + HoldLadder::DEFAULT + .rung(1) + .expect("the default hold ladder has a depth-1 rung") +} + /// The longest the intake listener holds a push: under the upstream stage's -/// send deadline, over [`RETURN_HOLD`]. -pub const INTAKE_HOLD: Duration = Duration::from_secs(18); +/// send deadline, over [`return_hold`]. +#[must_use] +pub fn intake_hold() -> Duration { + rung().hold +} /// The longest the listener Vector's sink pushes into holds a push: inside -/// [`INTAKE_HOLD`] by more than Vector's one-second sink batching, over -/// [`SEND_TIMEOUT_MS`]. -pub const RETURN_HOLD: Duration = Duration::from_millis(16_500); +/// [`intake_hold`] by [`SINK_BATCHING_MARGIN`], over [`send_timeout_ms`]. +#[must_use] +pub fn return_hold() -> Duration { + intake_hold().saturating_sub(SINK_BATCHING_MARGIN) +} /// Each send to the next stage, and each dial to Vector, gives up after this, -/// inside [`RETURN_HOLD`], so the hold can still retry it. -pub const SEND_TIMEOUT_MS: u64 = 15_000; +/// inside [`return_hold`], so the hold can still retry it. +#[must_use] +pub fn send_timeout_ms() -> u64 { + u64::try_from(rung().next_hop_deadline.as_millis()).unwrap_or(u64::MAX) +} /// Payload bytes one `PushEvents` to Vector carries at most, bar a single /// record over it, which goes alone. @@ -186,7 +207,7 @@ impl Bridge { &GrpcConfig::server(&config.source.listen), acknowledgements, Hold { - max: INTAKE_HOLD, + max: intake_hold(), pressure: runtime.pressure.clone(), memory_guard: runtime.memory_guard.clone(), }, @@ -195,7 +216,7 @@ impl Bridge { .await?; let to_vector = VectorCompatClient::connect_lazy_within( &format!("http://{}", bridge.to_vector), - SEND_TIMEOUT_MS, + send_timeout_ms(), ) .map_err(|e| { Error::Transport(format!( @@ -225,7 +246,7 @@ impl Bridge { &GrpcConfig::server(&bridge.from_vector).with_vector_compat(), acknowledgements, Hold { - max: RETURN_HOLD, + max: return_hold(), pressure: None, memory_guard: runtime.memory_guard.clone(), }, @@ -348,7 +369,7 @@ fn leg_ended( /// The client for the next stage's Push listener. fn downstream_config(endpoint: &str) -> GrpcConfig { let mut config = GrpcConfig::client(endpoint); - config.send_timeout_ms = SEND_TIMEOUT_MS; + config.send_timeout_ms = send_timeout_ms(); config } @@ -510,55 +531,60 @@ async fn run_outbound( |batch: &WorkBatch, pieces: &BlockPieces<'_>| { let records: Vec = batch.records.clone(); let downstream = Arc::clone(downstream); - // Carries only what the send left out; the pipeline's own piece - // carries whether the batch was delivered. - let left_out = pieces.piece(); - async move { - match sent(downstream.send_batch(&records).await, records.len()) { - Ok(status) => { - left_out.report(status); - Ok(()) - } - Err(e) => { - // Delivered is the merge's identity, so a failed - // attempt adds nothing to the block's status. - left_out.report(DeliveryStatus::Delivered); - Err(e) - } - } - } + let refusals = pieces.refusals(); + async move { send_to_next_stage(&downstream, &records, &refusals).await } }, ) .await } -/// What one send to the next stage adds to its block's status, or the refusal -/// the pipeline retries. -fn sent(result: SendResult, records: usize) -> std::result::Result { - match result { - SendResult::Ok => Ok(DeliveryStatus::Delivered), - // Every record was over the next stage's size ceiling, which the screen - // should have caught: released dropped, never delivered, never resent. - SendResult::FilteredDlq => { - warn!( - records, - "next stage could take none of the batch -- releasing it as dropped" - ); - Ok(DeliveryStatus::Dropped) - } - SendResult::Backpressured => { - debug!(records, "next stage is backpressured -- holding the batch"); - Err(retry_later()) - } - SendResult::Fatal(e) => { - error!( - error = %e, - records, - "next stage refused the batch -- holding it and retrying" - ); - Err(retry_later()) +/// Send `records` to the next stage, or return the refusal the pipeline +/// retries. +/// +/// A `FilteredDlq` answer to a batch names no record, so the batch is sent +/// again one record at a time. A record the stage dead-letters goes back to +/// the pipeline through `refusals`, which writes it to the DLQ before the +/// source is released. Records already sent go out twice: a duplicate, never +/// a loss. +async fn send_to_next_stage( + downstream: &GrpcTransport, + records: &[Record], + refusals: &Refusals, +) -> std::result::Result<(), EngineError> { + match downstream.send_batch(records).await { + SendResult::Ok => return Ok(()), + SendResult::FilteredDlq => {} + refused => return Err(refusal(refused, records.len())), + } + for record in records { + let one = if records.len() == 1 { + SendResult::FilteredDlq + } else { + downstream.send_batch(std::slice::from_ref(record)).await + }; + match one { + SendResult::Ok => {} + SendResult::FilteredDlq => { + warn!("next stage dead-lettered a record -- handing it to the DLQ"); + refusals.dead_letter(record, "the next stage dead-lettered the record"); + } + refused => return Err(refusal(refused, 1)), } } + Ok(()) +} + +/// The retry for a send to the next stage that was not taken. +fn refusal(result: SendResult, records: usize) -> EngineError { + match result { + SendResult::Fatal(e) => error!( + error = %e, + records, + "next stage refused the batch -- holding it and retrying" + ), + _ => debug!(records, "next stage is backpressured -- holding the batch"), + } + retry_later() } /// A payload as the JSON event Vector's protocol carries. @@ -641,25 +667,16 @@ mod tests { )); } - /// A batch the next stage took none of is released dropped, so its source - /// is answered and it is counted, but never as delivered. + /// A send the next stage did not take is held and retried, never dropped. #[test] - fn a_send_that_left_every_record_out_is_dropped_never_delivered() { + fn a_send_not_taken_is_retried() { assert!(matches!( - sent(SendResult::FilteredDlq, 3), - Ok(DeliveryStatus::Dropped) - )); - assert!(matches!( - sent(SendResult::Ok, 3), - Ok(DeliveryStatus::Delivered) - )); - assert!(matches!( - sent(SendResult::Backpressured, 3), - Err(EngineError::Transport(ref e)) if e.is_recoverable() + refusal(SendResult::Backpressured, 3), + EngineError::Transport(ref e) if e.is_recoverable() )); assert!(matches!( - sent(SendResult::Fatal(TransportError::Send("refused".into())), 3), - Err(EngineError::Transport(ref e)) if e.is_recoverable() + refusal(SendResult::Fatal(TransportError::Send("refused".into())), 3), + EngineError::Transport(ref e) if e.is_recoverable() )); } @@ -670,15 +687,23 @@ mod tests { #[test] fn the_deadlines_nest_intake_over_return_over_send() { let send = Duration::from_millis(downstream_config("http://127.0.0.1:1").send_timeout_ms); - assert_eq!(send, Duration::from_millis(SEND_TIMEOUT_MS)); + let (intake, ret) = (intake_hold(), return_hold()); + assert_eq!(send, Duration::from_millis(send_timeout_ms())); + assert_eq!( + (intake, ret, send), + ( + Duration::from_secs(18), + Duration::from_millis(16_500), + Duration::from_secs(15) + ) + ); assert!( - INTAKE_HOLD > RETURN_HOLD + Duration::from_secs(1), - "the {RETURN_HOLD:?} return hold leaves Vector no room inside the {INTAKE_HOLD:?} \ - intake hold" + intake > ret + Duration::from_secs(1), + "the {ret:?} return hold leaves Vector no room inside the {intake:?} intake hold" ); assert!( - RETURN_HOLD > send, - "a {send:?} send outlives the {RETURN_HOLD:?} return hold" + ret > send, + "a {send:?} send outlives the {ret:?} return hold" ); } diff --git a/src/config/loader.rs b/src/config/loader.rs index ef592be..f30e3c9 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -1537,7 +1537,7 @@ impl Config { /// Hold the direct sink's batching inside what the bridge allows it. /// /// The bridge's hold deadlines leave Vector's sink one second of batching - /// (see [`crate::bridge::RETURN_HOLD`]), and the return listener decodes + /// (see [`crate::bridge::return_hold`]), and the return listener decodes /// at most its gRPC message size per push. fn validate_direct_batch(batch: &BatchConfig) -> Result<()> { if batch.timeout_secs > 1 { diff --git a/tests/integration/direct_transport.rs b/tests/integration/direct_transport.rs index 8ef641a..a55e96a 100644 --- a/tests/integration/direct_transport.rs +++ b/tests/integration/direct_transport.rs @@ -22,7 +22,7 @@ use std::time::Duration; use bytes::Bytes; use dfe_transform_vector::bridge::{ - Bridge, BridgeRuntime, INTAKE_HOLD, RETURN_HOLD, SEND_TIMEOUT_MS, + Bridge, BridgeRuntime, intake_hold, return_hold, send_timeout_ms, }; use dfe_transform_vector::config::assembler; use dfe_transform_vector::config::loader::{ @@ -519,12 +519,14 @@ async fn vectors_push_is_not_answered_ok_before_the_next_stage_takes_it() { } /// The outbound leg keeps the stage's deadlines: Vector's push is given up at -/// `RETURN_HOLD`, inside the intake's hold, and inside that a next stage that -/// never answers is abandoned at `SEND_TIMEOUT_MS` and tried again. On scalo's +/// `return_hold`, inside the intake's hold, and inside that a next stage that +/// never answers is abandoned at `send_timeout_ms` and tried again. On scalo's /// defaults, a 25 s hold and a 30 s send, the push is held to 25 s and the next /// stage is tried once. #[tokio::test] async fn the_outbound_leg_gives_a_hung_next_stage_up_inside_the_hold() { + let (intake_hold, return_hold, send_timeout_ms) = + (intake_hold(), return_hold(), send_timeout_ms()); let work = TempDir::new().expect("work dir"); let mut config = direct_config(&work, "/nonexistent/vector"); config.source.transport = Transport::Bus; @@ -588,14 +590,14 @@ async fn the_outbound_leg_gives_a_hung_next_stage_up_inside_the_hold() { ); // The upper bound sits below where the intake's hold would have answered. assert!( - held_for >= RETURN_HOLD - Duration::from_secs(1) - && held_for < RETURN_HOLD + Duration::from_secs(1), - "Vector's push must be held for the {RETURN_HOLD:?} return hold, not the intake's \ - {INTAKE_HOLD:?} or scalo's 25 s default; it was held {held_for:?}" + held_for >= return_hold - Duration::from_secs(1) + && held_for < return_hold + Duration::from_secs(1), + "Vector's push must be held for the {return_hold:?} return hold, not the intake's \ + {intake_hold:?} or scalo's 25 s default; it was held {held_for:?}" ); assert!( attempts >= 2, - "a next stage that never answers must be given up after {SEND_TIMEOUT_MS} ms and tried \ + "a next stage that never answers must be given up after {send_timeout_ms} ms and tried \ again inside the hold; it saw {attempts} attempt(s)" ); } @@ -604,6 +606,7 @@ async fn the_outbound_leg_gives_a_hung_next_stage_up_inside_the_hold() { /// Vector holds its own, so on direct to direct the inner answer lands first. #[tokio::test] async fn the_intake_holds_a_push_for_the_outer_hold() { + let intake_hold = intake_hold(); let work = TempDir::new().expect("work dir"); let mut config = direct_config(&work, "/nonexistent/vector"); // Inbound only, towards a Vector that is not running. @@ -635,9 +638,9 @@ async fn the_intake_holds_a_push_for_the_outer_hold() { "a push Vector never took must not be acknowledged: {result:?}" ); assert!( - held_for >= INTAKE_HOLD - Duration::from_secs(1) - && held_for < INTAKE_HOLD + Duration::from_secs(1), - "the intake must hold a push for its {INTAKE_HOLD:?} hold; it was held {held_for:?}" + held_for >= intake_hold - Duration::from_secs(1) + && held_for < intake_hold + Duration::from_secs(1), + "the intake must hold a push for its {intake_hold:?} hold; it was held {held_for:?}" ); }