From 6be78de9bbf3abf5313c9671cbf6d442b0bfffc2 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 2 Sep 2026 19:54:51 +1000 Subject: [PATCH 1/4] feat(rs): surface rotation drain signal via decrypt_indexed (LAB-1678) Bump cachekit-core 0.5 -> 0.6 and read through Keyring::decrypt_indexed so the winning key position reaches the SDK. EncryptionLayer counts reads served by each previous key (AtomicU64 per position, current-key reads not counted) and exposes them via previous_key_hits(), also on SecureCache. An operator running a rotation grace window watches the retiring key's count stop growing before dropping it, instead of guessing at a hard cut-over. Error-class mapping and attempt sequencing are unchanged (core owns the loop, LAB-683). No key material in the signal: positions and counts only. --- Cargo.lock | 4 +- README.md | 6 ++ crates/cachekit/Cargo.toml | 2 +- crates/cachekit/src/client.rs | 8 ++ crates/cachekit/src/encryption.rs | 122 +++++++++++++++++++++- crates/cachekit/tests/encryption_tests.rs | 57 ++++++++++ 6 files changed, 193 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48b324a..8416abe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "cachekit-core" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12089baacc5ff661a62d2071588c895973bc48e42ed359178afaee22decb5559" +checksum = "93adc5646956ba8da140f4179a02e60a2cc7a401b83ebb1270cc4d03748e1fac" dependencies = [ "aes", "aes-gcm", diff --git a/README.md b/README.md index a771728..bb6a3fe 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,12 @@ let cache = CacheKit::builder() Rotation is forward-only: a retired key is never re-promoted (re-promoting would resume a used AES-GCM nonce budget), and a config listing the current key among the previous keys is rejected at load. +**Knowing when to drop the old key.** Every read served by a previous key is counted against that key's position; `cache.secure()?.previous_key_hits()` returns the counts (`hits[i]` for `previous_keys[i]`, current-key reads not counted, no key material). During the grace window, watch the retiring key's count: once it stops growing — every live entry has aged out via TTL or been re-encrypted on write — the key is no longer serving reads and can be dropped from `CACHEKIT_PREVIOUS_MASTER_KEYS` safely, instead of guessing and risking a hard cut-over. + +```rust +let hits = cache.secure()?.previous_key_hits(); // e.g. [0] once k1 has drained +``` + --- ## Cross-SDK Interop Mode diff --git a/crates/cachekit/Cargo.toml b/crates/cachekit/Cargo.toml index 709574c..a4cada7 100644 --- a/crates/cachekit/Cargo.toml +++ b/crates/cachekit/Cargo.toml @@ -45,7 +45,7 @@ reliability = ["tokio/time"] unsync = [] [dependencies] -cachekit-core = { version = "0.5", features = ["messagepack"] } +cachekit-core = { version = "0.6", features = ["messagepack"] } serde = { version = "1", features = ["derive"] } rmp-serde = "1" thiserror = "2.0" diff --git a/crates/cachekit/src/client.rs b/crates/cachekit/src/client.rs index c6e958a..639deeb 100644 --- a/crates/cachekit/src/client.rs +++ b/crates/cachekit/src/client.rs @@ -709,6 +709,14 @@ impl std::fmt::Debug for SecureCache<'_> { #[cfg(feature = "encryption")] impl SecureCache<'_> { + /// Rotation drain signal: reads decrypted by each previous master key, by + /// position in the previous-key list. Watch the retiring key's count stop + /// growing before dropping it. See + /// [`EncryptionLayer::previous_key_hits`](crate::EncryptionLayer::previous_key_hits). + pub fn previous_key_hits(&self) -> Vec { + self.encryption.previous_key_hits() + } + /// Encrypt and store `value` under `key` using the client's default TTL. pub async fn set(&self, key: &str, value: &T) -> Result<(), CachekitError> { self.set_with_ttl(key, value, self.client.default_ttl).await diff --git a/crates/cachekit/src/encryption.rs b/crates/cachekit/src/encryption.rs index fcee1c6..c61cc8f 100644 --- a/crates/cachekit/src/encryption.rs +++ b/crates/cachekit/src/encryption.rs @@ -13,6 +13,8 @@ //! Each component is length-prefixed with a 4-byte big-endian u32 to prevent //! collision attacks from boundary confusion. +use std::sync::atomic::{AtomicU64, Ordering}; + use zeroize::Zeroizing; use cachekit_core::{Keyring, ZeroKnowledgeEncryptor}; @@ -39,11 +41,25 @@ const AAD_VERSION: u8 = 0x03; /// (cachekit-rs entries carry no per-entry key identity — sequential /// attempts are the spec-assigned branch, `protocol/spec/encryption.md` → /// "Key Rotation (Keyring)"). +/// +/// ## Rotation drain signal +/// +/// Every read served by a previous key is counted against that key's +/// position; [`Self::previous_key_hits`] returns the counts. During a +/// rotation grace window, watch the retiring key's count: once it stops +/// growing — every live entry has aged out via TTL or been re-encrypted on +/// write — the key is no longer serving reads and can be dropped from the +/// previous list safely, instead of guessing and risking a hard cut-over. +/// Reads served by the current key are not counted, and the signal carries +/// no key material — positions and counts only. pub struct EncryptionLayer { encryptor: ZeroKnowledgeEncryptor, derived_key: Zeroizing<[u8; 32]>, keyring: Keyring, tenant_id: String, + /// `previous_key_hits[i]` = reads decrypted by `previous_keys[i]` + /// (keyring index `i + 1`). Current-key reads are not counted. + previous_key_hits: Vec, } impl EncryptionLayer { @@ -132,6 +148,7 @@ impl EncryptionLayer { derived_key: Zeroizing::new(tenant_keys.encryption_key), keyring, tenant_id: tenant_id.to_owned(), + previous_key_hits: previous_keys.iter().map(|_| AtomicU64::new(0)).collect(), }) } @@ -155,11 +172,15 @@ impl EncryptionLayer { /// attempted first, then each decrypt-only previous key in order, with /// the identical AAD per attempt. Entries written before a key rotation /// stay readable as long as their key remains in the previous list. + /// + /// A read served by a previous key is counted in + /// [`Self::previous_key_hits`] (the rotation drain signal). pub fn decrypt(&self, ciphertext: &[u8], cache_key: &str) -> Result, CachekitError> { // compressed=false is normative, not a stub — see build_aad's invariant note. let aad = self.build_aad(cache_key, false); - self.keyring - .decrypt(&self.encryptor, ciphertext, &self.tenant_id, &aad) + let (plaintext, index) = self + .keyring + .decrypt_indexed(&self.encryptor, ciphertext, &self.tenant_id, &aad) .map_err(|e| match e { // Config-class errors stay config-class (LAB-683 decision): // a derivation failure or caller bug must never masquerade as @@ -169,7 +190,48 @@ impl EncryptionLayer { CachekitError::Config(format!("keyring decrypt misconfiguration: {e}")) } _ => CachekitError::Encryption(format!("decrypt failed: {e}")), - }) + })?; + // Index 0 is the current key: no drain signal. Index i >= 1 is + // previous_keys[i - 1]; the keyring was built from the same slice, so + // the slot always exists — `get` only guards against a core bug. + if let Some(hits) = index + .checked_sub(1) + .and_then(|i| self.previous_key_hits.get(i)) + { + hits.fetch_add(1, Ordering::Relaxed); + } + Ok(plaintext) + } + + /// Reads decrypted by each previous key, by position in the previous-key + /// list (`hits[i]` ↔ `previous_keys[i]`); empty when there are none. + /// + /// This is the rotation drain signal — see the [type docs](Self) for the + /// operator workflow. Counts are monotonic for the life of the layer and + /// carry no key material. + /// + /// ``` + /// use cachekit::EncryptionLayer; + /// + /// let k1 = [0x11u8; 32]; // retiring master key + /// let k2 = [0x22u8; 32]; // current master key after rotation + /// + /// // Encrypted under k1, before the rotation... + /// let ciphertext = EncryptionLayer::new(&k1, "tenant-123")?.encrypt(b"cached value", "user:1")?; + /// + /// // ...a layer [current=k2, previous=[k1]] serves it from previous[0]: + /// // the retiring key is still draining, not yet safe to drop. + /// let layer = EncryptionLayer::with_previous_keys(&k2, &[&k1], "tenant-123")?; + /// assert_eq!(layer.previous_key_hits(), vec![0]); + /// assert_eq!(layer.decrypt(&ciphertext, "user:1")?, b"cached value"); + /// assert_eq!(layer.previous_key_hits(), vec![1]); + /// # Ok::<(), cachekit::CachekitError>(()) + /// ``` + pub fn previous_key_hits(&self) -> Vec { + self.previous_key_hits + .iter() + .map(|h| h.load(Ordering::Relaxed)) + .collect() } /// Return the tenant ID used for key derivation. @@ -444,6 +506,60 @@ mod tests { assert!(matches!(result, Err(CachekitError::Config(_)))); } + // ── Rotation drain signal (LAB-1678) ───────────────────────────────────── + + #[test] + fn current_key_hit_is_not_counted() { + // No previous keys: the drain signal has nothing to report. + let single = EncryptionLayer::new(K2, TEST_TENANT).unwrap(); + let ct = single.encrypt(b"v", "user:3").unwrap(); + single.decrypt(&ct, "user:3").unwrap(); + assert!(single.previous_key_hits().is_empty()); + + // With a previous key, a current-key read (index 0) is a zero reading. + let rotated = EncryptionLayer::with_previous_keys(K2, &[K1], TEST_TENANT).unwrap(); + let ct = rotated.encrypt(b"fresh write", "user:3").unwrap(); + rotated.decrypt(&ct, "user:3").unwrap(); + assert_eq!(rotated.previous_key_hits(), vec![0]); + } + + #[test] + fn previous_key_hit_is_counted_at_its_position() { + const K3: &[u8] = &[0x33; 32]; + let k1_ct = EncryptionLayer::new(K1, TEST_TENANT) + .unwrap() + .encrypt(b"k1 era", "user:4") + .unwrap(); + let k2_ct = EncryptionLayer::new(K2, TEST_TENANT) + .unwrap() + .encrypt(b"k2 era", "user:5") + .unwrap(); + + // current=k3, previous=[k2, k1]: counts index by position in the previous list. + let rotated = EncryptionLayer::with_previous_keys(K3, &[K2, K1], TEST_TENANT).unwrap(); + assert_eq!(rotated.previous_key_hits(), vec![0, 0]); + + rotated.decrypt(&k1_ct, "user:4").unwrap(); + assert_eq!(rotated.previous_key_hits(), vec![0, 1], "k1 is previous[1]"); + + rotated.decrypt(&k2_ct, "user:5").unwrap(); + rotated.decrypt(&k2_ct, "user:5").unwrap(); + assert_eq!(rotated.previous_key_hits(), vec![2, 1], "k2 is previous[0]"); + } + + #[test] + fn failed_decrypt_is_not_a_previous_key_hit() { + let k1_ct = EncryptionLayer::new(K1, TEST_TENANT) + .unwrap() + .encrypt(b"v", "key:a") + .unwrap(); + let rotated = EncryptionLayer::with_previous_keys(K2, &[K1], TEST_TENANT).unwrap(); + + // Wrong cache key: every attempt fails authentication — no key won. + assert!(rotated.decrypt(&k1_ct, "key:b").is_err()); + assert_eq!(rotated.previous_key_hits(), vec![0]); + } + #[test] fn debug_redacts_key() { let layer = EncryptionLayer::new(TEST_MASTER_KEY, TEST_TENANT).unwrap(); diff --git a/crates/cachekit/tests/encryption_tests.rs b/crates/cachekit/tests/encryption_tests.rs index 491cc3f..1b95dc4 100644 --- a/crates/cachekit/tests/encryption_tests.rs +++ b/crates/cachekit/tests/encryption_tests.rs @@ -417,3 +417,60 @@ async fn rotation_round_trip_without_reencryption() { "dropped-key read must surface as an encryption error, got {result:?}" ); } + +/// Rotation drain signal (LAB-1678): an operator running a grace window reads +/// per-previous-key hit counts off the secure handle. A read served by the +/// retiring key increments its slot; a read served by the current key does not. +#[tokio::test] +async fn rotation_drain_signal_is_visible_on_secure_cache() { + const K1: &[u8] = &[0x11; 32]; + const K2: &[u8] = &[0x22; 32]; + + let backend = common::MockBackend::shared(); + + let writer = CacheKit::builder() + .backend(backend.clone()) + .default_ttl(Duration::from_secs(60)) + .no_l1() + .encryption_from_bytes(K1, "test-tenant") + .expect("encryption setup") + .build() + .expect("client builds"); + writer + .secure() + .expect("secure()") + .set("drain:old", &"written under k1") + .await + .expect("secure set under k1"); + + let rotated = CacheKit::builder() + .backend(backend) + .default_ttl(Duration::from_secs(60)) + .no_l1() + .encryption_from_bytes_with_previous(K2, &[K1], "test-tenant") + .expect("keyring setup") + .build() + .expect("client builds"); + let secure = rotated.secure().expect("secure()"); + assert_eq!(secure.previous_key_hits(), vec![0], "nothing read yet"); + + // Fresh write + read under the current key: no drain signal. + secure + .set("drain:new", &"written under k2") + .await + .expect("secure set under k2"); + let _: Option = secure.get("drain:new").await.expect("secure get"); + assert_eq!( + secure.previous_key_hits(), + vec![0], + "current-key hit is silent" + ); + + // The k1-era entry is served by previous[0]: the grace window is still live. + let _: Option = secure.get("drain:old").await.expect("secure get"); + assert_eq!( + secure.previous_key_hits(), + vec![1], + "previous-key hit is counted" + ); +} From 048413a154a549a15013864c032f55e66e9246dd Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 2 Sep 2026 20:06:41 +1000 Subject: [PATCH 2/4] refactor(rs): apply panel findings to the drain signal (LAB-1678) Docs: state that counts are per process and reset on restart, so operators aggregate across every instance holding the retiring key and watch a full TTL window before dropping it (a single restarted replica reading [0] is not a drained key). Operator workflow now lives in one Rust location (previous_key_hits rustdoc); type/field/handle docs point there. README code fence that restated the inline call removed. Code: debug_assert on the keyring index so an out-of-range index from core is loud under test while the release path stays panic-free. Integration test trimmed to the handle-wiring proof; index-0 silence is owned at the layer. --- README.md | 6 +--- crates/cachekit/src/client.rs | 4 +-- crates/cachekit/src/encryption.rs | 41 ++++++++++++----------- crates/cachekit/tests/encryption_tests.rs | 18 ++-------- 4 files changed, 26 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index bb6a3fe..e3b9b70 100644 --- a/README.md +++ b/README.md @@ -183,11 +183,7 @@ let cache = CacheKit::builder() Rotation is forward-only: a retired key is never re-promoted (re-promoting would resume a used AES-GCM nonce budget), and a config listing the current key among the previous keys is rejected at load. -**Knowing when to drop the old key.** Every read served by a previous key is counted against that key's position; `cache.secure()?.previous_key_hits()` returns the counts (`hits[i]` for `previous_keys[i]`, current-key reads not counted, no key material). During the grace window, watch the retiring key's count: once it stops growing — every live entry has aged out via TTL or been re-encrypted on write — the key is no longer serving reads and can be dropped from `CACHEKIT_PREVIOUS_MASTER_KEYS` safely, instead of guessing and risking a hard cut-over. - -```rust -let hits = cache.secure()?.previous_key_hits(); // e.g. [0] once k1 has drained -``` +**Knowing when to drop the old key.** Every read served by a previous key is counted against that key's position; `cache.secure()?.previous_key_hits()` returns the counts (`hits[i]` for `previous_keys[i]`, current-key reads not counted, no key material). During the grace window, watch the retiring key's count: once it stops growing — every live entry has aged out via TTL or been re-encrypted on write — the key is no longer serving reads and can be dropped from `CACHEKIT_PREVIOUS_MASTER_KEYS` safely, instead of guessing and risking a hard cut-over. Counts are per process and reset on restart: aggregate across every instance holding the retiring key, and watch for growth over a full TTL window, before dropping it. --- diff --git a/crates/cachekit/src/client.rs b/crates/cachekit/src/client.rs index 639deeb..f98fa8d 100644 --- a/crates/cachekit/src/client.rs +++ b/crates/cachekit/src/client.rs @@ -709,9 +709,7 @@ impl std::fmt::Debug for SecureCache<'_> { #[cfg(feature = "encryption")] impl SecureCache<'_> { - /// Rotation drain signal: reads decrypted by each previous master key, by - /// position in the previous-key list. Watch the retiring key's count stop - /// growing before dropping it. See + /// Rotation drain signal; see /// [`EncryptionLayer::previous_key_hits`](crate::EncryptionLayer::previous_key_hits). pub fn previous_key_hits(&self) -> Vec { self.encryption.previous_key_hits() diff --git a/crates/cachekit/src/encryption.rs b/crates/cachekit/src/encryption.rs index c61cc8f..ea37dd3 100644 --- a/crates/cachekit/src/encryption.rs +++ b/crates/cachekit/src/encryption.rs @@ -40,25 +40,14 @@ const AAD_VERSION: u8 = 0x03; /// previous key in order, rebuilding the identical AAD per attempt /// (cachekit-rs entries carry no per-entry key identity — sequential /// attempts are the spec-assigned branch, `protocol/spec/encryption.md` → -/// "Key Rotation (Keyring)"). -/// -/// ## Rotation drain signal -/// -/// Every read served by a previous key is counted against that key's -/// position; [`Self::previous_key_hits`] returns the counts. During a -/// rotation grace window, watch the retiring key's count: once it stops -/// growing — every live entry has aged out via TTL or been re-encrypted on -/// write — the key is no longer serving reads and can be dropped from the -/// previous list safely, instead of guessing and risking a hard cut-over. -/// Reads served by the current key are not counted, and the signal carries -/// no key material — positions and counts only. +/// "Key Rotation (Keyring)"). Reads served by a previous key are counted — +/// see [`Self::previous_key_hits`] for the rotation drain workflow. pub struct EncryptionLayer { encryptor: ZeroKnowledgeEncryptor, derived_key: Zeroizing<[u8; 32]>, keyring: Keyring, tenant_id: String, - /// `previous_key_hits[i]` = reads decrypted by `previous_keys[i]` - /// (keyring index `i + 1`). Current-key reads are not counted. + /// `hits[i]` ↔ `previous_keys[i]`; see [`Self::previous_key_hits`]. previous_key_hits: Vec, } @@ -191,9 +180,13 @@ impl EncryptionLayer { } _ => CachekitError::Encryption(format!("decrypt failed: {e}")), })?; - // Index 0 is the current key: no drain signal. Index i >= 1 is - // previous_keys[i - 1]; the keyring was built from the same slice, so - // the slot always exists — `get` only guards against a core bug. + // index 0 = current key (no signal); i >= 1 = previous_keys[i - 1]. + // The keyring was built from the same slice, so the slot always + // exists; `get` keeps a core bug from panicking the read path. + debug_assert!( + index <= self.previous_key_hits.len(), + "keyring index {index} beyond previous-key slots" + ); if let Some(hits) = index .checked_sub(1) .and_then(|i| self.previous_key_hits.get(i)) @@ -205,10 +198,18 @@ impl EncryptionLayer { /// Reads decrypted by each previous key, by position in the previous-key /// list (`hits[i]` ↔ `previous_keys[i]`); empty when there are none. + /// Reads served by the current key are not counted. + /// + /// This is the rotation **drain signal**. During a rotation grace window, + /// watch the retiring key's count: once it stops growing — every live + /// entry has aged out via TTL or been re-encrypted on write — the key is + /// no longer serving reads and can be dropped from the previous list + /// safely, instead of guessing and risking a hard cut-over. /// - /// This is the rotation drain signal — see the [type docs](Self) for the - /// operator workflow. Counts are monotonic for the life of the layer and - /// carry no key material. + /// Counts are per process and reset on restart: aggregate across every + /// instance holding the retiring key, and watch for growth over a full + /// TTL window, before dropping it. The signal carries no key material — + /// positions and counts only. /// /// ``` /// use cachekit::EncryptionLayer; diff --git a/crates/cachekit/tests/encryption_tests.rs b/crates/cachekit/tests/encryption_tests.rs index 1b95dc4..ed933d9 100644 --- a/crates/cachekit/tests/encryption_tests.rs +++ b/crates/cachekit/tests/encryption_tests.rs @@ -418,9 +418,9 @@ async fn rotation_round_trip_without_reencryption() { ); } -/// Rotation drain signal (LAB-1678): an operator running a grace window reads -/// per-previous-key hit counts off the secure handle. A read served by the -/// retiring key increments its slot; a read served by the current key does not. +/// Rotation drain signal (LAB-1678): the builder wires the counters into the +/// user-held secure handle, so a read served by the retiring key is visible +/// there. Index-0 silence is owned and tested at the layer (`encryption.rs`). #[tokio::test] async fn rotation_drain_signal_is_visible_on_secure_cache() { const K1: &[u8] = &[0x11; 32]; @@ -454,18 +454,6 @@ async fn rotation_drain_signal_is_visible_on_secure_cache() { let secure = rotated.secure().expect("secure()"); assert_eq!(secure.previous_key_hits(), vec![0], "nothing read yet"); - // Fresh write + read under the current key: no drain signal. - secure - .set("drain:new", &"written under k2") - .await - .expect("secure set under k2"); - let _: Option = secure.get("drain:new").await.expect("secure get"); - assert_eq!( - secure.previous_key_hits(), - vec![0], - "current-key hit is silent" - ); - // The k1-era entry is served by previous[0]: the grace window is still live. let _: Option = secure.get("drain:old").await.expect("secure get"); assert_eq!( From 067ad65ac9abca82ea8f3e525423f32ad0315e9e Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Thu, 3 Sep 2026 09:49:44 +1000 Subject: [PATCH 3/4] docs(rs): state the runbook preconditions for the drain signal (LAB-1678) The 'safe to drop' guidance said a flat counter over one TTL window was enough. The normative runbook (protocol decisions/key-rotation.md) is stricter, and the counter cannot see the gap: during a rolling promotion a lagging instance still writes under the retiring key and reads it as its own current key, which is deliberately silent (index 0). Per-entry TTLs via set_with_ttl can also outlive the default window. Both surfaces (README Key Rotation, previous_key_hits rustdoc) now state: complete the two-phase promotion first, start the clock when that deploy finishes fleet-wide, wait the longest TTL in use, aggregate across instances, and only then read a flat counter as drained. Adversarial finding by Helly R on PR #74. --- README.md | 2 +- crates/cachekit/src/encryption.rs | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e3b9b70..02c6b4e 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ let cache = CacheKit::builder() Rotation is forward-only: a retired key is never re-promoted (re-promoting would resume a used AES-GCM nonce budget), and a config listing the current key among the previous keys is rejected at load. -**Knowing when to drop the old key.** Every read served by a previous key is counted against that key's position; `cache.secure()?.previous_key_hits()` returns the counts (`hits[i]` for `previous_keys[i]`, current-key reads not counted, no key material). During the grace window, watch the retiring key's count: once it stops growing — every live entry has aged out via TTL or been re-encrypted on write — the key is no longer serving reads and can be dropped from `CACHEKIT_PREVIOUS_MASTER_KEYS` safely, instead of guessing and risking a hard cut-over. Counts are per process and reset on restart: aggregate across every instance holding the retiring key, and watch for growth over a full TTL window, before dropping it. +**Knowing when to drop the old key.** Every read served by a previous key is counted against that key's position; `cache.secure()?.previous_key_hits()` returns the counts (`hits[i]` for `previous_keys[i]`, current-key reads not counted, no key material). The signal confirms a grace window has drained; it does not shorten one. Follow the protocol's [scheduled-rotation runbook](https://github.com/cachekit-io/protocol/blob/main/decisions/key-rotation.md#runbooks-normative-for-docs): audit for non-expiring entries, add the incoming key as decrypt-only fleet-wide, then promote it. The clock starts only when the promotion deploy has completed on every instance — a lagging instance still writes under the retiring key and reads it silently as *its* current key. From then, wait at least the longest TTL in use (including any explicit `set_with_ttl` values), aggregating counts across every instance (they are per process and reset on restart). Once the retiring key's count has stayed flat over that whole window, every live entry has aged out or been re-encrypted on write, and the key can be dropped from `CACHEKIT_PREVIOUS_MASTER_KEYS` without a hard cut-over. --- diff --git a/crates/cachekit/src/encryption.rs b/crates/cachekit/src/encryption.rs index ea37dd3..0ae4294 100644 --- a/crates/cachekit/src/encryption.rs +++ b/crates/cachekit/src/encryption.rs @@ -200,16 +200,23 @@ impl EncryptionLayer { /// list (`hits[i]` ↔ `previous_keys[i]`); empty when there are none. /// Reads served by the current key are not counted. /// - /// This is the rotation **drain signal**. During a rotation grace window, - /// watch the retiring key's count: once it stops growing — every live - /// entry has aged out via TTL or been re-encrypted on write — the key is - /// no longer serving reads and can be dropped from the previous list - /// safely, instead of guessing and risking a hard cut-over. - /// - /// Counts are per process and reset on restart: aggregate across every - /// instance holding the retiring key, and watch for growth over a full - /// TTL window, before dropping it. The signal carries no key material — - /// positions and counts only. + /// This is the rotation **drain signal**. It confirms that a grace window + /// has drained; it does not shorten one. Follow the protocol's + /// [scheduled-rotation runbook](https://github.com/cachekit-io/protocol/blob/main/decisions/key-rotation.md#runbooks-normative-for-docs): + /// audit for non-expiring entries first, add the incoming key as + /// decrypt-only fleet-wide, then promote it. The window clock starts only + /// when that promotion deploy has completed on every instance — until + /// then a lagging instance still writes fresh ciphertext under the + /// retiring key and reads it silently as *its* current key (index 0). + /// From that point, wait at least the longest TTL in use (per-entry TTLs + /// passed to `set_with_ttl` count, not just the default), aggregating + /// counts across every instance — they are per process and reset on + /// restart. Only when the retiring key's count has stayed flat over that + /// whole window has every live entry aged out or been re-encrypted on + /// write, and the key can be dropped from the previous list without a + /// hard cut-over. Positions are comparable across instances only once + /// they all run the same keyring configuration. The signal carries no + /// key material — positions and counts only. /// /// ``` /// use cachekit::EncryptionLayer; From d3600e08f201bb838e3f4658146b818c108db7d4 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 9 Sep 2026 23:56:57 +1000 Subject: [PATCH 4/4] fix(rs): bump async-trait to 0.1.92 to clear the beta clippy canary (LAB-1678) The beta job on this PR fails on 9 double_must_use errors in backend/mod.rs, all emitted by the async_trait macro: async-trait <=0.1.91 injects a bare #[must_use] onto every generated method, and beta clippy now flags that as redundant on methods already returning a must_use Result. Upstream fixed it in 0.1.92 (LAB-2545). That bump has sat unmerged in #72 for a week, so this PR carries the same one-package lockfile change rather than waiting on it; whichever lands second sees a trivial Cargo.lock conflict. Pulls syn 3.0.5 as a new transitive build dependency alongside syn 2 (deny.toml: multiple-versions = warn). Verified locally: beta clippy -D warnings clean, stable clippy clean, cargo +1.85 check passes, wasm32 check builds, 279 tests green. --- Cargo.lock | 65 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8416abe..8af65f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,13 +106,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -217,7 +217,7 @@ version = "0.7.0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -273,7 +273,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn", + "syn 2.0.117", "tempfile", "toml", ] @@ -467,7 +467,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -485,7 +485,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -575,7 +575,7 @@ checksum = "1458c6e22d36d61507034d5afecc64f105c1d39712b7ac6ec3b352c423f715cc" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -634,7 +634,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1257,7 +1257,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1315,7 +1315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -1729,7 +1729,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1789,7 +1789,7 @@ checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1870,6 +1870,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -1887,7 +1898,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1935,7 +1946,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1946,7 +1957,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1997,7 +2008,7 @@ checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2315,7 +2326,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -2358,7 +2369,7 @@ checksum = "3c81b9fef827e575e0e54431736d1baa0d700315d8c62cfef1f61fa3aad0cbeb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2650,7 +2661,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -2666,7 +2677,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -2763,7 +2774,7 @@ dependencies = [ "async-trait", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-macro-support", @@ -2813,7 +2824,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2834,7 +2845,7 @@ checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2854,7 +2865,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2875,7 +2886,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2908,7 +2919,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]]