From 0601bc8b08f0a5bd50483ac22c352090e5d22a0f Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:01:27 -0500 Subject: [PATCH 1/3] perf(control): commit only the rows an update changes, and allow 256 processes per release Control state's maps are now imbl OrdMaps. Cloning the published state shares its structure, and the commit diff skips shared subtrees, so a commit costs in proportion to the rows it touches rather than to the whole state. Stamping visits only the rows written with a pending generation, since committed state never names one. Code that changed many rows in place now changes only the rows it selects. A release may run up to 256 processes, so one release seats up to 32,768 players at 128 per process. The bench fixture allows 64 processes. Refs #49, #155 --- Cargo.lock | 76 ++++++++++++++++++++-- crates/chunk-bench/README.md | 4 +- crates/chunk-bench/src/config.rs | 6 +- crates/chunk-build/README.md | 2 +- crates/chunk-build/src/project.rs | 4 +- crates/chunk-build/src/project/tests.rs | 1 + crates/chunk-control/Cargo.toml | 1 + crates/chunk-control/README.md | 2 +- crates/chunk-control/src/capacity.rs | 18 ++--- crates/chunk-control/src/drain.rs | 6 +- crates/chunk-control/src/draining.rs | 14 ++-- crates/chunk-control/src/idle.rs | 38 ++++++----- crates/chunk-control/src/launches.rs | 2 +- crates/chunk-control/src/lib.rs | 2 +- crates/chunk-control/src/releases.rs | 12 ++-- crates/chunk-control/src/sessions.rs | 9 ++- crates/chunk-control/src/state.rs | 65 +++++++++++++----- crates/chunk-control/src/state/store.rs | 34 ++++++---- crates/chunk-control/src/tests.rs | 8 +-- crates/chunk-control/src/tests/releases.rs | 10 +++ 20 files changed, 218 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db30ee07..37bbdd4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "archery" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca55ee147b1926dbea904f50fe4902494e97bc742205abbbf10c709e43815f" + [[package]] name = "arcstr" version = "1.2.0" @@ -425,6 +431,12 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" @@ -497,7 +509,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -671,6 +683,7 @@ dependencies = [ "chunk-service", "chunk-store", "command-fds", + "imbl", "prost", "rustix 1.1.4", "serde", @@ -1932,7 +1945,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "rand_core", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -2378,6 +2391,27 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb" +[[package]] +name = "imbl" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" +dependencies = [ + "archery", + "equivalent", + "imbl-sized-chunks", + "rand_core 0.9.5", + "rand_xoshiro", + "version_check", + "wide", +] + +[[package]] +name = "imbl-sized-chunks" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" + [[package]] name = "indexmap" version = "2.14.2" @@ -4182,9 +4216,15 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" + [[package]] name = "rand_core" version = "0.10.1" @@ -4197,7 +4237,16 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", ] [[package]] @@ -5077,6 +5126,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6470,6 +6528,16 @@ dependencies = [ "winsafe", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/crates/chunk-bench/README.md b/crates/chunk-bench/README.md index 51e1884d..4a20ab1a 100644 --- a/crates/chunk-bench/README.md +++ b/crates/chunk-bench/README.md @@ -48,9 +48,9 @@ no server broadcast, slow reader or backpressure workload. **Control workloads** run core in the target with its normal background tasks and on-disk SQLite, and call it as core's own gateway. Synthetic JVMs, one per host, follow their `jvm/` topics inside the target process, so their work counts toward the target; they report sessions and arrivals but launch no JVMs and simulate no ticks or failures. The -fixture declares 128-player sessions, one session per process and at most 32 processes. `control-population` offers 100 +fixture declares 128-player sessions, one session per process and at most 64 processes. `control-population` offers 100 snapshots per second by default, each what a reconnecting gateway reads. `control-churn` reports complete player -lifecycles per second; `population` (plus `concurrency` for churn) may be at most 4096. Released claims are kept for +lifecycles per second; `population` (plus `concurrency` for churn) may be at most 8192. Released claims are kept for five minutes, so shorter runs measure churn with growing history. Setup and a successful run end by checking that a fresh snapshot holds exactly the seeded population. diff --git a/crates/chunk-bench/src/config.rs b/crates/chunk-bench/src/config.rs index 907e1eca..fc1f1705 100644 --- a/crates/chunk-bench/src/config.rs +++ b/crates/chunk-bench/src/config.rs @@ -3,7 +3,7 @@ use clap::{Parser, ValueEnum}; use serde::{Deserialize, Serialize}; /// Players the control fixture's release can seat: control's limit of 32 processes, each one 128-player session. -pub const CONTROL_SLOTS: u32 = 32 * 128; +pub const CONTROL_SLOTS: u32 = 64 * 128; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ValueEnum)] #[serde(rename_all = "kebab-case")] @@ -181,10 +181,10 @@ mod tests { #[test] fn churn_budget_includes_in_flight_and_seeded_players() { - let mut config = Config::parse_from(["bench", "control-churn", "--population", "4032", "--concurrency", "64"]); + let mut config = Config::parse_from(["bench", "control-churn", "--population", "8128", "--concurrency", "64"]); assert!(config.validate().is_ok()); config.population += 1; - assert!(config.validate().unwrap_err().to_string().contains("4096")); + assert!(config.validate().unwrap_err().to_string().contains("8192")); config.scenario = Scenario::ControlPopulation; assert!(config.validate().is_ok()); } diff --git a/crates/chunk-build/README.md b/crates/chunk-build/README.md index 45251fd8..f5de81ee 100644 --- a/crates/chunk-build/README.md +++ b/crates/chunk-build/README.md @@ -15,7 +15,7 @@ A project has a `chunk.toml` at its root, its apps under `apps/`, its backend un environment = "local" # the local environment's name machine_profile = "local" # the default profile for sessions capacity = 16 # default players per session, 1 to 128 -max_processes = 4 # JVMs at most, 1 to 32 +max_processes = 4 # JVMs at most, 1 to 256 # idle_node_timeout_seconds = 60 # stop a JVM with no unfinished session after this long, 0 to 3600 [local.profiles.local] diff --git a/crates/chunk-build/src/project.rs b/crates/chunk-build/src/project.rs index cf65dcaa..ad7ccf01 100644 --- a/crates/chunk-build/src/project.rs +++ b/crates/chunk-build/src/project.rs @@ -284,8 +284,8 @@ impl LocalConfig { if !(1..=128).contains(&self.capacity) { return Err(invalid(path, "local.capacity must be between 1 and 128")); } - if !(1..=32).contains(&self.max_processes) { - return Err(invalid(path, "local.max_processes must be between 1 and 32")); + if !(1..=256).contains(&self.max_processes) { + return Err(invalid(path, "local.max_processes must be between 1 and 256")); } if self.idle_node_timeout_seconds.is_some_and(|seconds| seconds > 3600) { return Err(invalid(path, "local.idle_node_timeout_seconds must be between 0 and 3600")); diff --git a/crates/chunk-build/src/project/tests.rs b/crates/chunk-build/src/project/tests.rs index c8b0a9c9..cb2fba4b 100644 --- a/crates/chunk-build/src/project/tests.rs +++ b/crates/chunk-build/src/project/tests.rs @@ -97,6 +97,7 @@ fn local_defaults_and_app_overrides_are_validated_against_profiles_and_limits() ("machine_profile = \"small\"", "machine_profile = 'missing'", "unknown profile"), ("capacity = 16", "capacity = 129", "local.capacity"), ("max_processes = 4", "max_processes = 0", "local.max_processes"), + ("max_processes = 4", "max_processes = 257", "local.max_processes"), ("memory_mib = 512", "memory_mib = 1", "local.profiles.small"), ("max_sessions = 2", "max_sessions = 17", "local.profiles.small"), ] { diff --git a/crates/chunk-control/Cargo.toml b/crates/chunk-control/Cargo.toml index 0c3f67fb..984fa124 100644 --- a/crates/chunk-control/Cargo.toml +++ b/crates/chunk-control/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] base64 = "0.22" +imbl = "7" tokio-stream = { version = "0.1", features = ["net"] } tokio-util = { version = "0.7", features = ["rt"] } chunk-backend = { path = "../chunk-backend" } diff --git a/crates/chunk-control/README.md b/crates/chunk-control/README.md index d6f0f2ff..1f8385de 100644 --- a/crates/chunk-control/README.md +++ b/crates/chunk-control/README.md @@ -71,7 +71,7 @@ nothing replays packets to restore them. ## Limits -A release allows at most 32 processes. A machine profile allows 1 to 16 sessions per process, a session type 1 to 128 +A release allows at most 256 processes. A machine profile allows 1 to 16 sessions per process, a session type 1 to 128 players, and a host at most 128 players across its sessions. Control retains up to 256 sessions. At most 1024 claim, activation and cancellation operations are in flight; beyond that, calls fail with `OVERLOADED` and should be retried. Released claims are forgotten five minutes after release. diff --git a/crates/chunk-control/src/capacity.rs b/crates/chunk-control/src/capacity.rs index 7d69585f..8134215f 100644 --- a/crates/chunk-control/src/capacity.rs +++ b/crates/chunk-control/src/capacity.rs @@ -11,7 +11,7 @@ use crate::{ Control, Result, host::Progress, placement::runs_host, - state::{Capacity, HostState, Phase, State}, + state::{self, Capacity, HostState, Phase, State}, }; impl Control { @@ -123,10 +123,14 @@ fn released(state: &mut State, id: &str, from: Capacity) -> Result<()> { }; host.capacity = Capacity::Released; host.retired = true; - for session in state.sessions.values_mut().filter(|session| session.host == id) { - session.retired = true; - session.finished = true; - } + state::modify( + &mut state.sessions, + |_, session| session.host == id, + |_, session| { + session.retired = true; + session.finished = true; + }, + ); let prepared: Vec<_> = state .claims .iter() @@ -148,7 +152,5 @@ pub(crate) fn stop(state: &mut State, id: &str, failure: Option) { host.capacity = Capacity::Releasing; host.failure = failure; host.retired = true; - for session in state.sessions.values_mut().filter(|session| session.host == id) { - session.retired = true; - } + state::modify(&mut state.sessions, |_, session| session.host == id, |_, session| session.retired = true); } diff --git a/crates/chunk-control/src/drain.rs b/crates/chunk-control/src/drain.rs index 089f2d71..7c528197 100644 --- a/crates/chunk-control/src/drain.rs +++ b/crates/chunk-control/src/drain.rs @@ -3,7 +3,7 @@ use tokio::task::JoinSet; use crate::{ Control, Error, Result, - state::{Capacity, Claim, Drain, Phase, State}, + state::{self, Capacity, Claim, Drain, Phase, State}, }; /// Records a drain once per operation, retiring the resolved host and its sessions until the deadline. @@ -23,9 +23,7 @@ pub(crate) fn retire_host( } let host = host(state)?; state.hosts.get_mut(&host).ok_or(Error::Invalid("unknown host"))?.retired = true; - for session in state.sessions.values_mut().filter(|s| s.host == host) { - session.retired = true; - } + state::modify(&mut state.sessions, |_, session| session.host == host, |_, session| session.retired = true); let deadline_ms = crate::now_ms() + u64::from(timeout_seconds) * 1000; state.drains.insert(operation, Drain { request, host, deadline_ms, automatic }); Ok(()) diff --git a/crates/chunk-control/src/draining.rs b/crates/chunk-control/src/draining.rs index a72013ee..386833bf 100644 --- a/crates/chunk-control/src/draining.rs +++ b/crates/chunk-control/src/draining.rs @@ -9,7 +9,7 @@ use std::{ use crate::{ Control, Error, Generation, Result, - state::{Capacity, Claim, Phase, ReleaseDrain, ReleaseState, State}, + state::{self, Capacity, Claim, Phase, ReleaseDrain, ReleaseState, State}, }; /// How long after leaving a session of a draining release a player who logs in again returns to it. @@ -63,9 +63,15 @@ impl Control { return Ok(()); } self.update(|state| { - for drain in state.releases.values_mut().filter_map(|release| release.drain.as_mut()) { - drain.shorten(policy); - } + state::modify( + &mut state.releases, + |_, release| release.drain.as_ref().is_some_and(shortened), + |_, release| { + if let Some(drain) = &mut release.drain { + drain.shorten(policy); + } + }, + ); Ok(()) }) } diff --git a/crates/chunk-control/src/idle.rs b/crates/chunk-control/src/idle.rs index e190f22d..1dd3050a 100644 --- a/crates/chunk-control/src/idle.rs +++ b/crates/chunk-control/src/idle.rs @@ -1,13 +1,14 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use chunk_proto::control::v1::{ClaimRequest, ShutdownNodeRequest}; +use imbl::OrdMap; use prost::Message; use crate::{ Control, Error, Result, drain::retire_host, releases::Launches, - state::{Capacity, OperatorMethod, Phase, State}, + state::{self, Capacity, HostState, OperatorMethod, Phase, State}, }; /// Long enough for callers to retry an operation after its release, short enough to bound history. @@ -111,20 +112,21 @@ impl Control { .map(|(_, session)| session.host.clone()) .collect(); let mut expired = Vec::new(); - for (id, host) in state.hosts.iter_mut().filter(|(_, host)| !host.retired) { - let timeout = state - .releases - .get(&host.release) - .map_or(0, |release| u64::from(release.release.idle_node_timeout_seconds) * 1000); - if timeout == 0 { - continue; - } - let since = (!busy.contains(id)).then(|| host.idle_since_ms.unwrap_or(now)); - host.idle_since_ms = since; - if since.is_some_and(|since| now.saturating_sub(since) >= timeout) { - expired.push(id.clone()); - } - } + let releases = &state.releases; + let timeout = |host: &HostState| { + releases.get(&host.release).map_or(0, |release| u64::from(release.release.idle_node_timeout_seconds) * 1000) + }; + state::modify( + &mut state.hosts, + |_, host| !host.retired && timeout(host) != 0, + |id, host| { + let since = (!busy.contains(id)).then(|| host.idle_since_ms.unwrap_or(now)); + host.idle_since_ms = since; + if since.is_some_and(|since| now.saturating_sub(since) >= timeout(host)) { + expired.push(id.to_owned()); + } + }, + ); for id in expired { let operation = format!("idle/{id}"); let request = @@ -136,11 +138,11 @@ impl Control { } /// The IDs of rows `forget` accepts. -fn select(rows: &BTreeMap, mut forget: impl FnMut(&str, &T) -> bool) -> Vec { +fn select(rows: &OrdMap, mut forget: impl FnMut(&str, &T) -> bool) -> Vec { rows.iter().filter(|(id, row)| forget(id, row)).map(|(id, _)| id.clone()).collect() } -fn remove(rows: &mut BTreeMap, ids: Vec) { +fn remove(rows: &mut OrdMap, ids: Vec) { for id in ids { rows.remove(&id); } diff --git a/crates/chunk-control/src/launches.rs b/crates/chunk-control/src/launches.rs index a131076d..3d6833c7 100644 --- a/crates/chunk-control/src/launches.rs +++ b/crates/chunk-control/src/launches.rs @@ -14,7 +14,7 @@ pub use crate::state::Launch; /// # Errors /// Reports a store that cannot be read, or that a serving control holds. pub fn recorded_launches(system: chunk_backend::System) -> Result> { - Ok(state::load(system)?.launches.into_keys().collect()) + Ok(state::load(system)?.launches.keys().cloned().collect()) } impl Control { diff --git a/crates/chunk-control/src/lib.rs b/crates/chunk-control/src/lib.rs index f887b12f..58c20989 100644 --- a/crates/chunk-control/src/lib.rs +++ b/crates/chunk-control/src/lib.rs @@ -157,7 +157,7 @@ impl Release { || self.deployment.deployment.is_empty() || !plain(&self.release_id) || self.max_processes == 0 - || self.max_processes > 32 + || self.max_processes > 256 || self.idle_node_timeout_seconds > 3600 || self.session_types.is_empty() || self diff --git a/crates/chunk-control/src/releases.rs b/crates/chunk-control/src/releases.rs index 56231633..afdba691 100644 --- a/crates/chunk-control/src/releases.rs +++ b/crates/chunk-control/src/releases.rs @@ -7,7 +7,7 @@ use std::{collections::BTreeSet, sync::Arc}; use crate::{ Control, DrainPolicy, Error, Release, Result, - state::{Capacity, ReleaseState, State}, + state::{self, Capacity, ReleaseState, State}, }; /// The launches whose release control may not know, read before the state they are checked against. @@ -65,11 +65,11 @@ impl Control { if let Some(release) = state.releases.get_mut(&name) { release.drain = None; } - for release in state.releases.iter_mut().filter(|(other, _)| **other != name).map(|(_, release)| release) { - if !release.retired { - release.start_draining(now, policy); - } - } + state::modify( + &mut state.releases, + |other, release| other != name && !release.retired, + |_, release| release.start_draining(now, policy), + ); state.stopping.extend(stop.iter().cloned()); state.stopping.remove(&name); state.current = Some(name); diff --git a/crates/chunk-control/src/sessions.rs b/crates/chunk-control/src/sessions.rs index 2ffaa02b..2c608f9a 100644 --- a/crates/chunk-control/src/sessions.rs +++ b/crates/chunk-control/src/sessions.rs @@ -5,7 +5,7 @@ use chunk_proto::{ use crate::{ Control, Error, Result, - state::{Phase, SessionState, State}, + state::{self, Phase, SessionState, State}, }; impl Control { @@ -70,9 +70,8 @@ impl Control { .map(|claim| claim.session.clone()) .collect(); let reconnectable = crate::draining::reconnectable(state, now); - for (id, session) in - state.sessions.iter_mut().filter(|(_, session)| session.host == host && !session.finished) - { + let select = |_: &str, session: &SessionState| session.host == host && !session.finished; + state::modify(&mut state.sessions, select, |id, session| { let empty = !open.contains(id); if empty { session.empty_since_ms.get_or_insert(now); @@ -92,7 +91,7 @@ impl Control { session.retired = true; session.finish_requested = true; } - } + }); Ok(()) }) } diff --git a/crates/chunk-control/src/state.rs b/crates/chunk-control/src/state.rs index 66e596b1..e94658e4 100644 --- a/crates/chunk-control/src/state.rs +++ b/crates/chunk-control/src/state.rs @@ -3,11 +3,13 @@ pub(crate) mod feed; mod store; use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeSet, sync::{Arc, Mutex, MutexGuard}, }; use chunk_proto::control::v1::{ClaimIdentity, ClaimRequest}; +use chunk_store::DocumentKey; +use imbl::OrdMap; use prost::Message; use sha2::{Digest, Sha256}; @@ -19,6 +21,8 @@ pub(crate) use entities::{ }; pub use entities::{Generation, Launch, MachineKind}; +/// Its maps share structure with the states they were cloned from, so a clone is cheap and diffing it against its +/// origin visits only the rows that changed. #[derive(Clone, Default)] pub(crate) struct State { pub config: Vec, @@ -26,23 +30,23 @@ pub(crate) struct State { pub current: Option, /// The deployments asked to stop at once, rather than drain, until their backend versions are released. pub stopping: BTreeSet, - pub releases: BTreeMap, - pub hosts: BTreeMap, - pub sessions: BTreeMap, - pub players: BTreeMap, - pub claims: BTreeMap, + pub releases: OrdMap, + pub hosts: OrdMap, + pub sessions: OrdMap, + pub players: OrdMap, + pub claims: OrdMap, pub method_sequence: u64, - pub moves: BTreeMap, + pub moves: OrdMap, /// The operation IDs of the moves out of each claim, by the claim's operation ID. Derived from `moves`, not stored. - pub move_sources: BTreeMap>, - pub drains: BTreeMap, + pub move_sources: OrdMap>, + pub drains: OrdMap, /// What each of the operator's operation IDs is bound to. - pub operator_calls: BTreeMap, - pub rosters: BTreeMap, + pub operator_calls: OrdMap, + pub rosters: OrdMap, /// Machines core minted credentials for, by ID. - pub machines: BTreeMap, + pub machines: OrdMap, /// What remote runners start on each host, by host ID. - pub launches: BTreeMap, + pub launches: OrdMap, /// The store epoch, fixed while control runs. pub epoch: u64, /// The revision of the last commit this state includes. App commits share the log, so later revisions may @@ -108,10 +112,20 @@ impl State { } } - /// Replaces [`Generation::PENDING`] with `generation`, the commit that applied the update. - fn stamp(&mut self, generation: Generation) { - self.claims.values_mut().for_each(|claim| claim.stamp(generation)); - self.moves.values_mut().for_each(|intent| intent.stamp(generation)); + /// Replaces [`Generation::PENDING`] with `generation`, the commit that applied the update, in the `pending` rows: + /// the only ones that can name it, since committed state never does. + fn stamp<'a>(&mut self, generation: Generation, pending: impl IntoIterator) { + for key in pending { + if key.table == store::CLAIMS + && let Some(claim) = self.claims.get_mut(&key.id) + { + claim.stamp(generation); + } else if key.table == store::MOVES + && let Some(intent) = self.moves.get_mut(&key.id) + { + intent.stamp(generation); + } + } entities::stamp_wire(&mut self.method_sequence, generation); } @@ -131,6 +145,20 @@ impl State { } } +/// Applies `change` to the rows `select` accepts, copying only the nodes that hold them. +pub(crate) fn modify( + rows: &mut OrdMap, + mut select: impl FnMut(&str, &T) -> bool, + mut change: impl FnMut(&str, &mut T), +) { + let ids: Vec<_> = rows.iter().filter(|(id, row)| select(id, row)).map(|(id, _)| id.clone()).collect(); + for id in ids { + if let Some(row) = rows.get_mut(&id) { + change(&id, row); + } + } +} + /// The durable control state. Reads share the last committed state; writers serialize here, then commit through the /// environment's system lane. /// The control state `system` stores, read without serving it. Blocks on the store. @@ -234,6 +262,7 @@ impl Writer<'_> { let touched = feed::touched(&previous, &next, writes.keys().map(|(key, _)| key)); let moves = writes.keys().filter(|(key, _)| key.table == store::MOVES).map(|(key, _)| key.id.as_str()); next.index_moves(&previous, moves); + let pending: Vec<_> = writes.pending_keys().cloned().collect(); #[cfg(test)] if let Some(committing) = self.authority.committing.lock().ok().and_then(|mut hook| hook.take()) { committing(); @@ -242,7 +271,7 @@ impl Writer<'_> { Ok(revision) if revision > previous.revision => { next.revision = revision; let position = next.position(); - next.stamp(position); + next.stamp(position, &pending); self.authority.feed.record(next, touched)?; Ok(result) } diff --git a/crates/chunk-control/src/state/store.rs b/crates/chunk-control/src/state/store.rs index 9ab3ea80..0a3e86b3 100644 --- a/crates/chunk-control/src/state/store.rs +++ b/crates/chunk-control/src/state/store.rs @@ -1,7 +1,6 @@ -use std::collections::BTreeMap; - use chunk_backend::{ScopeLock, System}; use chunk_store::{DatabaseSchema, DocumentKey, KeyRange, ReadBudget, Revision, Snapshot, Write}; +use imbl::{OrdMap, ordmap::DiffItem}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; @@ -90,7 +89,7 @@ impl Store { players: scan(&snapshot, PLAYERS, budget)?, claims: scan(&snapshot, CLAIMS, budget)?, moves: scan(&snapshot, MOVES, budget)?, - move_sources: BTreeMap::new(), + move_sources: OrdMap::new(), drains: scan(&snapshot, DRAINS, budget)?, operator_calls: scan(&snapshot, OPERATOR_CALLS, budget)?, rosters: scan(&snapshot, ROSTERS, budget)?, @@ -145,9 +144,13 @@ fn key(table: &str, id: &str) -> Result { Ok(DocumentKey::new(table, id)?) } -fn scan(snapshot: &Snapshot, table: &str, budget: &mut ReadBudget) -> Result> { +fn scan( + snapshot: &Snapshot, + table: &str, + budget: &mut ReadBudget, +) -> Result> { let range = KeyRange { table: table.into(), start: None, end: None }; - let mut rows = BTreeMap::new(); + let mut rows = OrdMap::new(); for (id, document) in snapshot.scan_bounded(&range, budget)? { rows.insert(id, serde_json::from_value(document.value)?); } @@ -156,15 +159,17 @@ fn scan(snapshot: &Snapshot, table: &str, budget: &mut Read fn diff( table: &str, - previous: &BTreeMap, - next: &BTreeMap, + previous: &OrdMap, + next: &OrdMap, writes: &mut Writes, ) -> Result<()> { - for id in previous.keys().filter(|id| !next.contains_key(*id)) { - writes.ready.push(Write { key: key(table, id)?, value: None }); - } - for (id, value) in next.iter().filter(|(id, value)| previous.get(*id) != Some(*value)) { - writes.put(key(table, id)?, value.clone())?; + for item in previous.diff(next) { + match item { + DiffItem::Remove(id, _) => writes.ready.push(Write { key: key(table, id)?, value: None }), + DiffItem::Add(id, value) | DiffItem::Update { new: (id, value), .. } => { + writes.put(key(table, id)?, value.clone())?; + } + } } Ok(()) } @@ -191,6 +196,11 @@ impl Writes { ready.chain(self.pending.iter().map(|(key, _)| (key, false))) } + /// The keys of rows that name [`Generation::PENDING`]. + pub fn pending_keys(&self) -> impl Iterator { + self.pending.iter().map(|(key, _)| key) + } + fn put(&mut self, key: DocumentKey, mut row: T) -> Result<()> { if row.pending() { let stamp = move |generation| { diff --git a/crates/chunk-control/src/tests.rs b/crates/chunk-control/src/tests.rs index 96eded42..a5bce6c5 100644 --- a/crates/chunk-control/src/tests.rs +++ b/crates/chunk-control/src/tests.rs @@ -304,9 +304,7 @@ async fn expiry_releases_only_unactivated_reservations_and_confirmed_death_fence control.activate(assignment.claim.unwrap()).await.unwrap(); control .update(|state| { - for claim in state.claims.values_mut() { - claim.created_at_ms = 0; - } + crate::state::modify(&mut state.claims, |_, _| true, |_, claim| claim.created_at_ms = 0); Ok(()) }) .unwrap(); @@ -315,9 +313,7 @@ async fn expiry_releases_only_unactivated_reservations_and_confirmed_death_fence assert!(control.state().unwrap().claims["waiting"].phase == Phase::Reserved); control .update(|state| { - for claim in state.claims.values_mut() { - claim.assigned_at_ms = Some(0); - } + crate::state::modify(&mut state.claims, |_, _| true, |_, claim| claim.assigned_at_ms = Some(0)); Ok(()) }) .unwrap(); diff --git a/crates/chunk-control/src/tests/releases.rs b/crates/chunk-control/src/tests/releases.rs index 2f1e4fb9..5b90b7ec 100644 --- a/crates/chunk-control/src/tests/releases.rs +++ b/crates/chunk-control/src/tests/releases.rs @@ -25,6 +25,16 @@ async fn arrived(fixture: &Fixture, control: &Control, operation: &str) -> Claim request } +#[tokio::test] +async fn a_release_runs_at_most_256_processes() { + let mut fixture = Fixture::new(); + fixture.release.max_processes = 256; + assert!(fixture.release.validate().is_ok()); + fixture.release.max_processes = 257; + assert!(fixture.release.validate().is_err()); + fixture.close().await; +} + #[tokio::test] async fn new_logins_and_moves_use_the_current_release_while_earlier_sessions_stay() { let fixture = Fixture::new(); From 61eb100748f1ae39aa32f27051349f98f6aa4c7a Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:02:38 -0500 Subject: [PATCH 2/3] perf(environment): let managed releases run as many JVMs as control allows Refs #155 --- crates/chunk-control/src/lib.rs | 4 +++- crates/chunk-environment/src/managed/release.rs | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/chunk-control/src/lib.rs b/crates/chunk-control/src/lib.rs index 58c20989..fcf38212 100644 --- a/crates/chunk-control/src/lib.rs +++ b/crates/chunk-control/src/lib.rs @@ -63,6 +63,8 @@ pub struct SessionType { } pub const DEFAULT_IDLE_NODE_TIMEOUT_SECONDS: u32 = 60; +/// The most JVMs one release may run at once. +pub const MAX_PROCESSES: u16 = 256; /// Control's reply to a login routed with a release that no longer accepts logins. Nothing was reserved, so the proxy /// routes the login again through the current release. @@ -157,7 +159,7 @@ impl Release { || self.deployment.deployment.is_empty() || !plain(&self.release_id) || self.max_processes == 0 - || self.max_processes > 256 + || self.max_processes > MAX_PROCESSES || self.idle_node_timeout_seconds > 3600 || self.session_types.is_empty() || self diff --git a/crates/chunk-environment/src/managed/release.rs b/crates/chunk-environment/src/managed/release.rs index 6d3b44d5..e433f47c 100644 --- a/crates/chunk-environment/src/managed/release.rs +++ b/crates/chunk-environment/src/managed/release.rs @@ -21,8 +21,6 @@ const STALL_TIMEOUT: Duration = Duration::from_secs(30); /// A download attempt may take this long, plus a second for every `MIN_BYTES_PER_SECOND` of the archive. const DOWNLOAD_BUDGET: Duration = Duration::from_secs(60); const MIN_BYTES_PER_SECOND: u64 = 256 * 1024; -/// JVMs one release may run at once: control's limit, since capacity bounds them. -const MAX_PROCESSES: u16 = 32; /// Where loaded releases live: each unpacked at `releases/`, beside the archive it was installed from at /// `archives/.tar.gz` and the digest that archive was verified against at `archives/.json`. Core's archive @@ -349,7 +347,7 @@ impl Loaded { release_id: release.id.clone(), profiles: profiles.collect(), session_types, - max_processes: MAX_PROCESSES, + max_processes: chunk_control::MAX_PROCESSES, idle_node_timeout_seconds: chunk_control::DEFAULT_IDLE_NODE_TIMEOUT_SECONDS, assets: chunk_control::DeploymentAssets::new(assets, release.assets.clone(), pack_url_prefix), contracts: chunk_control::Contracts { From ae45db9e70352797e753390977bd0eea2d7075f5 Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:07:06 -0500 Subject: [PATCH 3/3] docs(bench): describe the control fixture's 64 processes --- crates/chunk-bench/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/chunk-bench/src/config.rs b/crates/chunk-bench/src/config.rs index fc1f1705..91d0ab26 100644 --- a/crates/chunk-bench/src/config.rs +++ b/crates/chunk-bench/src/config.rs @@ -2,7 +2,7 @@ use anyhow::{Result, ensure}; use clap::{Parser, ValueEnum}; use serde::{Deserialize, Serialize}; -/// Players the control fixture's release can seat: control's limit of 32 processes, each one 128-player session. +/// Players the control fixture's release can seat: 64 processes, each one 128-player session. pub const CONTROL_SLOTS: u32 = 64 * 128; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]