Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 72 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions crates/chunk-bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ no server broadcast, slow reader or backpressure workload.
**Control workloads** run core in the target with its normal background tasks and on-disk SQLite, and call it as core's
own gateway. Synthetic JVMs, one per host, follow their `jvm/<host>` topics inside the target process, so their work
counts toward the target; they report sessions and arrivals but launch no JVMs and simulate no ticks or failures. The
fixture declares 128-player sessions, one session per process and at most 32 processes. `control-population` offers 100
fixture declares 128-player sessions, one session per process and at most 64 processes. `control-population` offers 100
snapshots per second by default, each what a reconnecting gateway reads. `control-churn` reports complete player
lifecycles per second; `population` (plus `concurrency` for churn) may be at most 4096. Released claims are kept for
lifecycles per second; `population` (plus `concurrency` for churn) may be at most 8192. Released claims are kept for
five minutes, so shorter runs measure churn with growing history. Setup and a successful run end by checking that a
fresh snapshot holds exactly the seeded population.

Expand Down
8 changes: 4 additions & 4 deletions crates/chunk-bench/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ use anyhow::{Result, ensure};
use clap::{Parser, ValueEnum};
use serde::{Deserialize, Serialize};

/// Players the control fixture's release can seat: control's limit of 32 processes, each one 128-player session.
pub const CONTROL_SLOTS: u32 = 32 * 128;
/// Players the control fixture's release can seat: 64 processes, each one 128-player session.
pub const CONTROL_SLOTS: u32 = 64 * 128;

#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]
#[serde(rename_all = "kebab-case")]
Expand Down Expand Up @@ -181,10 +181,10 @@ mod tests {

#[test]
fn churn_budget_includes_in_flight_and_seeded_players() {
let mut config = Config::parse_from(["bench", "control-churn", "--population", "4032", "--concurrency", "64"]);
let mut config = Config::parse_from(["bench", "control-churn", "--population", "8128", "--concurrency", "64"]);
assert!(config.validate().is_ok());
config.population += 1;
assert!(config.validate().unwrap_err().to_string().contains("4096"));
assert!(config.validate().unwrap_err().to_string().contains("8192"));
config.scenario = Scenario::ControlPopulation;
assert!(config.validate().is_ok());
}
Expand Down
2 changes: 1 addition & 1 deletion crates/chunk-build/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ A project has a `chunk.toml` at its root, its apps under `apps/`, its backend un
environment = "local" # the local environment's name
machine_profile = "local" # the default profile for sessions
capacity = 16 # default players per session, 1 to 128
max_processes = 4 # JVMs at most, 1 to 32
max_processes = 4 # JVMs at most, 1 to 256
# idle_node_timeout_seconds = 60 # stop a JVM with no unfinished session after this long, 0 to 3600

[local.profiles.local]
Expand Down
4 changes: 2 additions & 2 deletions crates/chunk-build/src/project.rs
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,8 @@ impl LocalConfig {
if !(1..=128).contains(&self.capacity) {
return Err(invalid(path, "local.capacity must be between 1 and 128"));
}
if !(1..=32).contains(&self.max_processes) {
return Err(invalid(path, "local.max_processes must be between 1 and 32"));
if !(1..=256).contains(&self.max_processes) {
return Err(invalid(path, "local.max_processes must be between 1 and 256"));
}
if self.idle_node_timeout_seconds.is_some_and(|seconds| seconds > 3600) {
return Err(invalid(path, "local.idle_node_timeout_seconds must be between 0 and 3600"));
Expand Down
1 change: 1 addition & 0 deletions crates/chunk-build/src/project/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ fn local_defaults_and_app_overrides_are_validated_against_profiles_and_limits()
("machine_profile = \"small\"", "machine_profile = 'missing'", "unknown profile"),
("capacity = 16", "capacity = 129", "local.capacity"),
("max_processes = 4", "max_processes = 0", "local.max_processes"),
("max_processes = 4", "max_processes = 257", "local.max_processes"),
("memory_mib = 512", "memory_mib = 1", "local.profiles.small"),
("max_sessions = 2", "max_sessions = 17", "local.profiles.small"),
] {
Expand Down
1 change: 1 addition & 0 deletions crates/chunk-control/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ publish = false

[dependencies]
base64 = "0.22"
imbl = "7"
tokio-stream = { version = "0.1", features = ["net"] }
tokio-util = { version = "0.7", features = ["rt"] }
chunk-backend = { path = "../chunk-backend" }
Expand Down
2 changes: 1 addition & 1 deletion crates/chunk-control/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ nothing replays packets to restore them.

## Limits

A release allows at most 32 processes. A machine profile allows 1 to 16 sessions per process, a session type 1 to 128
A release allows at most 256 processes. A machine profile allows 1 to 16 sessions per process, a session type 1 to 128
players, and a host at most 128 players across its sessions. Control retains up to 256 sessions. At most 1024 claim,
activation and cancellation operations are in flight; beyond that, calls fail with `OVERLOADED` and should be retried.
Released claims are forgotten five minutes after release.
Expand Down
18 changes: 10 additions & 8 deletions crates/chunk-control/src/capacity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::{
Control, Result,
host::Progress,
placement::runs_host,
state::{Capacity, HostState, Phase, State},
state::{self, Capacity, HostState, Phase, State},
};

impl Control {
Expand Down Expand Up @@ -123,10 +123,14 @@ fn released(state: &mut State, id: &str, from: Capacity) -> Result<()> {
};
host.capacity = Capacity::Released;
host.retired = true;
for session in state.sessions.values_mut().filter(|session| session.host == id) {
session.retired = true;
session.finished = true;
}
state::modify(
&mut state.sessions,
|_, session| session.host == id,
|_, session| {
session.retired = true;
session.finished = true;
},
);
let prepared: Vec<_> = state
.claims
.iter()
Expand All @@ -148,7 +152,5 @@ pub(crate) fn stop(state: &mut State, id: &str, failure: Option<String>) {
host.capacity = Capacity::Releasing;
host.failure = failure;
host.retired = true;
for session in state.sessions.values_mut().filter(|session| session.host == id) {
session.retired = true;
}
state::modify(&mut state.sessions, |_, session| session.host == id, |_, session| session.retired = true);
}
6 changes: 2 additions & 4 deletions crates/chunk-control/src/drain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use tokio::task::JoinSet;

use crate::{
Control, Error, Result,
state::{Capacity, Claim, Drain, Phase, State},
state::{self, Capacity, Claim, Drain, Phase, State},
};

/// Records a drain once per operation, retiring the resolved host and its sessions until the deadline.
Expand All @@ -23,9 +23,7 @@ pub(crate) fn retire_host(
}
let host = host(state)?;
state.hosts.get_mut(&host).ok_or(Error::Invalid("unknown host"))?.retired = true;
for session in state.sessions.values_mut().filter(|s| s.host == host) {
session.retired = true;
}
state::modify(&mut state.sessions, |_, session| session.host == host, |_, session| session.retired = true);
let deadline_ms = crate::now_ms() + u64::from(timeout_seconds) * 1000;
state.drains.insert(operation, Drain { request, host, deadline_ms, automatic });
Ok(())
Expand Down
14 changes: 10 additions & 4 deletions crates/chunk-control/src/draining.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::{

use crate::{
Control, Error, Generation, Result,
state::{Capacity, Claim, Phase, ReleaseDrain, ReleaseState, State},
state::{self, Capacity, Claim, Phase, ReleaseDrain, ReleaseState, State},
};

/// How long after leaving a session of a draining release a player who logs in again returns to it.
Expand Down Expand Up @@ -63,9 +63,15 @@ impl Control {
return Ok(());
}
self.update(|state| {
for drain in state.releases.values_mut().filter_map(|release| release.drain.as_mut()) {
drain.shorten(policy);
}
state::modify(
&mut state.releases,
|_, release| release.drain.as_ref().is_some_and(shortened),
|_, release| {
if let Some(drain) = &mut release.drain {
drain.shorten(policy);
}
},
);
Ok(())
})
}
Expand Down
38 changes: 20 additions & 18 deletions crates/chunk-control/src/idle.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
use std::collections::{BTreeMap, BTreeSet};
use std::collections::BTreeSet;

use chunk_proto::control::v1::{ClaimRequest, ShutdownNodeRequest};
use imbl::OrdMap;
use prost::Message;

use crate::{
Control, Error, Result,
drain::retire_host,
releases::Launches,
state::{Capacity, OperatorMethod, Phase, State},
state::{self, Capacity, HostState, OperatorMethod, Phase, State},
};

/// Long enough for callers to retry an operation after its release, short enough to bound history.
Expand Down Expand Up @@ -111,20 +112,21 @@ impl Control {
.map(|(_, session)| session.host.clone())
.collect();
let mut expired = Vec::new();
for (id, host) in state.hosts.iter_mut().filter(|(_, host)| !host.retired) {
let timeout = state
.releases
.get(&host.release)
.map_or(0, |release| u64::from(release.release.idle_node_timeout_seconds) * 1000);
if timeout == 0 {
continue;
}
let since = (!busy.contains(id)).then(|| host.idle_since_ms.unwrap_or(now));
host.idle_since_ms = since;
if since.is_some_and(|since| now.saturating_sub(since) >= timeout) {
expired.push(id.clone());
}
}
let releases = &state.releases;
let timeout = |host: &HostState| {
releases.get(&host.release).map_or(0, |release| u64::from(release.release.idle_node_timeout_seconds) * 1000)
};
state::modify(
&mut state.hosts,
|_, host| !host.retired && timeout(host) != 0,
|id, host| {
let since = (!busy.contains(id)).then(|| host.idle_since_ms.unwrap_or(now));
host.idle_since_ms = since;
if since.is_some_and(|since| now.saturating_sub(since) >= timeout(host)) {
expired.push(id.to_owned());
}
},
);
for id in expired {
let operation = format!("idle/{id}");
let request =
Expand All @@ -136,11 +138,11 @@ impl Control {
}

/// The IDs of rows `forget` accepts.
fn select<T>(rows: &BTreeMap<String, T>, mut forget: impl FnMut(&str, &T) -> bool) -> Vec<String> {
fn select<T: Clone>(rows: &OrdMap<String, T>, mut forget: impl FnMut(&str, &T) -> bool) -> Vec<String> {
rows.iter().filter(|(id, row)| forget(id, row)).map(|(id, _)| id.clone()).collect()
}

fn remove<T>(rows: &mut BTreeMap<String, T>, ids: Vec<String>) {
fn remove<T: Clone>(rows: &mut OrdMap<String, T>, ids: Vec<String>) {
for id in ids {
rows.remove(&id);
}
Expand Down
2 changes: 1 addition & 1 deletion crates/chunk-control/src/launches.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ pub use crate::state::Launch;
/// # Errors
/// Reports a store that cannot be read, or that a serving control holds.
pub fn recorded_launches(system: chunk_backend::System) -> Result<BTreeSet<String>> {
Ok(state::load(system)?.launches.into_keys().collect())
Ok(state::load(system)?.launches.keys().cloned().collect())
}

impl Control {
Expand Down
Loading