From 80680b805503797728209191a9ffd54ebeb38f80 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Tue, 3 Nov 2026 23:26:06 +0100 Subject: [PATCH] examples(sweeps): drop seeds from the example sweeps Each condition runs once on the scenario's run_seed by default; seeds stay a documented option. The plan line now says how to add repeats. --- docs/SWEEPS.md | 11 ++++++----- examples/sweeps/governance_grid.yaml | 1 - examples/sweeps/model_comparison.yaml | 3 +-- src/verdi/experiments/sweep.py | 5 ++++- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/SWEEPS.md b/docs/SWEEPS.md index f1e0557..c11b42d 100644 --- a/docs/SWEEPS.md +++ b/docs/SWEEPS.md @@ -2,7 +2,7 @@ A sweep reruns one scenario many times. You list what should change, for example the model, the quorum or the shock size. Verdi runs every combination -on several seeds and gives you one spreadsheet and one comparison page. +and gives you one spreadsheet and one comparison page. The baseline scenario file is never edited. ## The 3 commands you need @@ -40,7 +40,6 @@ scenario: examples/full.yaml # the baseline vary: # every combination is run model.name: [default, Borealis2-preview] -seeds: [42, 43, 44] # or a count: `seeds: 3` parallel_runs: 2 # simulations at the same time agent_concurrency: 4 # model calls at the same time, per simulation # ticks: 10 # shorten every run for a cheap pilot @@ -86,7 +85,7 @@ rest is kept. Put each key in `.env` (`DEEPSEEK_API_KEY=...`), never in the YAML ### Several things at once -Every combination is run. This runs 2 x 2 x 2 = 8 conditions, each on every seed: +Every combination is run. This runs 2 x 2 x 2 = 8 conditions: ```yaml vary: @@ -103,7 +102,7 @@ See [examples/sweeps/governance_grid.yaml](../examples/sweeps/governance_grid.ya python -m verdi.sweep --scenario examples/full.yaml \ --vary model.name=default,Borealis2-preview \ --vary constitution.quorum=0.5,0.75 \ - --seeds 42,43,44 --parallel-runs 2 --dry-run + --parallel-runs 2 --dry-run ``` The other settings work as flags too: `--ticks`, `--agent-concurrency`, @@ -136,7 +135,9 @@ To rebuild `summary.csv` and `comparison.html` without running anything: - **Pilot first.** Run with `ticks: 5` and one seed, check the comparison page, then remove `ticks` for the real sweep. -- **Use the same seeds for every condition.** The sweep does this for you. +- **Repeat with seeds when you need it.** Without `seeds:` every condition runs + once, on the scenario's own `run_seed`. Add `seeds: [42, 43, 44]` (or a count, + `seeds: 3`) to repeat every condition on each seed; all conditions get the same seeds. - **Compare across seeds, not single runs.** The same seed does not give the same model output, especially with remote APIs. The comparison page shows the mean and standard deviation over seeds for this reason. diff --git a/examples/sweeps/governance_grid.yaml b/examples/sweeps/governance_grid.yaml index 9dfecc3..8c9ea29 100644 --- a/examples/sweeps/governance_grid.yaml +++ b/examples/sweeps/governance_grid.yaml @@ -13,6 +13,5 @@ vary: mild: -9.0 severe: -18.0 -seeds: [42, 43] parallel_runs: 2 agent_concurrency: 4 diff --git a/examples/sweeps/model_comparison.yaml b/examples/sweeps/model_comparison.yaml index db47cbc..ef653e6 100644 --- a/examples/sweeps/model_comparison.yaml +++ b/examples/sweeps/model_comparison.yaml @@ -1,5 +1,5 @@ # Does the underlying LLM change how the society behaves? -# Reruns the FULL scenario once per model and seed; nothing else changes. +# Reruns the FULL scenario once per model; nothing else changes. # # python -m verdi.sweep examples/sweeps/model_comparison.yaml --dry-run # plan + cost only # python -m verdi.sweep examples/sweeps/model_comparison.yaml # run (asks first) @@ -17,7 +17,6 @@ vary: # qwen: {name: Qwen3.8-27B} # deepseek: {name: deepseek-v4, base_url: https://api.deepseek.com/v1, api_key_env: DEEPSEEK_API_KEY} -seeds: [42, 43, 44] # same seeds for every model, so comparisons are fair parallel_runs: 2 # simulations at the same time agent_concurrency: 4 # model calls at the same time, per simulation # ticks: 10 # uncomment for a quick, cheap pilot first diff --git a/src/verdi/experiments/sweep.py b/src/verdi/experiments/sweep.py index b4e0fde..6173c7a 100644 --- a/src/verdi/experiments/sweep.py +++ b/src/verdi/experiments/sweep.py @@ -392,7 +392,10 @@ def describe(cfg: SweepConfig, runs: list[RunSpec]) -> str: lines += [f" {f.path.ljust(width)} {' | '.join(f.labels)}" for f in cfg.factors] else: lines.append("Varying: nothing (seed replication only)") - lines.append(f"Seeds: {', '.join(map(str, cfg.seeds))}") + if len(cfg.seeds) == 1: + lines.append(f"Seed: {cfg.seeds[0]} (add `seeds:` to repeat each condition)") + else: + lines.append(f"Seeds: {', '.join(map(str, cfg.seeds))}") if cfg.ticks: lines.append(f"Ticks: {cfg.ticks} (shortened from {cfg.base.get('total_ticks')})") lines += [