From b457ffca015df25049ce8668d355567556b1bcac Mon Sep 17 00:00:00 2001 From: defangdevs Date: Fri, 11 Sep 2026 22:43:05 +0000 Subject: [PATCH 1/3] fix(sessions): a died hook-* session never holds a capacity slot (#523) capacity_check() counted a `died` entry as `pending` (its `stopped` flag is never set, issue #516) and its lingering post-mortem-shell pane as `live`, so a crashed hook-* session held its slot forever -- only `agent-box-session rm` ever freed it. Four such corpses made every standing watch inert with nothing actually running, the same shape as the pre-#280 bug this admission policy was built to fix. Exclude any session the registry flags `died` from both `live` and `pending`, in the single shared session-capacity.py the CLI, supervisor, webhook spawner and settings daemon all admit through since #668. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0188bYSAQvRmVjNDtsgznirR --- modules/agent-box.nix | 30 +++++++++++++++++-- modules/src/lib/session-capacity.py | 15 +++++++++- .../bin/agent-box-session-capacity | 15 +++++++++- .../agent-box-settings/bin/agent-box-settings | 15 +++++++++- tests/test-session-capacity.py | 10 +++++++ 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/modules/agent-box.nix b/modules/agent-box.nix index e189cfef..786a1fe8 100644 --- a/modules/agent-box.nix +++ b/modules/agent-box.nix @@ -2043,8 +2043,21 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc + # A crash is flagged `died`, not `stopped` (issue #516), so it stays + # listed and attachable for inspection -- but that leaves its pane a + # post-mortem shell tmux still reports, and its registry entry still + # lacking `stopped`, so before this it held its slot for good: nothing + # but `agent-box-session rm` ever cleared it, and enough of them made + # every standing watch inert with nothing actually running (issue #523). + # Nobody respawns a died session either -- same as a stopped one -- so it + # is excluded from both `live` and `pending`, not only the latter: its + # pane would otherwise still count it as running. + died = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("died") is not None} + live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True} + if isinstance(entry, dict) and entry.get("stopped") is not True + and entry.get("died") is None} used = live | pending targets = set(targets) if spawning: @@ -2125,8 +2138,21 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc + # A crash is flagged `died`, not `stopped` (issue #516), so it stays + # listed and attachable for inspection -- but that leaves its pane a + # post-mortem shell tmux still reports, and its registry entry still + # lacking `stopped`, so before this it held its slot for good: nothing + # but `agent-box-session rm` ever cleared it, and enough of them made + # every standing watch inert with nothing actually running (issue #523). + # Nobody respawns a died session either -- same as a stopped one -- so it + # is excluded from both `live` and `pending`, not only the latter: its + # pane would otherwise still count it as running. + died = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("died") is not None} + live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True} + if isinstance(entry, dict) and entry.get("stopped") is not True + and entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/modules/src/lib/session-capacity.py b/modules/src/lib/session-capacity.py index 892cb466..45bc6383 100644 --- a/modules/src/lib/session-capacity.py +++ b/modules/src/lib/session-capacity.py @@ -57,8 +57,21 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc + # A crash is flagged `died`, not `stopped` (issue #516), so it stays + # listed and attachable for inspection -- but that leaves its pane a + # post-mortem shell tmux still reports, and its registry entry still + # lacking `stopped`, so before this it held its slot for good: nothing + # but `agent-box-session rm` ever cleared it, and enough of them made + # every standing watch inert with nothing actually running (issue #523). + # Nobody respawns a died session either -- same as a stopped one -- so it + # is excluded from both `live` and `pending`, not only the latter: its + # pane would otherwise still count it as running. + died = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("died") is not None} + live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True} + if isinstance(entry, dict) and entry.get("stopped") is not True + and entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity b/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity index 8da2b056..e2132f77 100644 --- a/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity +++ b/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity @@ -58,8 +58,21 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc + # A crash is flagged `died`, not `stopped` (issue #516), so it stays + # listed and attachable for inspection -- but that leaves its pane a + # post-mortem shell tmux still reports, and its registry entry still + # lacking `stopped`, so before this it held its slot for good: nothing + # but `agent-box-session rm` ever cleared it, and enough of them made + # every standing watch inert with nothing actually running (issue #523). + # Nobody respawns a died session either -- same as a stopped one -- so it + # is excluded from both `live` and `pending`, not only the latter: its + # pane would otherwise still count it as running. + died = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("died") is not None} + live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True} + if isinstance(entry, dict) and entry.get("stopped") is not True + and entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings b/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings index a633be01..b9adcbee 100644 --- a/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings +++ b/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings @@ -389,8 +389,21 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc + # A crash is flagged `died`, not `stopped` (issue #516), so it stays + # listed and attachable for inspection -- but that leaves its pane a + # post-mortem shell tmux still reports, and its registry entry still + # lacking `stopped`, so before this it held its slot for good: nothing + # but `agent-box-session rm` ever cleared it, and enough of them made + # every standing watch inert with nothing actually running (issue #523). + # Nobody respawns a died session either -- same as a stopped one -- so it + # is excluded from both `live` and `pending`, not only the latter: its + # pane would otherwise still count it as running. + died = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("died") is not None} + live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True} + if isinstance(entry, dict) and entry.get("stopped") is not True + and entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/tests/test-session-capacity.py b/tests/test-session-capacity.py index 3503690d..c52fd87d 100644 --- a/tests/test-session-capacity.py +++ b/tests/test-session-capacity.py @@ -37,6 +37,16 @@ def test_stopped_is_free_only_after_pane_exits(self): with self.assertRaises(capacity.SessionCapacityError): capacity.capacity_check(sessions, ["new"], live={"stopped"}, limit=1) + def test_died_never_holds_a_slot_even_with_its_pane_still_up(self): + # A crash is flagged `died`, not `stopped` (issue #516), so its pane + # lingers as a post-mortem shell tmux still reports, and its entry + # never gets `stopped` either -- before this fix that meant a died + # session held its slot for good (issue #523), unlike a stopped one + # which frees its slot once the pane actually exits. + sessions = {"crashed": {"died": 1}} + capacity.capacity_check(sessions, ["new"], live=set(), limit=1) + capacity.capacity_check(sessions, ["new"], live={"crashed"}, limit=1) + def test_restart_does_not_need_a_second_slot(self): capacity.capacity_check({"a": {}, "b": {}}, ["a"], live={"a"}, limit=1) From 8670abc53958659c5102ceea2b22cc534db5aa05 Mon Sep 17 00:00:00 2001 From: defangdevs Date: Fri, 11 Sep 2026 22:46:43 +0000 Subject: [PATCH 2/3] fix: avoid W503 in session-capacity.py's new pending filter writePython3Bin's flakeIgnore replaces pycodestyle's defaults, so W503 (normally off) became a build error on the `and` line break split across two continuation lines in capacity_check()'s `pending` comprehension -- caught by CI's native-checks build, not by anything runnable without Nix. Rewritten as three `if` clauses (implicitly ANDed in a comprehension), which sidesteps the rule instead of choosing a side of it. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0188bYSAQvRmVjNDtsgznirR --- modules/agent-box.nix | 10 ++++++---- modules/src/lib/session-capacity.py | 5 +++-- .../bin/agent-box-session-capacity | 5 +++-- .../payloads/agent-box-settings/bin/agent-box-settings | 5 +++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/modules/agent-box.nix b/modules/agent-box.nix index 786a1fe8..c144df13 100644 --- a/modules/agent-box.nix +++ b/modules/agent-box.nix @@ -2056,8 +2056,9 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): if isinstance(entry, dict) and entry.get("died") is not None} live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True - and entry.get("died") is None} + if isinstance(entry, dict) + if entry.get("stopped") is not True + if entry.get("died") is None} used = live | pending targets = set(targets) if spawning: @@ -2151,8 +2152,9 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): if isinstance(entry, dict) and entry.get("died") is not None} live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True - and entry.get("died") is None} + if isinstance(entry, dict) + if entry.get("stopped") is not True + if entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/modules/src/lib/session-capacity.py b/modules/src/lib/session-capacity.py index 45bc6383..d0b071cc 100644 --- a/modules/src/lib/session-capacity.py +++ b/modules/src/lib/session-capacity.py @@ -70,8 +70,9 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): if isinstance(entry, dict) and entry.get("died") is not None} live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True - and entry.get("died") is None} + if isinstance(entry, dict) + if entry.get("stopped") is not True + if entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity b/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity index e2132f77..46473ba1 100644 --- a/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity +++ b/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity @@ -71,8 +71,9 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): if isinstance(entry, dict) and entry.get("died") is not None} live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True - and entry.get("died") is None} + if isinstance(entry, dict) + if entry.get("stopped") is not True + if entry.get("died") is None} used = live | pending targets = set(targets) if spawning: diff --git a/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings b/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings index b9adcbee..21c8a320 100644 --- a/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings +++ b/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings @@ -402,8 +402,9 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): if isinstance(entry, dict) and entry.get("died") is not None} live = live - died pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) and entry.get("stopped") is not True - and entry.get("died") is None} + if isinstance(entry, dict) + if entry.get("stopped") is not True + if entry.get("died") is None} used = live | pending targets = set(targets) if spawning: From 694ec3410ea8aa8c06011f85c0f8f1bacde657cc Mon Sep 17 00:00:00 2001 From: defangdevs Date: Fri, 11 Sep 2026 23:01:54 +0000 Subject: [PATCH 3/3] fix: a died entry must stay its own revival candidate CI's VM (sessions) lane caught what the native tests couldn't: excluding a `died` entry from `pending` (not just from counting against everyone else) made it structurally impossible for the supervisor to ever re-admit that SAME name. `agent-box-session restart` on a died session kills its old pane and relies on the supervisor's own spawn admission to start a fresh one and clear the flag -- but with the died name absent from `pending`, it could never again appear in `admitted`, so the restart hung forever (CI: "a spawn clears a stale died flag" timed out at 120s). Keep `pending` unfiltered (a died entry stays a valid candidate for its own restart), and instead exclude `died` names only from `used` (the total charged against the limit) and from the `live` count `available` subtracts -- so a died session's corpse still doesn't cost anyone ELSE a slot, but remains admissible for itself. Added a regression test at the unit level (`test_died_entry_remains_its_own_revival_candidate`) so this deadlock shape doesn't need a 12-minute VM run to catch again. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0188bYSAQvRmVjNDtsgznirR --- modules/agent-box.nix | 60 +++++++++---------- modules/src/lib/session-capacity.py | 30 +++++----- .../bin/agent-box-session-capacity | 30 +++++----- .../agent-box-settings/bin/agent-box-settings | 30 +++++----- tests/test-session-capacity.py | 13 ++++ 5 files changed, 83 insertions(+), 80 deletions(-) diff --git a/modules/agent-box.nix b/modules/agent-box.nix index c144df13..73aa69d3 100644 --- a/modules/agent-box.nix +++ b/modules/agent-box.nix @@ -2043,26 +2043,24 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc - # A crash is flagged `died`, not `stopped` (issue #516), so it stays - # listed and attachable for inspection -- but that leaves its pane a - # post-mortem shell tmux still reports, and its registry entry still - # lacking `stopped`, so before this it held its slot for good: nothing - # but `agent-box-session rm` ever cleared it, and enough of them made - # every standing watch inert with nothing actually running (issue #523). - # Nobody respawns a died session either -- same as a stopped one -- so it - # is excluded from both `live` and `pending`, not only the latter: its - # pane would otherwise still count it as running. + pending = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("stopped") is not True} + # A crash is flagged `died`, not `stopped` (issue #516), so a died entry + # stays in `pending` -- it must remain its own candidate for revival by + # `agent-box-session restart`, or a stale flag on a session that already + # respawned fine (a race the pane epilogue and the supervisor both write + # `died`/`stopped` into) can never be admitted again to clear it. But its + # pane is a post-mortem shell doing no real work, so unlike a genuinely + # running session it must not cost anyone ELSE a slot: before this fix a + # died session's pane counted as real, running capacity forever, and + # enough of them stalled every OTHER pending session too, with nothing to + # clear it but `agent-box-session rm` (issue #523). died = {name for name, entry in sessions.items() if isinstance(entry, dict) and entry.get("died") is not None} - live = live - died - pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) - if entry.get("stopped") is not True - if entry.get("died") is None} - used = live | pending + used = (live | pending) - died targets = set(targets) if spawning: - available = max(0, limit - len(live)) + available = max(0, limit - len(live - died)) admitted = live | set(sorted(pending - live)[:available]) allowed = targets <= admitted else: @@ -2139,26 +2137,24 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc - # A crash is flagged `died`, not `stopped` (issue #516), so it stays - # listed and attachable for inspection -- but that leaves its pane a - # post-mortem shell tmux still reports, and its registry entry still - # lacking `stopped`, so before this it held its slot for good: nothing - # but `agent-box-session rm` ever cleared it, and enough of them made - # every standing watch inert with nothing actually running (issue #523). - # Nobody respawns a died session either -- same as a stopped one -- so it - # is excluded from both `live` and `pending`, not only the latter: its - # pane would otherwise still count it as running. + pending = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("stopped") is not True} + # A crash is flagged `died`, not `stopped` (issue #516), so a died entry + # stays in `pending` -- it must remain its own candidate for revival by + # `agent-box-session restart`, or a stale flag on a session that already + # respawned fine (a race the pane epilogue and the supervisor both write + # `died`/`stopped` into) can never be admitted again to clear it. But its + # pane is a post-mortem shell doing no real work, so unlike a genuinely + # running session it must not cost anyone ELSE a slot: before this fix a + # died session's pane counted as real, running capacity forever, and + # enough of them stalled every OTHER pending session too, with nothing to + # clear it but `agent-box-session rm` (issue #523). died = {name for name, entry in sessions.items() if isinstance(entry, dict) and entry.get("died") is not None} - live = live - died - pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) - if entry.get("stopped") is not True - if entry.get("died") is None} - used = live | pending + used = (live | pending) - died targets = set(targets) if spawning: - available = max(0, limit - len(live)) + available = max(0, limit - len(live - died)) admitted = live | set(sorted(pending - live)[:available]) allowed = targets <= admitted else: diff --git a/modules/src/lib/session-capacity.py b/modules/src/lib/session-capacity.py index d0b071cc..872f0324 100644 --- a/modules/src/lib/session-capacity.py +++ b/modules/src/lib/session-capacity.py @@ -57,26 +57,24 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc - # A crash is flagged `died`, not `stopped` (issue #516), so it stays - # listed and attachable for inspection -- but that leaves its pane a - # post-mortem shell tmux still reports, and its registry entry still - # lacking `stopped`, so before this it held its slot for good: nothing - # but `agent-box-session rm` ever cleared it, and enough of them made - # every standing watch inert with nothing actually running (issue #523). - # Nobody respawns a died session either -- same as a stopped one -- so it - # is excluded from both `live` and `pending`, not only the latter: its - # pane would otherwise still count it as running. + pending = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("stopped") is not True} + # A crash is flagged `died`, not `stopped` (issue #516), so a died entry + # stays in `pending` -- it must remain its own candidate for revival by + # `agent-box-session restart`, or a stale flag on a session that already + # respawned fine (a race the pane epilogue and the supervisor both write + # `died`/`stopped` into) can never be admitted again to clear it. But its + # pane is a post-mortem shell doing no real work, so unlike a genuinely + # running session it must not cost anyone ELSE a slot: before this fix a + # died session's pane counted as real, running capacity forever, and + # enough of them stalled every OTHER pending session too, with nothing to + # clear it but `agent-box-session rm` (issue #523). died = {name for name, entry in sessions.items() if isinstance(entry, dict) and entry.get("died") is not None} - live = live - died - pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) - if entry.get("stopped") is not True - if entry.get("died") is None} - used = live | pending + used = (live | pending) - died targets = set(targets) if spawning: - available = max(0, limit - len(live)) + available = max(0, limit - len(live - died)) admitted = live | set(sorted(pending - live)[:available]) allowed = targets <= admitted else: diff --git a/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity b/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity index 46473ba1..94d7a730 100644 --- a/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity +++ b/tests/golden/vm/payloads/agent-box-session-capacity/bin/agent-box-session-capacity @@ -58,26 +58,24 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc - # A crash is flagged `died`, not `stopped` (issue #516), so it stays - # listed and attachable for inspection -- but that leaves its pane a - # post-mortem shell tmux still reports, and its registry entry still - # lacking `stopped`, so before this it held its slot for good: nothing - # but `agent-box-session rm` ever cleared it, and enough of them made - # every standing watch inert with nothing actually running (issue #523). - # Nobody respawns a died session either -- same as a stopped one -- so it - # is excluded from both `live` and `pending`, not only the latter: its - # pane would otherwise still count it as running. + pending = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("stopped") is not True} + # A crash is flagged `died`, not `stopped` (issue #516), so a died entry + # stays in `pending` -- it must remain its own candidate for revival by + # `agent-box-session restart`, or a stale flag on a session that already + # respawned fine (a race the pane epilogue and the supervisor both write + # `died`/`stopped` into) can never be admitted again to clear it. But its + # pane is a post-mortem shell doing no real work, so unlike a genuinely + # running session it must not cost anyone ELSE a slot: before this fix a + # died session's pane counted as real, running capacity forever, and + # enough of them stalled every OTHER pending session too, with nothing to + # clear it but `agent-box-session rm` (issue #523). died = {name for name, entry in sessions.items() if isinstance(entry, dict) and entry.get("died") is not None} - live = live - died - pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) - if entry.get("stopped") is not True - if entry.get("died") is None} - used = live | pending + used = (live | pending) - died targets = set(targets) if spawning: - available = max(0, limit - len(live)) + available = max(0, limit - len(live - died)) admitted = live | set(sorted(pending - live)[:available]) allowed = targets <= admitted else: diff --git a/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings b/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings index 21c8a320..f9b1a9ee 100644 --- a/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings +++ b/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings @@ -389,26 +389,24 @@ def capacity_check(sessions, targets=(), spawning=False, live=None, limit=None): live = capacity_live() if live is None else set(live) except (OSError, ValueError, capacity_subprocess.TimeoutExpired) as exc: raise SessionCapacityError("Cannot check session capacity: %s" % exc) from exc - # A crash is flagged `died`, not `stopped` (issue #516), so it stays - # listed and attachable for inspection -- but that leaves its pane a - # post-mortem shell tmux still reports, and its registry entry still - # lacking `stopped`, so before this it held its slot for good: nothing - # but `agent-box-session rm` ever cleared it, and enough of them made - # every standing watch inert with nothing actually running (issue #523). - # Nobody respawns a died session either -- same as a stopped one -- so it - # is excluded from both `live` and `pending`, not only the latter: its - # pane would otherwise still count it as running. + pending = {name for name, entry in sessions.items() + if isinstance(entry, dict) and entry.get("stopped") is not True} + # A crash is flagged `died`, not `stopped` (issue #516), so a died entry + # stays in `pending` -- it must remain its own candidate for revival by + # `agent-box-session restart`, or a stale flag on a session that already + # respawned fine (a race the pane epilogue and the supervisor both write + # `died`/`stopped` into) can never be admitted again to clear it. But its + # pane is a post-mortem shell doing no real work, so unlike a genuinely + # running session it must not cost anyone ELSE a slot: before this fix a + # died session's pane counted as real, running capacity forever, and + # enough of them stalled every OTHER pending session too, with nothing to + # clear it but `agent-box-session rm` (issue #523). died = {name for name, entry in sessions.items() if isinstance(entry, dict) and entry.get("died") is not None} - live = live - died - pending = {name for name, entry in sessions.items() - if isinstance(entry, dict) - if entry.get("stopped") is not True - if entry.get("died") is None} - used = live | pending + used = (live | pending) - died targets = set(targets) if spawning: - available = max(0, limit - len(live)) + available = max(0, limit - len(live - died)) admitted = live | set(sorted(pending - live)[:available]) allowed = targets <= admitted else: diff --git a/tests/test-session-capacity.py b/tests/test-session-capacity.py index c52fd87d..a04631c1 100644 --- a/tests/test-session-capacity.py +++ b/tests/test-session-capacity.py @@ -47,6 +47,19 @@ def test_died_never_holds_a_slot_even_with_its_pane_still_up(self): capacity.capacity_check(sessions, ["new"], live=set(), limit=1) capacity.capacity_check(sessions, ["new"], live={"crashed"}, limit=1) + def test_died_entry_remains_its_own_revival_candidate(self): + # A died entry must still admit ITSELF: the supervisor's own spawn + # decision for the name being revived has to see it in `pending`, or + # `agent-box-session restart` on a died session can never re-admit it + # to clear the stale flag -- a real deadlock hit in CI (issue #523): + # excluding a died name from `pending` too, not just from counting + # against everyone else, made `agent-box-session restart` on a died + # session hang forever, because the one name being spawned was never + # a candidate for its own slot. + sessions = {"revived": {"died": 99}} + capacity.capacity_check(sessions, ["revived"], spawning=True, + live=set(), limit=1) + def test_restart_does_not_need_a_second_slot(self): capacity.capacity_check({"a": {}, "b": {}}, ["a"], live={"a"}, limit=1)