From 90cd1f79e6ed3b0aad49959d4d5aa3e0678958b4 Mon Sep 17 00:00:00 2001 From: GitHub CI Date: Sun, 23 Aug 2026 00:26:48 -0700 Subject: [PATCH] ops(vera): schedule a checkout-cache prune until the plugin prunes itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pr-reviewer's `CheckoutCache.prune()` is defined, documented in its module docstring, and unit-tested — and nothing in the plugin calls it (pr-reviewer-plugin#87). The cache reached 43 GiB across 1248 entries against its own 5 GiB / 50-entry / 1-hour caps, with 1247 already past the TTL, and took ava to 92% disk. A manual sweep reclaimed 36 GiB, and the cache regrew ~0.5 GiB in the next 90 minutes — roughly 8 GiB/day. So this is a schedule, not a cleanup: daily at 05:41, through the same wrapper as the other checks so a failure is a Discord alert rather than a silent log line. The policy is deliberately not just the plugin's TTL: * keep the 3 newest entries per repo — a pure TTL sweep evicts checkouts the next review immediately re-clones, and the cache exists so an unchanged head reaffirms in under a second; buying disk with latency on every repo is a bad trade * keep anything touched in the last hour regardless of count — reviews run 3-10 minutes with several in flight, and deleting a checkout out from under a running panel is the one way this could cause the failure it exists to prevent Both rules are additive, the script dry-runs by default, and it is scoped to delete only inside the cache root. DELETE ALL OF THIS when #87 ships: the script, the wrapper mode, the cron entry, and the README row. A cache with a documented, tested, uncalled pruner is indistinguishable from a cache with no pruner — which is exactly why it went unnoticed for two weeks and why this stopgap should not outlive the fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FnbQVnHNvTsvbnJ1pdUDdD --- README.md | 3 +- scripts/prune_checkout_cache.py | 122 ++++++++++++++++++++++++++++++++ scripts/vera-watchdog.sh | 15 +++- 3 files changed, 136 insertions(+), 4 deletions(-) create mode 100755 scripts/prune_checkout_cache.py diff --git a/README.md b/README.md index 435f69a..bac7ea1 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,8 @@ exit 2 = unreachable kept as distinct alarms). Every one of them exists because | `health` | `check_review_health.py` | is the gate still producing verdicts? (growth in unreviewed/exhausted, completion rate) | | `drift` | `check_card_drift.py` | does the live card still match the seed? | | `fallback` | `check_model_fallback.py` | did she silently answer from her fallback model? (gateway-metrics inference — protoAgent#2956) | -| `oauth` | `check_oauth_health.py` | is the subscription credential still signed in, refreshable, and coherent with `model.name`? | +| `oauth` | `check_oauth_health.py` | is the subscription credential still signed in, refreshable, and coherent with `model.name`? (a no-op on a gateway lane) | +| `prune` | `prune_checkout_cache.py` | **stopgap** — bound the checkout cache, since the plugin's own `prune()` is never called (pr-reviewer-plugin#87) | The wrapper runs **installed copies** in `~/.local/bin`, not `scripts/*.py` — this repo is also the deploy source, so a branch switch would silently disarm a guard that lived inside diff --git a/scripts/prune_checkout_cache.py b/scripts/prune_checkout_cache.py new file mode 100755 index 0000000..3e92bfa --- /dev/null +++ b/scripts/prune_checkout_cache.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Stopgap: bound Vera's checkout cache, because the plugin's own pruner never runs. + +`CheckoutCache.prune()` exists in pr-reviewer, is documented in its module docstring, +and is unit-tested — and nothing in the plugin ever calls it (filed as +pr-reviewer-plugin#87). Measured consequence on this deployment: 43 GiB across 1248 +entries against the module's own 5 GiB / 50-entry / 1-hour-TTL caps, with 1247 of those +entries already past the TTL. It was the largest consumer on the host and took it to +92% disk. After a manual sweep the cache regrew ~0.5 GiB in 90 minutes (~8 GiB/day), so +this is not a one-time cleanup — it needs a schedule until the upstream fix lands. + +DELETE THIS SCRIPT once #87 ships and the plugin prunes itself. It exists only because +a cache with a documented, tested, uncalled pruner is indistinguishable from a cache +with no pruner at all — and the disk is where you find out. + +POLICY, and why it is not simply the plugin's TTL: + + * Keep the N newest entries per repo (default 3). A pure TTL sweep would evict a + checkout the very next review re-clones — the cache exists so an unchanged head + reaffirms in under a second, and buying disk with latency on every repo is a bad + trade. Three covers the head plus a re-review or two. + * Keep ANYTHING touched inside the protect window (default 1h), regardless of count. + Reviews run 3-10 minutes and several can be in flight; deleting a checkout out from + under a running panel is the one way this script could cause the failure it exists + to prevent. The window is deliberately much longer than the longest observed review. + +Both rules are additive — an entry survives if EITHER holds. + +Usage (dry run by default; --apply to delete): + python3 scripts/prune_checkout_cache.py --container vera + python3 scripts/prune_checkout_cache.py --container vera --apply + +Exit 0 = swept (or nothing to do), 2 = could not reach the container. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys + +CACHE_ROOT = "/sandbox/pr-reviewer/checkouts" +DEFAULT_KEEP_PER_REPO = 3 +DEFAULT_PROTECT_MIN = 60 + +# Runs INSIDE the container: the cache lives on a named volume owned by uid 1001, and +# reaching it from the host would mean guessing the volume mountpoint and the uid. The +# body is kept dependency-free (stdlib only) because the image's python is not ours to +# add packages to. +_SWEEP = r""" +import json, os, shutil, sys, time +ROOT, KEEP, PROTECT, APPLY = sys.argv[1], int(sys.argv[2]), float(sys.argv[3]), sys.argv[4] == "apply" +now = time.time() +def dsize(p): + n = 0 + for dp, _, fs in os.walk(p): + for f in fs: + try: n += os.lstat(os.path.join(dp, f)).st_size + except OSError: pass + return n +deleted = kept = 0 +freed = 0 +errors = [] +if os.path.isdir(ROOT): + for repo in sorted(os.listdir(ROOT)): + rp = os.path.join(ROOT, repo) + if not os.path.isdir(rp): continue + try: + ents = [(os.path.getmtime(os.path.join(rp, d)), os.path.join(rp, d)) + for d in os.listdir(rp) if os.path.isdir(os.path.join(rp, d))] + except OSError as e: + errors.append(f"{repo}: {e}"); continue + ents.sort(reverse=True) + keep = {p for _, p in ents[:KEEP]} | {p for m, p in ents if now - m < PROTECT} + for _, p in ents: + if p in keep: + kept += 1 + continue + freed += dsize(p) + deleted += 1 + if APPLY: + shutil.rmtree(p, ignore_errors=True) +print(json.dumps({"deleted": deleted, "kept": kept, "freed_bytes": freed, "errors": errors[:5]})) +""" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--container", default="vera") + ap.add_argument("--keep-per-repo", type=int, default=DEFAULT_KEEP_PER_REPO) + ap.add_argument("--protect-min", type=float, default=DEFAULT_PROTECT_MIN) + ap.add_argument("--apply", action="store_true", help="actually delete (default is a dry run)") + args = ap.parse_args() + + try: + out = subprocess.run( + [ + "docker", "exec", args.container, "python3", "-c", _SWEEP, + CACHE_ROOT, str(args.keep_per_repo), str(args.protect_min * 60), + "apply" if args.apply else "dry", + ], + capture_output=True, text=True, timeout=1800, + ) + if out.returncode != 0: + print(f"UNREACHABLE: docker exec {args.container} failed: {out.stderr.strip()[:300]}") + return 2 + result = json.loads(out.stdout.strip().splitlines()[-1]) + except Exception as exc: # noqa: BLE001 — every failure here is operational + print(f"UNREACHABLE: {exc}") + return 2 + + gib = result["freed_bytes"] / 1024**3 + verb = "deleted" if args.apply else "would delete" + print(f"{verb} {result['deleted']} checkout entries ({gib:.1f} GiB), kept {result['kept']}") + for e in result.get("errors") or []: + print(f" warning: {e}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vera-watchdog.sh b/scripts/vera-watchdog.sh index 06125ad..9cff2e4 100755 --- a/scripts/vera-watchdog.sh +++ b/scripts/vera-watchdog.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runner + alerter for Vera's four watchdogs — health, drift, fallback, oauth +# Runner + alerter for Vera's watchdogs — health, drift, fallback, oauth, prune # (qaEngineer#37 shipped the first two; the model-lane pair arrived with the move to a # native Claude subscription). # @@ -18,6 +18,7 @@ # live inside the thing it watches. Refresh these copies when the repo version changes: # # install -m 644 ~/dev/qaEngineer/scripts/vera_api.py ~/.local/bin/vera_api.py +# install -m 755 ~/dev/qaEngineer/scripts/prune_checkout_cache.py ~/.local/bin/vera-prune-cache.py # install -m 755 ~/dev/qaEngineer/scripts/check_review_health.py ~/.local/bin/vera-review-health.py # install -m 755 ~/dev/qaEngineer/scripts/check_card_drift.py ~/.local/bin/vera-card-drift.py # install -m 755 ~/dev/qaEngineer/scripts/check_model_fallback.py ~/.local/bin/vera-model-fallback.py @@ -32,7 +33,7 @@ # agent) are deliberately different alerts — the scripts draw that line on purpose, and # collapsing it would let an outage read as a clean gate. # -# Usage: vera-watchdog.sh health|drift|fallback|oauth [extra args passed to the check] +# Usage: vera-watchdog.sh health|drift|fallback|oauth|prune [extra args passed to the check] # Exit: passes the underlying check's exit code through (0 ok, 1 verdict, 2 unreachable) set -uo pipefail @@ -112,8 +113,16 @@ case "$MODE" in # outlive its refresh token with no traffic to reveal it. out="$("$BIN/vera-oauth-health.py" --container vera "$@" 2>&1)"; rc=$? ;; + prune) + # STOPGAP, not a watchdog: bound the checkout cache, because pr-reviewer's own + # CheckoutCache.prune() is defined, documented, unit-tested and never called + # (pr-reviewer-plugin#87). It reached 43 GiB / 1248 entries against its own + # 5 GiB / 50-entry caps and took ava to 92% disk. Delete this mode when #87 ships. + # Runs with --apply here; the underlying script dry-runs by default. + out="$("$BIN/vera-prune-cache.py" --container vera --apply "$@" 2>&1)"; rc=$? + ;; *) - echo "usage: $(basename "$0") health|drift|fallback|oauth" >&2; exit 64 ;; + echo "usage: $(basename "$0") health|drift|fallback|oauth|prune" >&2; exit 64 ;; esac echo "$out"