diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6786a3..4df6ec8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,15 @@ jobs: assert m["archetype"]["soul"].strip(), "archetype soul present" print("manifest ok:", m["id"], "-", len(m["plugins"]), "members") PY + - name: Watchdog check logic + # The attribution rule in check_model_fallback.py and the verdict rules in + # check_oauth_health.py decide whether #alerts gets woken — a bug there is + # either a silent degrade nobody hears about or a pager that cries wolf, and + # neither shows up in a smoke run against a healthy container. Stdlib + # unittest: CI here is python3 + PyYAML, and a watchdog suite earning a + # dependency install is the wrong trade. + run: python3 -m unittest discover tests -v + - name: README pins match bundle manifest and Dockerfile # Prevents the member-table and base-image prose from drifting out of sync # with the canonical pins in protoagent.bundle.yaml and the Dockerfile FROM line. diff --git a/Dockerfile b/Dockerfile index b0eefe8..af34e8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ # protoAgent core forward on the same image roll — core and member bumps are # decoupled. Bump this deliberately (and re-verify), keeping it in step with the # manifest's `verified_against`. Tag format is bare semver (no `v` prefix). -FROM ghcr.io/protolabsai/protoagent:0.137.1 +FROM ghcr.io/protolabsai/protoagent:0.145.0 USER root @@ -27,7 +27,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ # Bake the bundle members at their RELEASE TAGS (both public — no build secrets). # The tags mirror protoagent.bundle.yaml's pins; bump both together (the manifest # is the source of truth, this bake is its image form). -ARG GITHUB_PLUGIN_REF=v0.3.0 +ARG GITHUB_PLUGIN_REF=v0.5.0 RUN git clone --depth 1 --branch "${GITHUB_PLUGIN_REF}" \ https://github.com/protoLabsAI/github-plugin.git /opt/protoagent/plugins/github \ && rm -rf /opt/protoagent/plugins/github/.git diff --git a/README.md b/README.md index 99ea2e8..435f69a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ python -m server plugin install https://github.com/protoLabsAI/qaEngineer | Member | Pin | Role | |---|---|---| | `workflows` (builtin) | core | the recipe engine the review panels run on | -| [github-plugin](https://github.com/protoLabsAI/github-plugin) | v0.3.0 | the verdict surface — formal Review API tools with CI-terminal + self-review guards inside the tools | +| [github-plugin](https://github.com/protoLabsAI/github-plugin) | v0.5.0 | the verdict surface — formal Review API tools with CI-terminal + self-review guards inside the tools | | [pr-reviewer-plugin](https://github.com/protoLabsAI/pr-reviewer-plugin) | v0.35.0 | the machinery — webhook chokepoint, structural trigger, panel dispatch, evidence grounding, convergence, approve-on-green sweep, on-demand summon, telemetry + eval | Persona: [`SOUL.md`](./SOUL.md) (Vera — verdict system, three-layer verification, 80% bar, @@ -84,7 +84,44 @@ plus prior-round context. In practice that's **~37k tokens per finder**. - **No aggressive TPM cap.** A free tier at 12,000 tokens/minute rejects a single finder outright. - Routed through a gateway alias, so swapping models is a gateway edit rather than a code - change. Model settings are host-scoped (ADR 0047). + change. Model settings are host-scoped (ADR 0047) — **or** a native subscription lane + (see below), which is not a gateway edit at all. + +**The reference host's lane (since 2026-08-23):** `protolabs/smart` on the gateway +(`model.provider: openai`), with `routing.fallback_models: ["protolabs/cloud"]` behind +it. She ran a native Claude subscription (`anthropic-oauth` / `claude-sonnet-5`) from +2026-08-21 to 08-23 and it reviewed well — ~25% faster, same grounding rate — but a +five-finder panel across a fleet's worth of repos **exhausted the subscription's 7-day +quota in under two days** (`anthropic-ratelimit-unified-7d-status: rejected`, +`retry-after: 356343` = 4.1 days). A reviewer that cannot review for four days is not a +reviewer, so the gateway is the sustainable primary and a subscription is the treat. +Set `model.name` and `model.provider` in the SAME config POST either way: they are one +decision (protoAgent#2623). + +**The old native-OAuth shape, for reference:** Native OAuth **bypasses the gateway entirely** (ADR 0097) — the fallback alias is +reachable only because protoAgent#2571 lets a namespaced slot name opt out of the native +provider, so `model.api_base` + a gateway key must stay set even though the primary never +uses them. Set `model.name` and `model.provider` in the SAME config POST: they are one +decision (protoAgent#2623), and a native provider paired with a `protolabs/*` name is +rejected on every call. + +**A fallback is SILENT.** protoAgent wires langchain's `ModelFallbackMiddleware` raw, and +that middleware swallows the primary's exception with no log, no counter and no event — +found on core 0.144.0 and filed as protoAgent#2956. So a dead subscription doesn't break +the review; it quietly changes which model writes the verdict. That is what +`scripts/check_model_fallback.py` exists to catch, by inference from gateway metrics: +protoAgent-UA traffic arriving at the gateway from Vera's container *is* a fallback, +because her primary never goes there. + +**#2956 is FIXED in core 0.145.0**, which the pins above now carry: +`ObservableModelFallbackMiddleware` logs a WARNING and publishes a `model.fallback` bus +event (ADR 0039). The inference script is therefore scheduled for deletion — but not +yet, and the distinction matters: **pinning a version is not running it.** Vera rolls on +watchtower after a merge, so between the pin landing and the roll completing she is on +the old core with no event at all. Retire the script once the running instance reports +0.145.0 *and* the event has been seen firing; deleting the inference before its +replacement is observed working would leave the silent-degrade window covered by +neither. **Reviews are not cheap.** One structural review is nine LLM steps and 5–9 minutes of wall clock. On a hosted frontier model that's roughly $0.12–0.15 each; on local inference @@ -164,7 +201,7 @@ from the payload): ## Deploying Vera (the reference host) This repo doubles as Vera's image source: `Dockerfile` = stock protoAgent (**pinned -base** — `protoagent:0.137.1`, in step with the manifest's `verified_against`; bump +base** — `protoagent:0.145.0`, in step with the manifest's `verified_against`; bump deliberately so a member-pin bump can't drag the core forward on the same roll) + node/`clawpatch` + the bundle members baked at their manifest pins + `deploy/vera.langgraph-config.yaml` (seed, not force) + `SOUL.md`. @@ -191,6 +228,31 @@ static check can't see a *running* instance whose live config drifted; `scripts/check_card_drift.py` is the runtime half — point it at the (tailnet-only) card from the ava fleet cron: `python3 scripts/check_card_drift.py` (exit 1 on drift). +### The watchdogs + +Four checks, all run from the ava fleet cron through `scripts/vera-watchdog.sh`, which is +the piece that makes a failure LOUD (a Discord `#alerts` post, with exit 1 = verdict and +exit 2 = unreachable kept as distinct alarms). Every one of them exists because Vera fails +*quietly* — a starved panel, a drifted card, a swapped model underneath a verdict. + +| Mode | Check | Asks | +|---|---|---| +| `health` | `check_review_health.py` | is the gate still producing verdicts? (growth in unreviewed/exhausted, completion rate) | +| `drift` | `check_card_drift.py` | does the live card still match the seed? | +| `fallback` | `check_model_fallback.py` | did she silently answer from her fallback model? (gateway-metrics inference — protoAgent#2956) | +| `oauth` | `check_oauth_health.py` | is the subscription credential still signed in, refreshable, and coherent with `model.name`? | + +The wrapper runs **installed copies** in `~/.local/bin`, not `scripts/*.py` — this repo is +also the deploy source, so a branch switch would silently disarm a guard that lived inside +it. Re-`install` them after changing a script (there is no auto-update) — the install lines +are in the wrapper's header. That includes **`scripts/vera_api.py`**, the shared +operator-API helper: Python puts a script's own directory on `sys.path[0]`, so a flat +module installed alongside imports cleanly — and a check installed *without* it dies on +ImportError. `python3 -m unittest discover tests` covers the attribution and verdict rules +(CI runs it). It reads `DISCORD_WEBHOOK_ALERTS` from `infisical run`, so an +expired Infisical session downgrades every alarm to a log line nobody reads; that is worth +checking whenever the alerts channel goes quiet for a suspiciously long time. + ## Other orgs Nothing here is protoLabs-specific except the pins and the seed: the bundle installs into diff --git a/protoagent.bundle.yaml b/protoagent.bundle.yaml index 50a4674..64a0f37 100644 --- a/protoagent.bundle.yaml +++ b/protoagent.bundle.yaml @@ -25,11 +25,11 @@ description: >- own work; never posts a blocking verdict against pending CI. # The core version this pin set was last verified against (ADR 0049 rule 2). -verified_against: 0.137.1 +verified_against: 0.145.0 plugins: - { id: workflows, builtin: true } # the recipe engine (code-review panels run through it) - - { id: github, url: https://github.com/protoLabsAI/github-plugin, ref: v0.3.0 } + - { id: github, url: https://github.com/protoLabsAI/github-plugin, ref: v0.5.0 } - { id: pr-reviewer, url: https://github.com/protoLabsAI/pr-reviewer-plugin, ref: v0.35.0 } enabled: [workflows, github, pr-reviewer] # suggested turn-on list (applied to plugins.enabled) diff --git a/scripts/check_model_fallback.py b/scripts/check_model_fallback.py new file mode 100755 index 0000000..501449c --- /dev/null +++ b/scripts/check_model_fallback.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Health check: is Vera silently answering from her FALLBACK model? + +`routing.fallback_models` is Vera's only degrade path, and when it fires it fires +SILENTLY. protoAgent wires langchain's `ModelFallbackMiddleware` raw (graph/agent.py), +and that middleware swallows the primary's exception with no log, no counter, and no +event — a successful fallback is byte-for-byte indistinguishable from a normal turn. +Filed upstream as protoAgent#2956 and **FIXED in core 0.145.0** — which this repo now +pins: `ObservableModelFallbackMiddleware` logs a WARNING and publishes a `model.fallback` +bus event (ADR 0039). So this script is already living on borrowed time, deliberately: + +RETIRE IT once the RUNNING instance is on 0.145.0 *and* the event has been seen firing. +Pinning a version is not the same as running it — Vera rolls on watchtower after this +merges, and until then she is on 0.137.1 with no event at all. Deleting the inference +before its replacement is observed working would leave the silent-degrade window +uncovered by both, which is the one outcome worth avoiding. Once the event is confirmed, +delete this file: the event is ground truth where this is an inference from traffic. + +THE RULE. Ask the LIVE CONFIG which models are the fallback, then count only gateway +traffic requesting THOSE: + + a protoAgent-UA chat completion from Vera's container IP, + whose `requested_model` is in `routing.fallback_models` == a fallback + +This is deliberately NOT "any gateway traffic from her container". That shortcut worked +only while her primary was a native-OAuth subscription bypassing the gateway entirely +(ADR 0097) — and it silently inverted the moment she moved back to a gateway primary on +2026-08-23, when it would have called every ordinary review a fallback and paged #alerts +every 15 minutes. Reading the fallback list from the live config instead means the check +is correct for BOTH lane shapes, and keeps working across the next switch without anyone +remembering to come back and edit it. + +Two things share that gateway key and must NOT be counted: + + * clawpatch (the protoPatch structural engine) — the pr-reviewer plugin exports + OPENAI_API_KEY to the subprocess, and it calls protolabs/smart too. It is a node + process, so it lands under `user_agent="node"` while protoAgent's own calls carry + `user_agent="protoAgent/0.1 (+...)"`. That label is the whole discriminator. + * embeddings (`qwen3-embedding`) — a different route and a different model; knowledge + embeddings are off today but that can be flipped from the console, so filter by + route rather than trusting the config. + +ALARM ON GROWTH, NOT ON DEPTH. `litellm_proxy_total_requests_metric_total` is a +lifetime counter, so a fixed ceiling would latch red forever after the first bad hour — +the same trap check_review_health.py documents. State carries the previous run's total; +a run alarms only on NEW fallback traffic since the last one. + +COOLDOWN, because the loud failure here is not one 429. A subscription that is +rate-limited stays rate-limited for a window, and every review in that window falls +back — alerting per run would post to #alerts every few minutes for an hour and train +everyone to mute the channel. After an alert, stay quiet for --cooldown-min while +still tracking the counter, then re-alarm if it is STILL growing. Sustained degradation +gets through; a burst gets one post. + +Run it from the ava fleet cron (the gateway's metrics port is container-local): + + python3 scripts/check_model_fallback.py --container vera + +Exit 0 = primary lane healthy (or inside a cooldown); exit 1 = fallback traffic since +the last run; exit 2 = could not reach the gateway or the container (an operational +error, NOT a verdict — a dead scraper must not read as a clean lane). +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +from vera_api import operator_api_get + +DEFAULT_STATE = Path.home() / ".cache" / "vera-model-fallback.json" +# The gateway publishes on the shared ai_default net; from ava's host namespace it is +# reachable on the published port. Overridable for a different host/stack. +DEFAULT_METRICS = "http://localhost:4000/metrics/" +# protoAgent stamps its own User-Agent on every model call it makes; clawpatch (node) +# and ad-hoc curl do not. This prefix IS the "was it the agent itself" test. +AGENT_UA_PREFIX = "protoAgent" +# Embeddings ride the same key and container but are not a chat lane. +EMBEDDING_ROUTE = "/v1/embeddings" +DEFAULT_COOLDOWN_MIN = 60 + +_SAMPLE = re.compile(r"^litellm_proxy_total_requests_metric_total\{(?P.*)\}\s+(?P[0-9.eE+-]+)$") +_LABEL = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"') + + +def _fallback_models(container: str) -> tuple[set[str], str]: + """(the configured fallback model names, the primary's name) from the LIVE config. + + Read fresh every run, never hardcoded: the whole point of this check is to notice a + lane change, so it must not carry a stale idea of which lane is which. + """ + cfg = operator_api_get(container, "/api/config").get("config", {}) + fallbacks = {str(m).strip() for m in (cfg.get("routing", {}).get("fallback_models") or []) if str(m).strip()} + primary = str(cfg.get("model", {}).get("name") or "") + if primary in fallbacks: + # A fallback identical to the primary is a no-op lane, and counting it would + # report every ordinary call as a degrade. Refuse rather than cry wolf. + raise RuntimeError(f"fallback_models contains the primary model {primary!r} — nothing to distinguish") + return fallbacks, primary + + +def _container_ip(container: str) -> str: + """Vera's current address on the gateway's network. + + Resolved fresh every run on purpose: a watchtower roll gives her a new IP, and a + hardcoded one would silently stop matching — the counter would flatline and the + check would report a healthy lane forever. That is the exact failure class this + script exists to catch, so it must not reproduce it. + """ + out = subprocess.run( + ["docker", "inspect", container, "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"], + capture_output=True, + text=True, + timeout=30, + ) + if out.returncode != 0: + raise RuntimeError(f"docker inspect {container} failed: {out.stderr.strip()[:200]}") + ips = out.stdout.split() + if not ips: + raise RuntimeError(f"{container} has no container IP (is it running?)") + return ips[0] + + +def _scrape(url: str) -> str: + try: + with urllib.request.urlopen(url, timeout=20) as resp: + return resp.read().decode("utf-8", "replace") + except (urllib.error.URLError, OSError) as exc: + raise RuntimeError(f"cannot scrape {url}: {exc}") from exc + + +def fallback_requests( + metrics_text: str, agent_ip: str, fallback_models: set[str] | None = None +) -> tuple[float, dict[str, float]]: + """Gateway chat requests from ``agent_ip`` that the AGENT made to a FALLBACK model. + + ``fallback_models`` is the live `routing.fallback_models`; None counts every model + (the old native-primary shape, kept for the case where nothing is configured). + + Pure over the scrape text so the attribution rule is unit-testable without a live + gateway — the rule is the load-bearing part, not the HTTP. + """ + total = 0.0 + by_model: dict[str, float] = {} + for line in metrics_text.splitlines(): + match = _SAMPLE.match(line.strip()) + if not match: + continue + labels = {k: v for k, v in _LABEL.findall(match.group("labels"))} + if labels.get("client_ip") != agent_ip: + continue + if not labels.get("user_agent", "").startswith(AGENT_UA_PREFIX): + continue + if labels.get("route") == EMBEDDING_ROUTE: + continue + if fallback_models is not None and labels.get("requested_model") not in fallback_models: + continue # the primary lane doing its job + value = float(match.group("value")) + total += value + model = labels.get("requested_model", "?") + by_model[model] = by_model.get(model, 0.0) + value + return total, by_model + + +def _load_state(path: Path) -> dict: + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + return {} + + +def _save_state(path: Path, state: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, indent=2, sort_keys=True)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--container", default="vera") + ap.add_argument("--metrics-url", default=DEFAULT_METRICS) + ap.add_argument("--state", type=Path, default=DEFAULT_STATE) + ap.add_argument("--cooldown-min", type=float, default=DEFAULT_COOLDOWN_MIN) + ap.add_argument("--no-save", action="store_true", help="do not update the stored baseline (test runs)") + args = ap.parse_args() + + try: + agent_ip = _container_ip(args.container) + fallbacks, primary = _fallback_models(args.container) + total, by_model = fallback_requests(_scrape(args.metrics_url), agent_ip, fallbacks) + except Exception as exc: # noqa: BLE001 — every failure here is operational, exit 2 + print(f"UNREACHABLE: {exc}") + return 2 + + state = _load_state(args.state) + previous = state.get("total") + now = time.time() + last_alert = state.get("last_alert_ts", 0.0) + split = ", ".join(f"{m}={int(v)}" for m, v in sorted(by_model.items())) or "(none)" + + new_state = dict(state) + new_state["total"] = total + new_state["checked_at"] = now + new_state["agent_ip"] = agent_ip + new_state["by_model"] = by_model + + verdict = 0 + if previous is None: + print(f"BASELINE: {args.container} @ {agent_ip} · primary={primary} fallback={sorted(fallbacks)} — " + f"{int(total)} fallback requests so far [{split}]") + print("First run: recorded. Growth is what alarms, so this run cannot.") + elif total < previous: + # The gateway restarted and its counters reset. Re-baseline rather than reading + # the negative delta as "healthy" — and say so, because a silent re-baseline + # across a restart could swallow a real burst. + print(f"COUNTER RESET: gateway counters went {int(previous)} → {int(total)} (restart). Re-baselined.") + else: + delta = total - previous + if delta <= 0: + print(f"OK: no fallback traffic since the last run ({int(total)} lifetime) [{split}]") + else: + cooling = (now - last_alert) < args.cooldown_min * 60 + head = ( + f"{int(delta)} FALLBACK requests since the last run " + f"({int(total)} lifetime) [{split}]" + ) + if cooling: + quiet_for = int((args.cooldown_min * 60 - (now - last_alert)) / 60) + print(f"DEGRADED (cooldown, quiet ~{quiet_for}m more): {head}") + else: + print(f"FALLBACK: {head}") + print( + f"Vera answered {int(delta)} model calls from her fallback lane, not " + f"{primary}. Check the primary lane: " + "`curl -X POST localhost:7870/api/config/test-model` in the container " + "(401/403 = re-auth needed, 429 = rate-limited, it will pass)." + ) + new_state["last_alert_ts"] = now + verdict = 1 + + if not args.no_save: + try: + _save_state(args.state, new_state) + except OSError as exc: + # Persisting the baseline is bookkeeping; the verdict is the product. A + # full disk must never turn a REAL fallback into an "unreachable" alarm — + # that would relabel a silent degrade as an outage and send the operator + # looking in the wrong place. So a save failure downgrades to exit 2 only + # when there was nothing else to report; a verdict of 1 survives it and + # says so. (Cost of not saving: the next run re-alarms off a stale + # baseline. A duplicate alert is strictly better than a missed one.) + print(f"WARNING: could not persist the baseline to {args.state}: {exc}") + if verdict == 0: + print("No fallback to report, but the next run cannot alarm on growth — treating as operational.") + return 2 + print("Keeping the FALLBACK verdict; the next run may re-alarm from a stale baseline.") + return verdict + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_oauth_health.py b/scripts/check_oauth_health.py new file mode 100755 index 0000000..fe74eb7 --- /dev/null +++ b/scripts/check_oauth_health.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Health check: is Vera's Claude subscription credential still one she can use? + +The companion to check_model_fallback.py, and deliberately a different question. That +script asks "did she already degrade?" (observed, after the fact). This one asks "is the +thing that would make her degrade still sound?" (the cause, before the fact). Both are +needed: the credential can be perfect and the lane still fall back on a 529, and the +credential can be dead for hours on an idle agent with no traffic to reveal it. + +WHY NOT JUST CALL /api/config/test-model. Because it lies in the direction that matters. +It streams a real 1-token turn through the subscription, which sounds like the perfect +liveness probe — but measured on 2026-08-21 it returned `429 rate_limit_error` on four +consecutive attempts WHILE a real A2A turn on the same credential completed fine and +telemetry recorded `model=claude-sonnet-5`. Wiring the alert to that probe would have +paged #alerts immediately and permanently, for a lane that was working. A monitor whose +false-positive rate is 100% on day one is worse than no monitor. So this check reads +STATE, not liveness, and leaves "did it actually degrade" to the fallback detector, +which is grounded in traffic that really happened. + +What it checks, all from `/api/config` + `/api/config/oauth-status` (core ≥0.137.1, +where protoAgent#2564 started publishing expiry and refreshability): + + * signed_in — the credential is gone or was disconnected. Every call raises; the + fallback lane carries 100% of her traffic. This is the "OAuth expired" case. + * refreshable — a credential that cannot refresh is a deadline, not a credential. + This is also the CLAUDE_CODE_OAUTH_TOKEN trap: that env path is never refreshed and + never inspectable, and reads `signed_in: true` right up until it 401s. + * provider/name coherence — protoAgent#2623 made model.name and model.provider ONE + decision; a native-OAuth provider with a namespaced name ("protolabs/smart") is + rejected by the native builder on every call. That config is silently fatal, and it + is exactly what a careless half-edit of the model config produces. + * expires_at — reported always, alarmed on only when the credential is NOT + refreshable. Refresh is ON USE, so a healthy busy agent legitimately sits near its + expiry all day; alarming on proximity alone would cry wolf every few hours. + +Run from the ava fleet cron: + + python3 scripts/check_oauth_health.py --container vera + +Exit 0 = credential sound; exit 1 = a real problem (prints which); exit 2 = could not +reach the agent (operational, NOT a verdict). +""" + +from __future__ import annotations + +import argparse +import sys +import time + +from vera_api import operator_api_get + +# Native-OAuth providers (ADR 0097). A gateway-backed agent has no credential of its own +# to check — this whole script is a no-op for one, and says so rather than passing mutely. +NATIVE_PROVIDERS = {"anthropic-oauth", "openai-codex"} + + +def evaluate(model_cfg: dict, providers: list[dict]) -> tuple[int, list[str]]: + """(exit_code, report lines). Pure over the two API bodies so the rules are testable.""" + lines: list[str] = [] + provider = (model_cfg.get("provider") or "").strip().lower() + name = (model_cfg.get("name") or "").strip() + + if provider not in NATIVE_PROVIDERS: + # No "OK: " here — main() prepends the verdict prefix from the exit code, and + # baking a second one in produced "OK: OK: …" for every gateway-backed agent. + return 0, [f"model.provider={provider!r} is not a native OAuth lane — no credential to check."] + + lines.append(f"lane: {provider} · {name}") + problems: list[str] = [] + + # #2623 — the two halves are one decision, and a mismatched pair fails every call. + if "/" in name: + problems.append( + f"model.name={name!r} is a gateway alias but model.provider={provider!r} is native — " + "the native builder rejects any name containing '/', so EVERY call raises and the " + "fallback lane is carrying all traffic. Set model.name and model.provider together." + ) + + status = next((p for p in providers if (p.get("provider") or "").lower() == provider), None) + if status is None: + problems.append(f"/api/config/oauth-status reports nothing for {provider!r}") + return 1, lines + problems + + if not status.get("signed_in"): + problems.append( + f"NOT SIGNED IN ({status.get('detail') or 'no detail'}) — re-auth with " + f"POST /api/config/oauth/start {{\"provider\":\"{provider}\"}} and approve on any device." + ) + else: + source = status.get("source") or "?" + durability = status.get("durability") or "?" + lines.append(f"signed in · source={source} · durability={durability}") + if not status.get("refreshable"): + problems.append( + f"credential is NOT refreshable (source={source}) — it is a deadline, not a " + "credential. If this is CLAUDE_CODE_OAUTH_TOKEN, drop the env var and sign in " + "through /api/config/oauth/start so protoAgent owns a refreshing copy." + ) + expires_at = status.get("expires_at") + if expires_at: + remaining_h = (float(expires_at) - time.time()) / 3600.0 + when = time.strftime("%Y-%m-%d %H:%M", time.localtime(float(expires_at))) + lines.append(f"access token expires {when} ({remaining_h:+.1f}h) — refreshed on use") + + return (1 if problems else 0), lines + problems + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--container", default="vera") + args = ap.parse_args() + + try: + model_cfg = operator_api_get(args.container, "/api/config").get("config", {}).get("model", {}) + providers = operator_api_get(args.container, "/api/config/oauth-status").get("providers", []) + except Exception as exc: # noqa: BLE001 — operational, exit 2 + print(f"UNREACHABLE: {exc}") + return 2 + + code, lines = evaluate(model_cfg, providers) + print(("FAIL: " if code else "OK: ") + lines[0]) + for line in lines[1:]: + print(f" {line}") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_review_health.py b/scripts/check_review_health.py index 32cecf8..1f35e2f 100755 --- a/scripts/check_review_health.py +++ b/scripts/check_review_health.py @@ -36,10 +36,11 @@ import argparse import json -import subprocess import sys from pathlib import Path +from vera_api import operator_api_get + # New PRs that exhausted and have no verdict SINCE THE LAST RUN. Not zero because the # backfill sweep recovers some on a delay — protoAgent#2546 sat unreviewed for ~35min # before a later pass posted — so a run landing inside that window sees a transient +1. @@ -55,17 +56,6 @@ DEFAULT_STATE = Path.home() / ".cache" / "vera-review-health.json" -def _api(container: str, path: str) -> dict: - """Read a token-gated operator-API endpoint from inside the container.""" - cmd = f'curl -s -m 25 -H "Authorization: Bearer $A2A_AUTH_TOKEN" localhost:7870{path}' - out = subprocess.run( - ["docker", "exec", container, "sh", "-c", cmd], capture_output=True, text=True, timeout=60 - ) - if out.returncode != 0: - raise RuntimeError(f"docker exec failed for {path}: {out.stderr.strip()[:200]}") - return json.loads(out.stdout) - - def _load_state(path: Path) -> dict: """Previous run's counters. A missing or corrupt file is not an error — it means 'no baseline yet', which suppresses the growth checks rather than failing the run.""" @@ -110,8 +100,8 @@ def main() -> int: args = ap.parse_args() try: - report = _api(args.container, "/api/plugins/pr-reviewer/eval") - inbox = _api(args.container, "/api/inbox") + report = operator_api_get(args.container, "/api/plugins/pr-reviewer/eval") + inbox = operator_api_get(args.container, "/api/inbox") except Exception as exc: # noqa: BLE001 — any failure here is operational, not a verdict print(f"UNREACHABLE: {exc}", file=sys.stderr) return 2 diff --git a/scripts/vera-watchdog.sh b/scripts/vera-watchdog.sh new file mode 100755 index 0000000..06125ad --- /dev/null +++ b/scripts/vera-watchdog.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Runner + alerter for Vera's four watchdogs — health, drift, fallback, oauth +# (qaEngineer#37 shipped the first two; the model-lane pair arrived with the move to a +# native Claude subscription). +# +# WHY THIS FILE EXISTS AT ALL, AND WHY IT LIVES HERE: +# +# The first two shipped in qaEngineer/scripts/ with docstrings saying "run from the ava +# fleet cron" — and then nothing ever ran them. The review-health alarm was written +# precisely because 24 PRs merged with no review and the only record was an inbox +# nobody read; leaving that alarm unscheduled reproduced the same failure one level up. +# +# It runs the INSTALLED copies in ~/.local/bin, NOT qaEngineer/scripts/*.py: the repo +# working tree is also the deploy source, so a branch switch deletes/reverts the +# in-repo script and the guard stops silently. That is not hypothetical — the same +# trap cost config-drift.sh ~5h of silent failure on 2026-08-10, and this repo sat on +# a feature branch for part of 2026-08-17 during the 0.137.1 bump. A watchdog must not +# live inside the thing it watches. Refresh these copies when the repo version changes: +# +# install -m 644 ~/dev/qaEngineer/scripts/vera_api.py ~/.local/bin/vera_api.py +# install -m 755 ~/dev/qaEngineer/scripts/check_review_health.py ~/.local/bin/vera-review-health.py +# install -m 755 ~/dev/qaEngineer/scripts/check_card_drift.py ~/.local/bin/vera-card-drift.py +# install -m 755 ~/dev/qaEngineer/scripts/check_model_fallback.py ~/.local/bin/vera-model-fallback.py +# install -m 755 ~/dev/qaEngineer/scripts/check_oauth_health.py ~/.local/bin/vera-oauth-health.py +# install -m 755 ~/dev/qaEngineer/scripts/vera-watchdog.sh ~/.local/bin/vera-watchdog.sh +# +# THIS FILE is the canonical copy (it was unversioned until 2026-08-21 — the alerting +# half of the guard living nowhere but one box's ~/.local/bin is its own quiet risk). +# The installed copy is still what cron runs, for the branch-trap reason above. +# +# A failing check must be LOUD. Exit 1 (a real verdict) and exit 2 (couldn't reach the +# agent) are deliberately different alerts — the scripts draw that line on purpose, and +# collapsing it would let an outage read as a clean gate. +# +# Usage: vera-watchdog.sh health|drift|fallback|oauth [extra args passed to the check] +# Exit: passes the underlying check's exit code through (0 ok, 1 verdict, 2 unreachable) + +set -uo pipefail + +REPO="${VERA_WATCHDOG_REPO:-$HOME/dev/qaEngineer}" +BIN="$HOME/.local/bin" +MODE="${1:-}"; shift || true # remaining args pass through to the underlying check + +# The alert path must not depend on a credential that can silently expire. On +# 2026-08-21 ava's `infisical login` session had lapsed and two health runs fell through +# to the no-secrets branch: the checks ran, passed, and would have alerted to NOTHING if +# they had failed — the guard disarmed with no signal, which is the exact failure class +# these watchdogs exist to catch, one level up. So: env first (infisical run, when a +# session is good), then a local 0600 file that no session can invalidate. +# +# install -d -m 700 ~/.config/vera +# printf 'DISCORD_WEBHOOK_ALERTS=%s\n' "$(infisical secrets get DISCORD_WEBHOOK_ALERTS --plain ...)" \ +# > ~/.config/vera/alerts.env && chmod 600 ~/.config/vera/alerts.env +# +# Regenerate it if the webhook is ever rotated — this is a cached copy, not the source. +ALERT_ENV="${VERA_ALERT_ENV:-$HOME/.config/vera/alerts.env}" + +alert() { # alert <body> + local hook="${DISCORD_WEBHOOK_ALERTS:-}" + if [ -z "$hook" ] && [ -r "$ALERT_ENV" ]; then + # shellcheck disable=SC1090 + . "$ALERT_ENV" + hook="${DISCORD_WEBHOOK_ALERTS:-}" + fi + if [ -z "$hook" ]; then + echo "vera-watchdog: no DISCORD_WEBHOOK_ALERTS (env or $ALERT_ENV) — NOT alerted" >&2 + return + fi + local body + body="$(printf '**%s** on %s\n```\n%s\n```' "$1" "$(hostname)" "$2")" + python3 -c 'import json,sys;print(json.dumps({"content":sys.argv[1][:1900]}))' "$body" \ + | curl -sf -X POST -H 'Content-Type: application/json' -d @- "$hook" >/dev/null \ + && echo "vera-watchdog: alerted Discord" >&2 \ + || echo "vera-watchdog: Discord post FAILED" >&2 +} + +case "$MODE" in + health) + out="$("$BIN/vera-review-health.py" --container vera "$@" 2>&1)"; rc=$? + ;; + drift) + # The seed is the reference this check compares the live card against, so it must + # come from the repo — but reading the WORKING TREE would reintroduce the branch + # trap through the back door (a feature branch's seed is not the deployed one). + # Read it out of the committed main ref instead: always the deployed truth, and + # immune to whatever the tree is currently checked out at. + seed="$(mktemp)"; trap 'rm -f "$seed"' EXIT + if ! git -C "$REPO" show main:deploy/vera.langgraph-config.yaml >"$seed" 2>/dev/null; then + out="cannot read deploy/vera.langgraph-config.yaml from main in $REPO"; rc=2 + else + out="$("$BIN/vera-card-drift.py" --seed "$seed" "$@" 2>&1)"; rc=$? + fi + ;; + fallback) + # Did Vera silently answer from her FALLBACK model? The check reads + # `routing.fallback_models` from her LIVE config and counts only gateway traffic + # requesting those, so it stays correct whichever lane is primary — an earlier + # version hardcoded "any protoAgent-UA gateway traffic is a fallback", which was + # true only while the primary was a native-OAuth subscription bypassing the gateway, + # and inverted the moment she moved back to a gateway primary. + # + # Why infer at all: on cores before 0.145.0 protoAgent emitted NOTHING on failover + # (langchain's ModelFallbackMiddleware swallows the primary's exception without so + # much as a log line; filed as protoAgent#2956, fixed there). Once the RUNNING + # instance is on 0.145.0+ and its `model.fallback` event is seen firing, retire this + # inference and subscribe to the event instead. + out="$("$BIN/vera-model-fallback.py" --container vera "$@" 2>&1)"; rc=$? + ;; + oauth) + # Is the subscription credential itself still sound (signed in, refreshable, and + # coherent with model.name)? The CAUSE side of the same failure — an idle agent can + # outlive its refresh token with no traffic to reveal it. + out="$("$BIN/vera-oauth-health.py" --container vera "$@" 2>&1)"; rc=$? + ;; + *) + echo "usage: $(basename "$0") health|drift|fallback|oauth" >&2; exit 64 ;; +esac + +echo "$out" +case "$rc" in + 0) ;; + 1) alert "vera $MODE check FAILED" "$out" ;; + *) alert "vera $MODE check UNREACHABLE (exit $rc)" "$out" ;; +esac +exit "$rc" diff --git a/scripts/vera_api.py b/scripts/vera_api.py new file mode 100644 index 0000000..121ce74 --- /dev/null +++ b/scripts/vera_api.py @@ -0,0 +1,50 @@ +"""The one way these checks talk to Vera's operator API. + +Extracted because `check_oauth_health.py` shipped a byte-for-byte copy of +`check_review_health.py`'s helper — caught by Vera's own panel reviewing the PR that +added it (qaEngineer#45, minor/conventions, confirmed). Two copies of the auth, timeout +and error-handling for the same endpoint is two places to fix when the port moves or the +bearer changes, and the second copy is the one that gets missed. + +WHY IT LIVES BESIDE THE SCRIPTS AND NOT IN A PACKAGE: cron runs INSTALLED copies out of +`~/.local/bin` (the repo tree is also the deploy source, so a branch switch would +silently disarm a guard living inside it). Python puts a script's own directory on +`sys.path[0]`, so a flat module installed alongside imports cleanly — but it MUST be +installed alongside, or every check dies on ImportError: + + install -m 644 ~/dev/qaEngineer/scripts/vera_api.py ~/.local/bin/vera_api.py + +The operator API is container-local and token-gated, so the call shape is fixed: exec +into the container and read the bearer from its own environment. There is no host-side +credential to leak or expire — which is the point. +""" + +from __future__ import annotations + +import json +import subprocess + +DEFAULT_PORT = 7870 +CURL_TIMEOUT_S = 25 +EXEC_TIMEOUT_S = 60 + + +def operator_api_get(container: str, path: str, *, port: int = DEFAULT_PORT) -> dict: + """GET a token-gated operator-API endpoint from inside ``container``. + + Raises ``RuntimeError`` when the container cannot be reached — callers turn that + into exit 2 (operational), never into a health verdict. + """ + cmd = ( + f'curl -s -m {CURL_TIMEOUT_S} -H "Authorization: Bearer $A2A_AUTH_TOKEN" ' + f"localhost:{port}{path}" + ) + out = subprocess.run( + ["docker", "exec", container, "sh", "-c", cmd], + capture_output=True, + text=True, + timeout=EXEC_TIMEOUT_S, + ) + if out.returncode != 0: + raise RuntimeError(f"docker exec failed for {path}: {out.stderr.strip()[:200]}") + return json.loads(out.stdout) diff --git a/tests/test_watchdog_checks.py b/tests/test_watchdog_checks.py new file mode 100644 index 0000000..395cded --- /dev/null +++ b/tests/test_watchdog_checks.py @@ -0,0 +1,205 @@ +"""Tests for the two watchdog checks whose logic decides whether #alerts gets woken. + +Vera's panel flagged their absence on qaEngineer#45 (minor/tests, confirmed): both +scripts document their core functions as "deliberately pure so the rule is testable", +and then shipped no tests. The rule IS the load-bearing part — an attribution bug in +`fallback_requests` means either a silent degrade nobody hears about or a pager that +cries wolf, and neither shows up in a smoke run against a healthy container. + +Stdlib unittest on purpose: CI here is python3 with PyYAML and nothing else, and a +watchdog's test suite earning a dependency install is the wrong trade. + + python3 -m unittest discover tests -v +""" + +from __future__ import annotations + +import sys +import time +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +from check_model_fallback import fallback_requests # noqa: E402 +from check_oauth_health import evaluate # noqa: E402 + +VERA_IP = "10.0.14.6" +AGENT_UA = "protoAgent/0.1 (+https://github.com/protoLabsAI/protoAgent)" + + +def sample(**labels) -> str: + """One `litellm_proxy_total_requests_metric_total` line, in the gateway's real shape.""" + value = labels.pop("value", 1.0) + base = { + "api_key_alias": "studio-gw-75bce515", + "client_ip": VERA_IP, + "requested_model": "protolabs/smart", + "route": "/v1/chat/completions", + "status_code": "200", + "user_agent": AGENT_UA, + } + base.update(labels) + rendered = ",".join(f'{k}="{v}"' for k, v in base.items()) + return f"litellm_proxy_total_requests_metric_total{{{rendered}}} {value}" + + +class FallbackAttribution(unittest.TestCase): + """The rule: a protoAgent-UA chat completion from Vera's IP IS a fallback.""" + + def test_counts_agent_traffic_from_vera(self): + total, by_model = fallback_requests(sample(value=7.0), VERA_IP) + self.assertEqual(total, 7.0) + self.assertEqual(by_model, {"protolabs/smart": 7.0}) + + def test_ignores_clawpatch(self): + # clawpatch shares the gateway key, the container AND the model alias — the + # user_agent is the only thing separating it from a real fallback. Miscounting + # it would report a permanent degrade on a perfectly healthy lane. + total, _ = fallback_requests(sample(user_agent="node", value=94.0), VERA_IP) + self.assertEqual(total, 0.0) + + def test_ignores_other_containers(self): + # Fleet peers share the gateway and the key; only the IP tells them apart. + total, _ = fallback_requests(sample(client_ip="10.0.14.14", value=171.0), VERA_IP) + self.assertEqual(total, 0.0) + + def test_ignores_embeddings(self): + # Same UA, same container, not a model-lane fallback. + line = sample(route="/v1/embeddings", requested_model="qwen3-embedding", value=45.0) + total, _ = fallback_requests(line, VERA_IP) + self.assertEqual(total, 0.0) + + def test_sums_across_models_and_skips_noise(self): + text = "\n".join( + [ + "# HELP litellm_proxy_total_requests_metric_total noise", + sample(value=3.0), + sample(requested_model="protolabs/cloud", value=2.0), + sample(user_agent="node", value=99.0), + "litellm_something_else_total{foo=\"bar\"} 5.0", + ] + ) + total, by_model = fallback_requests(text, VERA_IP) + self.assertEqual(total, 5.0) + self.assertEqual(by_model, {"protolabs/smart": 3.0, "protolabs/cloud": 2.0}) + + def test_empty_scrape_is_zero_not_an_error(self): + # A gateway that just restarted serves no samples yet; that is "nothing to + # report", not an alarm. + self.assertEqual(fallback_requests("", VERA_IP), (0.0, {})) + + +def oauth_status(**over) -> list[dict]: + base = { + "provider": "anthropic-oauth", + "signed_in": True, + "refreshable": True, + "source": "instance_store", + "expires_at": time.time() + 3600, + } + base.update(over) + return [base] + + +class FallbackModelFilter(unittest.TestCase): + """The lane-shape-independent rule: only traffic to a CONFIGURED fallback counts. + + Added when Vera moved from a native-OAuth primary back to a gateway primary + (2026-08-23). Under the old "any gateway traffic is a fallback" rule that switch + silently inverted the check — every ordinary review would have read as a degrade and + paged #alerts every 15 minutes. These pin the rule that makes it survive a lane + change without anyone editing the script. + """ + + def test_primary_traffic_is_not_a_fallback(self): + text = sample(requested_model="protolabs/smart", value=40.0) + total, _ = fallback_requests(text, VERA_IP, {"protolabs/cloud"}) + self.assertEqual(total, 0.0, "the primary lane doing its job is not a degrade") + + def test_fallback_traffic_counts(self): + text = sample(requested_model="protolabs/cloud", value=7.0) + total, by_model = fallback_requests(text, VERA_IP, {"protolabs/cloud"}) + self.assertEqual(total, 7.0) + self.assertEqual(by_model, {"protolabs/cloud": 7.0}) + + def test_mixed_traffic_counts_only_the_fallback(self): + text = "\n".join([ + sample(requested_model="protolabs/smart", value=100.0), # primary + sample(requested_model="protolabs/cloud", value=3.0), # fallback + sample(requested_model="protolabs/cloud", user_agent="node", value=50.0), # clawpatch + ]) + total, by_model = fallback_requests(text, VERA_IP, {"protolabs/cloud"}) + self.assertEqual(total, 3.0) + self.assertEqual(by_model, {"protolabs/cloud": 3.0}) + + def test_none_counts_every_model(self): + # The native-primary shape: nothing configured to filter on, so any agent + # gateway call is a fallback by construction. + text = sample(requested_model="protolabs/smart", value=9.0) + total, _ = fallback_requests(text, VERA_IP, None) + self.assertEqual(total, 9.0) + + +class OAuthHealth(unittest.TestCase): + NATIVE = {"provider": "anthropic-oauth", "name": "claude-sonnet-5"} + + def test_healthy_lane_passes(self): + code, lines = evaluate(self.NATIVE, oauth_status()) + self.assertEqual(code, 0) + self.assertTrue(any("claude-sonnet-5" in line for line in lines)) + + def test_signed_out_fails(self): + code, lines = evaluate(self.NATIVE, oauth_status(signed_in=False, detail="disconnected")) + self.assertEqual(code, 1) + self.assertIn("NOT SIGNED IN", lines[-1]) + + def test_unrefreshable_credential_fails(self): + # The CLAUDE_CODE_OAUTH_TOKEN trap: reads signed_in right up until it 401s. + code, lines = evaluate(self.NATIVE, oauth_status(refreshable=False, source="env")) + self.assertEqual(code, 1) + self.assertIn("not refreshable", lines[-1].lower()) + + def test_incoherent_provider_and_name_fails(self): + # protoAgent#2623: one decision, two fields. A gateway alias under a native + # provider is rejected on every call — silently fatal, and exactly what a + # half-edited model config produces. + code, lines = evaluate({"provider": "anthropic-oauth", "name": "protolabs/smart"}, oauth_status()) + self.assertEqual(code, 1) + self.assertIn("gateway alias", lines[-1]) + + def test_missing_status_entry_fails(self): + code, lines = evaluate(self.NATIVE, []) + self.assertEqual(code, 1) + self.assertIn("reports nothing", lines[-1]) + + def test_gateway_backed_agent_is_a_noop(self): + # No subscription, no credential to check — must pass, and say why. + code, lines = evaluate({"provider": "openai", "name": "protolabs/cloud"}, []) + self.assertEqual(code, 0) + self.assertIn("not a native OAuth lane", lines[0]) + + def test_messages_carry_no_verdict_prefix(self): + # main() prepends "OK: "/"FAIL: " from the exit code. A branch that bakes its + # own prefix in prints "OK: OK: …" — caught in review, and invisible to the + # other tests here because they all call evaluate() directly and never main(). + for cfg, status in ( + ({"provider": "openai", "name": "protolabs/cloud"}, []), + (self.NATIVE, oauth_status()), + (self.NATIVE, oauth_status(signed_in=False)), + ): + _, lines = evaluate(cfg, status) + self.assertFalse( + lines[0].startswith(("OK:", "FAIL:")), + f"evaluate() must not prefix its own verdict: {lines[0]!r}", + ) + + def test_expired_but_refreshable_is_not_an_alarm(self): + # Refresh is ON USE, so a busy agent legitimately sits at or past its access + # token's expiry. Alarming on proximity would cry wolf every few hours. + code, _ = evaluate(self.NATIVE, oauth_status(expires_at=time.time() - 60)) + self.assertEqual(code, 0) + + +if __name__ == "__main__": + unittest.main()