feature(client): summary-first SCT events in CLI and web UI

argus/backend/plugins/sct/testrun.py

@@ -86,6 +86,7 @@ class SCTEvent(Model):
     event_type = columns.Text()
     message = columns.Text(required=True)
     duplicate_id = columns.UUID()
+    summary = columns.Text()  # LLM-generated summary; null until/unless summarized (see argusAI)
 
     # DB Log columns
     node = columns.Text()

argus/backend/tests/sct_events/test_sct_events.py

@@ -6,8 +6,9 @@
 
 from flask.testing import FlaskClient
 
+from argus.backend.db import ScyllaCluster
 from argus.backend.models.web import ArgusRelease, ArgusGroup, ArgusTest
-from argus.backend.plugins.sct.testrun import SCTEventSeverity, SCTTestRun
+from argus.backend.plugins.sct.testrun import SCTEvent, SCTEventSeverity, SCTTestRun
 from argus.backend.service.client_service import ClientService
 from argus.backend.plugins.sct.service import SCTService
 from argus.backend.service.testrun import TestRunService
@@ -43,6 +44,52 @@ def test_submit_event(client_service: ClientService, sct_service: SCTService, te
     assert len(all_events) == 1, "Event not found"
 
 
+def test_get_events_includes_summary_field(client_service: ClientService, sct_service: SCTService, testrun_service: TestRunService, fake_test: ArgusTest):
+    run_type, run_req = get_fake_test_run(fake_test)
+    client_service.submit_run(run_type, asdict(run_req))
+    run: SCTTestRun = testrun_service.get_run(run_type, run_req.run_id)
+
+    message = "database error: sstable corruption at offset 42"
+    event_data: RawEventPayload = {
+        "duration": 30.0,
+        "event_type": "end",
+        "known_issue": None,
+        "message": message,
+        "nemesis_name": None,
+        "nemesis_status": None,
+        "node": None,
+        "received_timestamp": None,
+        "run_id": run.id,
+        "severity": SCTEventSeverity.ERROR.value,
+        "target_node": None,
+        "ts": datetime.now(tz=UTC).timestamp()
+    }
+    _ = sct_service.submit_event(str(run.id), event_data)
+
+    # A freshly submitted event has no summary yet: the field is present and null,
+    # and consumers fall back to the original message (goal 4).
+    before = run.get_events_limited(run.id, severities=[SCTEventSeverity.ERROR])
+    assert len(before) == 1, "Event not found"
+    assert "summary" in before[0], "summary column must ride along in the get_events payload"
+    assert before[0]["summary"] is None, "un-summarized event must report summary=null"
+    assert before[0]["message"] == message
+
+    # Simulate the argusAI worker writing a summary back with its exact raw-CQL upsert.
+    evt = before[0]
+    cluster = ScyllaCluster.get()
+    stmt = cluster.prepare(
+        f"UPDATE {SCTEvent.table_name()} SET summary = ? WHERE run_id = ? AND severity = ? AND ts = ?"
+    )
+    summary = "ERROR on sstable corruption (offset 42)"
+    cluster.session.execute(stmt, (summary, evt["run_id"], evt["severity"], evt["ts"]))
+
+    # Serving now carries the summary, and the original message is intact (additive, goal 3).
+    after = run.get_events_limited(run.id, severities=[SCTEventSeverity.ERROR])
+    assert len(after) == 1
+    assert after[0]["summary"] == summary
+    assert after[0]["message"] == message, "original message must remain intact alongside the summary"
+
+
 def test_get_events_by_severity(client_service: ClientService, sct_service: SCTService, testrun_service: TestRunService, fake_test: ArgusTest):
     run_type, run_req = get_fake_test_run(fake_test)
     client_service.submit_run(run_type, asdict(run_req))

argusAI/eval/.gitignore

@@ -0,0 +1,5 @@
+# Local eval working files — never commit outputs, working configs, or anything key-adjacent
+out/
+out_*/
+config.yaml
+*.local.yaml

argusAI/eval/README.md

@@ -0,0 +1,103 @@
+# Event Summarization — evaluation harness
+
+Offline harness that answers the two questions PR #1016 asked before rollout:
+
+1. **Which model?** — the cheapest one whose summaries stay faithful and lose no
+   information a triager (or Zeus) needs.
+2. **Is the prompt any good?** — iterate it on real events until the output is right.
+
+It pulls real SCT `ERROR`/`CRITICAL` events from Argus (via the `argus` CLI),
+summarizes each with several candidate models under a configurable prompt, scores every
+summary against the **original** event with a frontier judge model, and renders a
+self-contained HTML report: a quality-vs-cost decision graph plus every summary shown
+side-by-side with its source so a human can verify the judge.
+
+Nothing is hard-coded — **model and API key are configurable** (mandatory for
+production), as is the prompt, the judge, and the pricing table.
+
+## Setup
+
+```bash
+pip install -e '.[ai-eval]'          # installs openai + PyYAML
+export OPENAI_API_KEY=sk-...          # key comes from the env, never the config file
+```
+
+The `argus` CLI must be authenticated (`argus auth`) — the harness shells out to it.
+
+## Run
+
+```bash
+# 1. edit argusAI/eval/config.example.yaml: set run_ids/test_id, models, prompts, pricing
+cp argusAI/eval/config.example.yaml argusAI/eval/config.yaml
+
+# 2. (optional) collect events first, without spending any API tokens, and inspect them
+python -m argusAI.eval --config argusAI/eval/config.yaml --fetch-only
+
+# 3. full sweep: summarize × judge × render report
+python -m argusAI.eval --config argusAI/eval/config.yaml --open
+
+# re-render the HTML from an existing results.json (no API calls)
+python -m argusAI.eval --config argusAI/eval/config.yaml --report-only --open
+```
+
+Outputs land in `output_dir` (default `argusAI/eval/out/`):
+
+| File | What |
+|------|------|
+| `events.json`  | the deduplicated events that were evaluated (reusable via `events_file`) |
+| `results.json` | every (event × model × prompt) cell: summary, tokens, cost, latency, judge score |
+| `report.html`  | the presentable graph + per-event side-by-side summaries |
+
+## Config
+
+See `config.example.yaml` — commented in full. The key knobs:
+
+- `models` — list of candidates; each `params` block is forwarded verbatim to the API
+  call (`reasoning_effort`, `temperature`, `max_completion_tokens`, …).
+- `prompts` — names from `prompts.py`; list several to A/B them on the same events.
+- `judge_model` / `judge_enabled` — the frontier scorer (Opus-class / GPT-5-class).
+- `pricing` — USD per 1M tokens; **the defaults are placeholders**, set real numbers or
+  the cost axis is meaningless (unknown models are flagged in the report, never silent).
+
+## Regression baseline
+
+`out/` is gitignored, so a run's evidence lives only on your laptop. To keep a durable
+reference that future prompt/model changes are checked against, freeze the current winner
+into the tracked `baseline/` dir:
+
+```bash
+# seed a baseline (from a fresh sweep, or from an existing results.json without spending tokens)
+python -m argusAI.eval --config ... --write-baseline argusAI/eval/baseline
+python -m argusAI.eval --config ... --report-only --write-baseline argusAI/eval/baseline
+
+# later: re-run on the *frozen* events and check for regressions (exit 1 on FAIL)
+python -m argusAI.eval --config ... --baseline argusAI/eval/baseline
+```
+
+`baseline/` holds two committed files: `events.json` (the frozen input set — `--baseline`
+forces the run onto it so inputs are identical) and `baseline.json` (per-`(model,prompt)`
+mean scores). Comparison is **aggregate + tolerance**, not exact-match, because the
+summarizer and judge are non-deterministic. A series **FAILs** only when faithfulness or
+coverage drops more than `--tolerance` points (default 3) — the two axes that mean
+information was dropped or invented. Conciseness, cost, and latency are shown as deltas but
+never fail the check; new hallucinations or dropped-critical items are flagged inline.
+
+Deliberately not wired into CI: each run spends real OpenAI tokens and the judge is
+non-deterministic, so it's a manual pre-change gate, not an automated one.
+
+## Iterating the prompt
+
+`prompts.py` holds `PRODUCTION` (the current winner) and `BASELINE` (the naive one-liner,
+kept as an A/B control). To try a new idea, **add a new named entry** rather than editing
+`PRODUCTION` in place, list both in `prompts:`, and the report compares them head-to-head on
+identical events — so a regression is visible, not lost. If the challenger wins, promote its
+text into `PRODUCTION` here **and** into `argusAI/utils/summarizer.py`'s `DEFAULT_PROMPT`
+(the two are kept byte-identical), which becomes the production `EVENT_SUMMARIZATION_PROMPT`
+default. Older experiments aren't carried in this file — git history and the saved `out/`
+reports are the record.
+
+## Relation to production
+
+`argusAI/utils/summarizer.py` (the `Summarizer` class) is shared: this harness and the
+production worker call it identically, so a summary you approve here is byte-for-byte what
+the worker will store for the same `(model, prompt)`.

argusAI/eval/__init__.py

@@ -0,0 +1,14 @@
+"""Offline evaluation harness for event summarization.
+
+Extracts real SCT ERROR/CRITICAL events (via the ``argus`` CLI), summarizes each
+with several candidate models under a configurable prompt, scores the summaries for
+information preservation with a frontier judge model, and renders a comparison graph
+plus a per-event side-by-side HTML report.
+
+This is the harness the design doc (docs/plans/event-summarization.md §7) requires to
+pick ``OPENAI_SUMMARY_MODEL`` and to validate the prompt before rollout. Run it as::
+
+    python -m argusAI.eval --config argusAI/eval/config.example.yaml
+
+See ``argusAI/eval/README.md``.
+"""

argusAI/eval/__main__.py

@@ -0,0 +1,130 @@
+"""CLI: run the evaluation sweep and render the report.
+
+    python -m argusAI.eval --config argusAI/eval/config.example.yaml
+
+Sub-modes:
+    --fetch-only     collect events into out/events.json and stop (no API calls)
+    --report-only    re-render the HTML from an existing out/results.json (no API calls)
+    --write-baseline DIR   freeze the current run's aggregate scores as a regression baseline
+    --baseline DIR         re-run on the frozen set and check for regressions (exit 1 on FAIL)
+
+Everything else (models, prompts, events, judge) comes from the YAML config; only the
+API key comes from the environment (OPENAI_API_KEY by default).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+import webbrowser
+from pathlib import Path
+
+from .baseline import BASELINE_FILE, EVENTS_FILE, compare, load_baseline, render_table, save_baseline
+from .config import EvalConfig
+from .events_source import EventFetcher, save_events
+from .harness import EvalHarness
+from .report import report_from_file, write_report
+
+_DEFAULT_TOLERANCE = 3.0
+
+
+def _write_baseline(results: dict, out_dir: Path, tolerance: float) -> None:
+    """Persist baseline.json plus the frozen events beside it, so a later --baseline run
+    scores the identical inputs."""
+    path = save_baseline(results, out_dir, tolerance)
+    (out_dir / EVENTS_FILE).write_text(json.dumps(results.get("events", []), indent=2), encoding="utf-8")
+    print(f"Baseline: {path} (+ {out_dir / EVENTS_FILE}, {results.get('num_events')} events)")
+
+
+def _run_baseline_check(cfg: EvalConfig, base_dir: Path, tolerance: float | None, report_path: Path) -> int:
+    """Re-run the sweep on the frozen event set and diff aggregate scores against the stored
+    baseline. Exit non-zero if a gated axis regressed beyond tolerance."""
+    base_path, events_path = base_dir / BASELINE_FILE, base_dir / EVENTS_FILE
+    if not base_path.exists() or not events_path.exists():
+        print(f"Baseline incomplete: need both {base_path} and {events_path}.", file=sys.stderr)
+        return 1
+    baseline = load_baseline(base_path)
+    tol = tolerance if tolerance is not None else baseline.get("tolerance", _DEFAULT_TOLERANCE)
+    cfg.events_file = events_path  # regression check must score the identical frozen inputs
+    results = EvalHarness(cfg).run()
+    write_report(results, report_path)
+    rows, ok = compare(results, baseline, tol)
+    print("\n" + render_table(rows, baseline, tol))
+    print(f"\nReport: {report_path}\n{'PASS — no regressions' if ok else 'FAIL — regressions above'}")
+    return 0 if ok else 1
+
+
+def _setup_logging(verbose: bool) -> None:
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("openai").setLevel(logging.WARNING)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m argusAI.eval", description=__doc__)
+    parser.add_argument("--config", required=True, help="path to eval YAML config")
+    parser.add_argument("--fetch-only", action="store_true", help="only collect events, no API calls")
+    parser.add_argument("--report-only", action="store_true", help="only re-render report from results.json")
+    parser.add_argument("--write-baseline", metavar="DIR", help="freeze this run's scores as a regression baseline")
+    parser.add_argument("--baseline", metavar="DIR", help="re-run on the frozen set and check for regressions")
+    parser.add_argument(
+        "--tolerance", type=float, default=None,
+        help=f"allowed faithfulness/coverage drop in points (default: baseline's, else {_DEFAULT_TOLERANCE})",
+    )
+    parser.add_argument("--open", action="store_true", help="open the HTML report in a browser when done")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    args = parser.parse_args(argv)
+    _setup_logging(args.verbose)
+
+    cfg = EvalConfig.from_yaml(args.config)
+    out_dir = cfg.output_dir
+    results_path = out_dir / "results.json"
+    report_path = out_dir / "report.html"
+
+    if args.report_only:
+        if not results_path.exists():
+            print(f"No results at {results_path}; run the sweep first.", file=sys.stderr)
+            return 1
+        report_from_file(results_path, report_path)
+        print(f"Report: {report_path}")
+        if args.write_baseline:  # seed a baseline from an existing run, spending no tokens
+            results = json.loads(results_path.read_text(encoding="utf-8"))
+            _write_baseline(results, Path(args.write_baseline), args.tolerance or _DEFAULT_TOLERANCE)
+        if args.open:
+            webbrowser.open(report_path.resolve().as_uri())
+        return 0
+
+    if args.baseline:
+        return _run_baseline_check(cfg, Path(args.baseline), args.tolerance, report_path)
+
+    if args.fetch_only:
+        fetcher = EventFetcher(argus_cli=cfg.argus_cli, no_cache=cfg.no_cache)
+        events = fetcher.collect(
+            run_ids=cfg.run_ids,
+            test_id=cfg.test_id,
+            test_run_limit=cfg.test_run_limit,
+            max_events=cfg.max_events,
+        )
+        save_events(events, out_dir / "events.json")
+        print(f"Collected {len(events)} events -> {out_dir / 'events.json'}")
+        return 0 if events else 2
+
+    harness = EvalHarness(cfg)  # validates the API key up front
+    results = harness.run()
+    write_report(results, report_path)
+    print(f"\nResults: {results_path}\nReport:  {report_path}")
+    if args.write_baseline:
+        _write_baseline(results, Path(args.write_baseline), args.tolerance or _DEFAULT_TOLERANCE)
+    if args.open:
+        webbrowser.open(report_path.resolve().as_uri())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())