From 9b80d03ee1f1c78481d393c9ea27ee972901acba Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 01:35:41 +0000 Subject: [PATCH 1/2] CI recurrence study: Tier 0 fingerprinting, collector, and harness Schema, Tier 0 normalizer, and Tier 1 worker prompt for the CI failure recurrence study, plus the collector and the model-free analysis path. Tier 0 is deterministic regex normalization + sha1 fingerprinting: free, reproducible, and hashable, which an LLM label is not. Tiers 1/2 are only ever asked about what Tier 0 leaves as a singleton. Verified with known-answer fixtures (20/20 in test_pipeline.py): the normalizer collapses one bug across differing paths, line numbers, and float values while keeping distinct test files and distinct exit codes apart; the chronological walk reproduces a known rate; both flake shapes are detected; the ingest path handles annotation evidence with a step-name fallback. Two bugs found and fixed by that testing, both of which would have corrupted the headline rate: - a port rule fired on four-digit line numbers ahead of the line/col rule, shattering collapse across every large file - the absolute-path rule mangled repo-relative annotation paths, which are identical on every runner and therefore signal No recurrence rate is computed yet. This session cannot reach the Actions API for either target repo (403 per-repository authorization; the one reachable repo has zero CI failures), so collection stopped rather than building the Tier 1 loop on an unmeasured Tier 0. See findings.md. Stdlib only: no LangChain, DSPy, Celery, or Redis. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DJvuGv9fvnpNGXmJFp2HzQ --- studies/ci-recurrence/.gitignore | 4 + studies/ci-recurrence/README.md | 103 ++++++++++ studies/ci-recurrence/analyze.py | 193 ++++++++++++++++++ studies/ci-recurrence/db.py | 32 +++ studies/ci-recurrence/dispatch.py | 185 ++++++++++++++++++ studies/ci-recurrence/fetch.py | 258 +++++++++++++++++++++++++ studies/ci-recurrence/findings.md | 91 +++++++++ studies/ci-recurrence/normalize.py | 166 ++++++++++++++++ studies/ci-recurrence/prompts.py | 86 +++++++++ studies/ci-recurrence/pyproject.toml | 12 ++ studies/ci-recurrence/schema.sql | 138 +++++++++++++ studies/ci-recurrence/test_pipeline.py | 139 +++++++++++++ 12 files changed, 1407 insertions(+) create mode 100644 studies/ci-recurrence/.gitignore create mode 100644 studies/ci-recurrence/README.md create mode 100644 studies/ci-recurrence/analyze.py create mode 100644 studies/ci-recurrence/db.py create mode 100644 studies/ci-recurrence/dispatch.py create mode 100644 studies/ci-recurrence/fetch.py create mode 100644 studies/ci-recurrence/findings.md create mode 100644 studies/ci-recurrence/normalize.py create mode 100644 studies/ci-recurrence/prompts.py create mode 100644 studies/ci-recurrence/pyproject.toml create mode 100644 studies/ci-recurrence/schema.sql create mode 100644 studies/ci-recurrence/test_pipeline.py diff --git a/studies/ci-recurrence/.gitignore b/studies/ci-recurrence/.gitignore new file mode 100644 index 0000000..63646c9 --- /dev/null +++ b/studies/ci-recurrence/.gitignore @@ -0,0 +1,4 @@ +cache/ +out/ +__pycache__/ +*.pyc diff --git a/studies/ci-recurrence/README.md b/studies/ci-recurrence/README.md new file mode 100644 index 0000000..1fdf73c --- /dev/null +++ b/studies/ci-recurrence/README.md @@ -0,0 +1,103 @@ +# CI failure recurrence study + +**Question.** Over 90 days on one repo, walk failure events chronologically. A +failure is a RECURRENCE if its fingerprint appeared earlier in the window. +`rate = recurrences / total_failures`. + +**Decision rule.** Below roughly 30% at the most defensible fingerprinting, the +premise is dead. + +## Status + +Tier 0 and the collection/analysis path are built and tested. **No recurrence +rate has been computed yet** — the session this was written in cannot reach the +GitHub Actions API for `pytorch/pytorch` or `ankidroid/Anki-Android` (see +`findings.md`). Run the two commands below on the study machine and the numbers +drop out. + +The Tier 1 clustering loop is deliberately **not built yet**. It should not be +written until the Tier 0 collapse rate says how many singletons it would have +to chew through — that number decides whether Tier 1 is a night of local +inference or a week of it. + +## Run it + +```bash +export GITHUB_TOKEN= + +uv run fetch.py --repo ankidroid/Anki-Android --since-days 90 +uv run analyze.py --repo ankidroid/Anki-Android --top 50 +``` + +`fetch.py` caches every API response under `cache/` keyed by URL hash, so a +re-run costs no API calls and the window can be rebuilt offline. `analyze.py` +touches no network and no model — rerun it freely. + +For pytorch, scope the crawl or it will run for hours: + +```bash +uv run fetch.py --repo pytorch/pytorch --since-days 90 \ + --workflow trunk --workflow pull --max-runs 5000 +``` + +Tests (no network, no model, no tokens): + +```bash +uv run test_pipeline.py +``` + +## The cascade + +| Tier | Who | Handles | Cost | +|---|---|---|---| +| 0 | regex normalizer | everything with a matching fingerprint | free, reproducible, hashable | +| 1 | local model via Ollama | Tier 0 singletons only | electricity | +| 2 | Gemini Flash | low-confidence Tier 1, or when local is the bottleneck | metered | +| 3 | coordinator | audit of 20 sampled labels per batch | attention | + +Tier 0 is the one that matters. An LLM label is not reproducible and not +hashable; a regex fingerprint is both. Every event Tier 0 places in a group of +2+ is an event no worker ever has to look at. + +## Files + +| File | What | +|---|---| +| `schema.sql` | SQLite: queue, cache, and checkpoint in one file | +| `normalize.py` | **Tier 0.** Normalization rules + fingerprint | +| `fetch.py` | GitHub Actions collector, disk-cached, read-only | +| `analyze.py` | Flake split, Tier 0 collapse stats, chronological walk | +| `prompts.py` | **Tier 1 worker prompt.** Version it on every edit | +| `dispatch.py` | The whole harness: one `ask()`, content-hash cache, retry, tokens | +| `test_pipeline.py` | Known-answer tests for the model-free parts | + +## Three decisions worth arguing with + +**Exit codes survive number-stripping.** `exit code 137` is an OOM kill and +`exit code 143` is a timeout; `exit code 1` is a test failing. Stripping them as +"bare ints" merges infrastructure failures into test failures and inflates the +rate. `KEEP_EXIT_CODES` in `normalize.py` toggles it so the collapse rate can be +reported both ways. + +**Path basenames survive, directory prefixes don't.** `test_nn.py` and +`test_optim.py` are different failures; `/home/runner/work/...` vs +`/opt/actions-runner/_work/...` is the same failure on a different machine. So +absolute paths collapse to `/test_nn.py`. Repo-relative paths from Check +Run annotations are left fully intact — they are identical on every runner, so +they are signal, not noise. + +**The worker is told to answer "different" when torn.** Merging is the +destructive direction: every bad merge turns a first-seen failure into a +recurrence and pushes the rate up, toward the 30% line we are testing against. +The reported rate is therefore a lower bound, which is the only version worth +betting on. + +## One correctness trap, documented so nobody re-introduces it + +You cannot find flakes by listing `status=failure` runs. When a failed run is +re-run and passes, GitHub **rewrites the run's conclusion to `success`** — the +flaky run vanishes from the failure list, taking the fail-then-pass evidence +with it. So `fetch.py` lists all *completed* runs and pulls jobs for any run +that is non-success **or** has `run_attempt > 1`, and records every job outcome +including successes. Filtering on failure would silently drop the flakes and +leave a rate that cannot be corrected after the fact. diff --git a/studies/ci-recurrence/analyze.py b/studies/ci-recurrence/analyze.py new file mode 100644 index 0000000..0cfbd07 --- /dev/null +++ b/studies/ci-recurrence/analyze.py @@ -0,0 +1,193 @@ +"""Flake split, Tier 0 collapse stats, and the chronological recurrence walk. + +THE DEFINITION, verbatim from the brief: + Walk failure events chronologically over the window. A failure is a + RECURRENCE if its fingerprint appeared earlier in the window. + rate = recurrences / total_failures. + +Nothing here calls a model. Tier 0 numbers come out of this file alone, which +is the point: they are reproducible and cost nothing. +""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter + +import db + + +# ----------------------------------------------------------------- flake split +def mark_flakes(con, repo: str) -> dict[str, int]: + """Same head SHA, re-run, fail then pass => flake, not recurrence. + + Two shapes, both requiring an empty diff (identical head SHA, so nothing + changed between the fail and the pass): + A. same run_id + job_name, a LATER attempt succeeded (the classic re-run) + B. same head_sha + workflow + job_name, a later run succeeded + """ + con.execute("UPDATE failure_event SET is_flake=0, flake_reason=NULL WHERE repo=?", (repo,)) + + # A: later successful attempt of the same job in the same run + con.execute( + """UPDATE failure_event SET is_flake=1, flake_reason='rerun_attempt_passed' + WHERE repo=?1 AND EXISTS ( + SELECT 1 FROM job_outcome jo + WHERE jo.repo=?1 AND jo.run_id=failure_event.run_id + AND jo.job_name=failure_event.job_name + AND jo.conclusion='success' + AND jo.run_attempt > failure_event.run_attempt)""", + (repo,), + ) + # B: later successful run on the identical head SHA + con.execute( + """UPDATE failure_event SET is_flake=1, flake_reason='same_sha_later_pass' + WHERE repo=?1 AND is_flake=0 AND EXISTS ( + SELECT 1 FROM job_outcome jo + WHERE jo.repo=?1 AND jo.head_sha=failure_event.head_sha + AND jo.job_name=failure_event.job_name + AND IFNULL(jo.workflow,'')=IFNULL(failure_event.workflow,'') + AND jo.conclusion='success' + AND jo.started_at > failure_event.started_at)""", + (repo,), + ) + con.commit() + rows = con.execute( + "SELECT IFNULL(flake_reason,'not_flake') r, COUNT(*) c FROM failure_event WHERE repo=? GROUP BY r", + (repo,), + ).fetchall() + return {r["r"]: r["c"] for r in rows} + + +# ------------------------------------------------------------- recurrence walk +def walk(con, repo: str, *, key: str = "fingerprint", exclude_flakes: bool = False) -> dict: + """Chronological walk. Returns totals and the recurrence rate.""" + sql = ( + f"SELECT {key} AS k, started_at FROM failure_event " + "WHERE repo=? AND fingerprint != '' " + + ("AND is_flake=0 " if exclude_flakes else "") + + "ORDER BY started_at ASC, job_id ASC" + ) + seen: set[str] = set() + total = recurrences = 0 + for row in con.execute(sql, (repo,)): + k = row["k"] + if not k: + continue + total += 1 + if k in seen: + recurrences += 1 + else: + seen.add(k) + return { + "total_failures": total, + "recurrences": recurrences, + "distinct": len(seen), + "rate": (recurrences / total) if total else 0.0, + } + + +# -------------------------------------------------------------- tier 0 metrics +def tier0_stats(con, repo: str) -> dict: + total = con.execute("SELECT COUNT(*) c FROM failure_event WHERE repo=?", (repo,)).fetchone()["c"] + with_ev = con.execute( + "SELECT COUNT(*) c FROM failure_event WHERE repo=? AND fingerprint != ''", (repo,) + ).fetchone()["c"] + by_kind = { + r["evidence_kind"]: r["c"] + for r in con.execute( + "SELECT evidence_kind, COUNT(*) c FROM failure_event WHERE repo=? GROUP BY evidence_kind", + (repo,), + ) + } + fps = [ + r["c"] + for r in con.execute( + "SELECT fingerprint, COUNT(*) c FROM failure_event " + "WHERE repo=? AND fingerprint != '' GROUP BY fingerprint", + (repo,), + ) + ] + distinct = len(fps) + singletons = sum(1 for c in fps if c == 1) + clustered_events = sum(c for c in fps if c > 1) + return { + "failure_events": total, + "events_with_evidence": with_ev, + "log_coverage": (with_ev / total) if total else 0.0, + "evidence_kinds": by_kind, + "distinct_fingerprints": distinct, + # Fraction of events that Tier 0 placed in a group of 2+. This is the + # "Tier 0 handled it" number: those events never need a worker token. + "tier0_collapse": (clustered_events / with_ev) if with_ev else 0.0, + "singleton_fingerprints": singletons, + # What Tier 1 would actually have to chew through. + "tier1_candidates": singletons, + } + + +def top_fingerprints(con, repo: str, n: int = 50) -> list[dict]: + rows = con.execute( + """SELECT fingerprint, COUNT(*) c, + MIN(started_at) first_seen, MAX(started_at) last_seen, + MIN(norm_message) exemplar, MIN(evidence_kind) kind + FROM failure_event WHERE repo=? AND fingerprint != '' + GROUP BY fingerprint ORDER BY c DESC, first_seen ASC LIMIT ?""", + (repo, n), + ).fetchall() + return [dict(r) for r in rows] + + +def report(repo: str, out_json: str | None = None) -> dict: + con = db.connect() + flakes = mark_flakes(con, repo) + stats = tier0_stats(con, repo) + result = { + "repo": repo, + "window_start": db.meta_get(con, f"window_start:{repo}"), + "since_days": db.meta_get(con, f"since_days:{repo}", "90"), + "runs_scanned": db.meta_get(con, f"runs_scanned:{repo}", "0"), + "tier0": stats, + "flake_breakdown": flakes, + "recurrence": { + "tier0_with_flakes": walk(con, repo, key="fingerprint", exclude_flakes=False), + "tier0_without_flakes": walk(con, repo, key="fingerprint", exclude_flakes=True), + }, + "top_fingerprints": top_fingerprints(con, repo, 50), + } + if out_json: + with open(out_json, "w") as f: + json.dump(result, f, indent=2) + return result + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--repo", required=True) + p.add_argument("--json", default=None) + p.add_argument("--top", type=int, default=50) + a = p.parse_args() + r = report(a.repo, a.json) + t, rec = r["tier0"], r["recurrence"] + print(f"repo {r['repo']} window {r['since_days']}d from {r['window_start']}") + print(f"runs scanned {r['runs_scanned']}") + print(f"failure events {t['failure_events']}") + print(f"log coverage {t['log_coverage']:.1%} ({t['evidence_kinds']})") + print(f"distinct fingerprints {t['distinct_fingerprints']}") + print(f"TIER 0 COLLAPSE {t['tier0_collapse']:.1%}") + print(f"tier 1 candidates {t['tier1_candidates']} singletons") + print(f"flakes {r['flake_breakdown']}") + print(f"RECURRENCE with flakes {rec['tier0_with_flakes']['rate']:.1%} " + f"({rec['tier0_with_flakes']['recurrences']}/{rec['tier0_with_flakes']['total_failures']})") + print(f"RECURRENCE without flakes {rec['tier0_without_flakes']['rate']:.1%} " + f"({rec['tier0_without_flakes']['recurrences']}/{rec['tier0_without_flakes']['total_failures']})") + print() + print(f"--- top {a.top} Tier 0 fingerprints ---") + for i, fp in enumerate(r["top_fingerprints"][: a.top], 1): + print(f"{i:3}. {fp['fingerprint']} n={fp['c']:<5} {fp['kind']:<10} {fp['exemplar'][:110]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studies/ci-recurrence/db.py b/studies/ci-recurrence/db.py new file mode 100644 index 0000000..30f138c --- /dev/null +++ b/studies/ci-recurrence/db.py @@ -0,0 +1,32 @@ +"""SQLite handle + tiny helpers. SQLite is the queue, cache, and checkpoint.""" + +from __future__ import annotations + +import pathlib +import sqlite3 +from datetime import datetime, timezone + +ROOT = pathlib.Path(__file__).parent +DB_PATH = ROOT / "out" / "study.db" + + +def now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def connect(path: pathlib.Path | str = DB_PATH) -> sqlite3.Connection: + path = pathlib.Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(path) + con.row_factory = sqlite3.Row + con.executescript((ROOT / "schema.sql").read_text()) + return con + + +def meta_set(con: sqlite3.Connection, key: str, value: str) -> None: + con.execute("INSERT INTO meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value)) + + +def meta_get(con: sqlite3.Connection, key: str, default: str = "") -> str: + row = con.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() + return row["value"] if row else default diff --git a/studies/ci-recurrence/dispatch.py b/studies/ci-recurrence/dispatch.py new file mode 100644 index 0000000..3b32d85 --- /dev/null +++ b/studies/ci-recurrence/dispatch.py @@ -0,0 +1,185 @@ +"""The whole harness: one dispatch function, a content-hash cache, retry, tokens. + +Deliberately dumb. No framework, no provider abstraction beyond the two +providers actually in use, no config file. If this file starts growing a +plugin system, delete the plugin system. + + ask(con, tier, fp_a, text_a, fp_b, text_b) -> (same, confidence) + +Cache first, always. The cache key is a content hash over +(prompt_version, tier, model, text_a, text_b), so re-runs after a crash cost +nothing and a prompt revision correctly invalidates the old labels. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +import urllib.error +import urllib.request + +import db +import prompts + +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434/api/generate") +OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "qwen3:30b") +GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash") +GEMINI_KEY = os.environ.get("GEMINI_API_KEY", "") + +# USD per 1M tokens. VERIFY against current pricing before quoting a total -- +# these are the numbers the cost estimate multiplies, and stale rates make the +# dollar figure confidently wrong. Local inference is metered at zero: the M1 +# is already paid for, and electricity is not what this study is measuring. +PRICING = { + "local": (0.0, 0.0), + GEMINI_MODEL: (0.30, 2.50), +} + + +def cache_key(tier: int, model: str, a: str, b: str) -> str: + h = hashlib.sha256() + for part in (prompts.PROMPT_VERSION, str(tier), model, a, b): + h.update(part.encode("utf-8")) + h.update(b"\x1f") + return h.hexdigest()[:32] + + +def _parse(text: str) -> tuple[bool | None, float]: + """Pull the JSON verdict out of a worker reply. Models add fences and prose.""" + t = (text or "").strip() + if t.startswith("```"): + t = t.strip("`") + t = t.split("\n", 1)[1] if "\n" in t else t + i, j = t.find("{"), t.rfind("}") + if i == -1 or j == -1: + return None, 0.0 + try: + d = json.loads(t[i : j + 1]) + except json.JSONDecodeError: + return None, 0.0 + same = d.get("same") + if isinstance(same, str): + same = same.strip().lower() in ("true", "yes", "same") + if not isinstance(same, bool): + return None, 0.0 + try: + conf = float(d.get("confidence", 0.0)) + except (TypeError, ValueError): + conf = 0.0 + return same, max(0.0, min(1.0, conf)) + + +def _post(url: str, payload: dict, headers: dict, timeout: int) -> dict: + req = urllib.request.Request( + url, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", **headers} + ) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode()) + + +def _call_ollama(prompt: str) -> tuple[str, int, int]: + body = _post( + OLLAMA_URL, + { + "model": OLLAMA_MODEL, + "prompt": prompt, + "system": prompts.SYSTEM, + "stream": False, + "format": "json", + "options": {"temperature": 0, "num_predict": 64}, + }, + {}, + timeout=180, # Ollama stalls; the retry loop above handles the rest. + ) + return body.get("response", ""), body.get("prompt_eval_count", 0), body.get("eval_count", 0) + + +def _call_gemini(prompt: str) -> tuple[str, int, int]: + if not GEMINI_KEY: + raise RuntimeError("GEMINI_API_KEY not set") + url = f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent" + body = _post( + f"{url}?key={GEMINI_KEY}", + { + "systemInstruction": {"parts": [{"text": prompts.SYSTEM}]}, + "contents": [{"role": "user", "parts": [{"text": prompt}]}], + "generationConfig": {"temperature": 0, "maxOutputTokens": 64, "responseMimeType": "application/json"}, + }, + {}, + timeout=120, + ) + cand = (body.get("candidates") or [{}])[0] + text = "".join(p.get("text", "") for p in cand.get("content", {}).get("parts", [])) + u = body.get("usageMetadata", {}) + return text, u.get("promptTokenCount", 0), u.get("candidatesTokenCount", 0) + + +def ask(con, tier: int, fp_a: str, text_a: str, fp_b: str, text_b: str, repo: str = "") -> tuple[bool | None, float]: + """Cache-first worker call. tier 1 = local via Ollama, tier 2 = Gemini Flash.""" + model = "local" if tier == 1 else GEMINI_MODEL + key = cache_key(tier, model, text_a, text_b) + + row = con.execute("SELECT same, confidence FROM worker_call WHERE cache_key=?", (key,)).fetchone() + if row is not None: + return (None if row["same"] is None else bool(row["same"])), (row["confidence"] or 0.0) + + prompt = prompts.build_local(text_a, text_b) if tier == 1 else prompts.build(text_a, text_b) + caller = _call_ollama if tier == 1 else _call_gemini + + same, conf, ptok, otok, err = None, 0.0, 0, 0, None + t0 = time.time() + delay = 2.0 + for attempt in range(4): + try: + raw, ptok, otok = caller(prompt) + same, conf = _parse(raw) + if same is None: + err = f"unparseable: {raw[:120]}" + else: + err = None + break + except urllib.error.HTTPError as e: + err = f"http {e.code}" + if e.code in (429, 500, 502, 503, 504): + time.sleep(delay) + delay *= 2 + continue + break + except Exception as e: # timeouts, connection resets, Ollama stalls + err = f"{type(e).__name__}: {e}" + time.sleep(delay) + delay *= 2 + + con.execute( + """INSERT OR REPLACE INTO worker_call + (cache_key,repo,tier,model,prompt_version,fp_a,fp_b,same,confidence,rationale, + prompt_tokens,output_tokens,latency_ms,error,created_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + (key, repo, tier, model, prompts.PROMPT_VERSION, fp_a, fp_b, + None if same is None else int(same), conf, None, + ptok, otok, int((time.time() - t0) * 1000), err, db.now()), + ) + return same, conf + + +def spend(con) -> dict: + """Running token + dollar totals, by tier. Printed after every batch.""" + out: dict[str, dict] = {} + total = 0.0 + for r in con.execute( + """SELECT tier, model, SUM(prompt_tokens) p, SUM(output_tokens) o, + COUNT(*) n, SUM(error IS NOT NULL) errs + FROM worker_call GROUP BY tier, model""" + ): + pin, pout = PRICING.get(r["model"], (0.0, 0.0)) + usd = (r["p"] or 0) / 1e6 * pin + (r["o"] or 0) / 1e6 * pout + total += usd + out[f"tier{r['tier']}:{r['model']}"] = { + "calls": r["n"], "errors": r["errs"], + "prompt_tokens": r["p"] or 0, "output_tokens": r["o"] or 0, + "usd": round(usd, 4), + } + out["total_usd"] = round(total, 4) + return out diff --git a/studies/ci-recurrence/fetch.py b/studies/ci-recurrence/fetch.py new file mode 100644 index 0000000..8b4a7ad --- /dev/null +++ b/studies/ci-recurrence/fetch.py @@ -0,0 +1,258 @@ +"""Collect CI failure events from the GitHub Actions API. Read-only, disk-cached. + +Every API response is written to cache/ keyed by URL hash, so re-runs never +re-fetch and the 90-day window can be rebuilt offline. + +ONE NON-OBVIOUS CORRECTNESS POINT, and it drives the whole fetch strategy: + + You cannot find flakes by listing `status=failure` runs. When a failed run is + re-run and passes, GitHub REWRITES the run's conclusion to `success`. The + flaky run then disappears from the failure list entirely -- taking with it + exactly the fail-then-pass evidence the flake split depends on. Filtering on + failure would silently drop the flakes and leave a recurrence rate that + cannot be corrected afterwards. + + So we list ALL completed runs, then pull jobs only for runs that are either + non-success or have run_attempt > 1. A first-attempt success has no failed + jobs and is skipped, which keeps the extra cost small. + +Usage: + uv run fetch.py --repo pytorch/pytorch --since-days 90 --max-runs 5000 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +import db +from normalize import fingerprint, normalize + +API = "https://api.github.com" +CACHE = pathlib.Path(__file__).parent / "cache" +UA = "ci-recurrence-study" + + +# ------------------------------------------------------------------ HTTP layer +class Gh: + def __init__(self, token: str, cache: pathlib.Path = CACHE) -> None: + self.token = token + self.cache = cache + self.cache.mkdir(parents=True, exist_ok=True) + self.calls = 0 + self.cache_hits = 0 + + def _cache_path(self, url: str) -> pathlib.Path: + h = hashlib.sha256(url.encode()).hexdigest()[:24] + return self.cache / f"{h}.json" + + def get(self, url: str, *, allow_404: bool = False) -> tuple[dict | list | None, int]: + """Returns (payload, status). status 0 means served from disk cache.""" + cp = self._cache_path(url) + if cp.exists(): + self.cache_hits += 1 + try: + return json.loads(cp.read_text()), 0 + except json.JSONDecodeError: + cp.unlink() # corrupt cache entry; refetch + + delay = 2.0 + for attempt in range(5): + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": UA, + }, + ) + try: + with urllib.request.urlopen(req, timeout=60) as r: + body = json.loads(r.read().decode()) + self.calls += 1 + remaining = int(r.headers.get("X-RateLimit-Remaining", "1000")) + if remaining < 50: + reset = int(r.headers.get("X-RateLimit-Reset", "0")) + nap = max(0, reset - int(time.time())) + 5 + print(f" [rate limit] {remaining} left, sleeping {nap}s", file=sys.stderr) + time.sleep(nap) + cp.write_text(json.dumps(body)) + return body, 200 + except urllib.error.HTTPError as e: + if e.code == 404 and allow_404: + cp.write_text(json.dumps(None)) + return None, 404 + # 403/429 here are secondary rate limits, not policy denials. + if e.code in (403, 429): + reset = e.headers.get("X-RateLimit-Reset") + nap = max(0, int(reset) - int(time.time())) + 5 if reset else delay + print(f" [{e.code}] backing off {nap}s", file=sys.stderr) + time.sleep(min(nap, 900)) + delay *= 2 + continue + if 500 <= e.code < 600: + time.sleep(delay) + delay *= 2 + continue + raise + except (urllib.error.URLError, TimeoutError): + time.sleep(delay) + delay *= 2 + return None, -1 + + +# ------------------------------------------------------------- evidence picking +def pick_evidence(gh: Gh, repo: str, job: dict) -> tuple[str, str]: + """Return (evidence_kind, raw_message). Annotations preferred over logs.""" + job_id = job["id"] + ann, status = gh.get(f"{API}/repos/{repo}/check-runs/{job_id}/annotations", allow_404=True) + if isinstance(ann, list) and ann: + # failure annotations first; a job often carries warnings too + msgs = [a for a in ann if (a.get("annotation_level") or "").lower() == "failure"] or ann + parts = [] + for a in msgs[:5]: + loc = a.get("path") or "" + line = a.get("start_line") + title = (a.get("title") or "").strip() + body = (a.get("message") or "").strip() + parts.append(f"{loc}:{line}: {title} {body}".strip()) + return "annotation", "\n".join(parts) + + # Fallback: the failed step's name. Weak evidence, but it is evidence, and + # marking it as such is what keeps the coverage fraction honest. + for step in job.get("steps") or []: + if step.get("conclusion") == "failure": + return "step_name", f"step failed: {step.get('name')}" + return "none", "" + + +# ------------------------------------------------------------------ collection +def collect(repo: str, since_days: int, max_runs: int, token: str, workflows: set[str] | None) -> None: + con = db.connect() + gh = Gh(token) + since = (datetime.now(timezone.utc) - timedelta(days=since_days)).strftime("%Y-%m-%d") + db.meta_set(con, f"window_start:{repo}", since) + db.meta_set(con, f"since_days:{repo}", str(since_days)) + + seen_runs = 0 + page = 1 + events = 0 + while seen_runs < max_runs: + url = ( + f"{API}/repos/{repo}/actions/runs" + f"?status=completed&created=%3E%3D{since}&per_page=100&page={page}" + ) + payload, status = gh.get(url) + if not payload or not payload.get("workflow_runs"): + break + runs = payload["workflow_runs"] + con.execute( + "INSERT OR REPLACE INTO fetch_log(repo,kind,ref,status,ok,note,fetched_at) VALUES(?,?,?,?,?,?,?)", + (repo, "runs_page", str(page), status, 1, f"{len(runs)} runs", db.now()), + ) + + for run in runs: + seen_runs += 1 + if workflows and run.get("name") not in workflows: + continue + # See module docstring: a first-attempt success cannot contain a + # failed job, but a re-run success can (that is the flake). + if run.get("conclusion") == "success" and run.get("run_attempt", 1) <= 1: + continue + events += ingest_run(gh, con, repo, run) + + con.commit() + print(f" page {page}: {seen_runs} runs scanned, {events} failure events, " + f"{gh.calls} api calls, {gh.cache_hits} cached", file=sys.stderr) + page += 1 + if len(runs) < 100: + break + + db.meta_set(con, f"runs_scanned:{repo}", str(seen_runs)) + con.commit() + print(f"[{repo}] {seen_runs} runs scanned -> {events} failure events " + f"({gh.calls} api calls, {gh.cache_hits} cache hits)") + + +def ingest_run(gh: Gh, con, repo: str, run: dict) -> int: + run_id = run["id"] + # filter=all returns every ATTEMPT's jobs, which is what makes the + # fail-then-pass flake visible. + jobs_payload, status = gh.get( + f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?filter=all&per_page=100" + ) + con.execute( + "INSERT OR REPLACE INTO fetch_log(repo,kind,ref,status,ok,note,fetched_at) VALUES(?,?,?,?,?,?,?)", + (repo, "jobs", str(run_id), status, 1 if jobs_payload else 0, None, db.now()), + ) + if not jobs_payload: + return 0 + + n = 0 + for job in jobs_payload.get("jobs", []): + # Record every job outcome, not just failures: the successes are what + # make a fail-then-pass on one head SHA detectable as a flake. + con.execute( + """INSERT OR REPLACE INTO job_outcome + (repo,run_id,job_id,run_attempt,workflow,job_name,head_sha,conclusion,started_at) + VALUES(?,?,?,?,?,?,?,?,?)""", + ( + repo, run_id, job["id"], job.get("run_attempt", run.get("run_attempt", 1)), + run.get("name"), job.get("name"), run.get("head_sha", ""), + job.get("conclusion"), job.get("started_at") or run.get("created_at"), + ), + ) + if job.get("conclusion") != "failure": + continue + kind, raw = pick_evidence(gh, repo, job) + norm = normalize(raw) + fp = fingerprint(raw) + con.execute( + """INSERT OR REPLACE INTO failure_event + (event_id,repo,run_id,run_attempt,job_id,workflow,job_name,head_sha,head_branch, + trigger_event,started_at,completed_at,failed_step,evidence_kind,raw_message, + norm_message,fingerprint,fetched_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + ( + f"{repo}:{run_id}:{job['id']}", repo, run_id, + job.get("run_attempt", run.get("run_attempt", 1)), job["id"], + run.get("name"), job.get("name"), run.get("head_sha", ""), + run.get("head_branch"), run.get("event"), + job.get("started_at") or run.get("created_at"), job.get("completed_at"), + next((s.get("name") for s in job.get("steps") or [] + if s.get("conclusion") == "failure"), None), + kind, raw, norm, fp, db.now(), + ), + ) + n += 1 + return n + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--repo", required=True, help="owner/name") + p.add_argument("--since-days", type=int, default=90) + p.add_argument("--max-runs", type=int, default=100000) + p.add_argument("--workflow", action="append", default=None, + help="restrict to these workflow names (repeatable); useful for pytorch") + a = p.parse_args() + + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or "" + if not token: + print("set GITHUB_TOKEN (a classic or fine-grained PAT with public_repo)", file=sys.stderr) + return 2 + collect(a.repo, a.since_days, a.max_runs, token, set(a.workflow) if a.workflow else None) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studies/ci-recurrence/findings.md b/studies/ci-recurrence/findings.md new file mode 100644 index 0000000..1a8a436 --- /dev/null +++ b/studies/ci-recurrence/findings.md @@ -0,0 +1,91 @@ +# Findings + +## Verdict: not yet reached + +No recurrence rate has been computed, so the 30% decision rule has **not** been +evaluated. The premise is neither supported nor dead. This file gets the verdict +once `analyze.py` runs against fetched data. + +## Why no number yet + +The session this was built in cannot reach the GitHub Actions API for either +target repo. Both access paths are closed: + +| Path | Result | +|---|---| +| `curl api.github.com/repos/ankidroid/Anki-Android/actions/runs` | `403` — "GitHub access to this repository is not enabled for this session" | +| `curl .../repos/pytorch/pytorch/actions/runs` | `403`, same | +| `mcp__github__actions_list` on either target | `Access denied: not configured for this session. Allowed repositories: chrishonson/metacortex` | +| `add_repo` with API access for a third-party repo | denied by the permission classifier | + +`api.github.com` itself is reachable — `/rate_limit` returns 200 with a +15000/hr budget — so this is per-repository authorization, not a network or +egress block. + +The one repository this session *can* reach, `chrishonson/Metacortex`, has 24 +workflow runs in its entire history and **zero failures** (18 success, 6 +skipped, all from two Claude Code workflows). There is no failure corpus in it +to fingerprint, so it cannot stand in even as a smoke test of the real numbers. + +Per the brief's own instruction — *"if you find yourself three files deep in +infrastructure without a fingerprint computed, stop and tell me"* — collection +stopped here rather than continuing to build the Tier 1 loop on top of an +unmeasured Tier 0. + +## What is built and verified + +Everything up to the API call, tested with known-answer fixtures +(`uv run test_pipeline.py`, 20/20 passing): + +- **Tier 0 normalizer** — verified to collapse the same bug across differing + paths, line numbers, and float values; verified to keep `test_nn.py` distinct + from `test_optim.py`, and `exit code 1` distinct from `exit code 137`; + verified idempotent on its own output. +- **Chronological recurrence walk** — verified against a hand-built sequence + with a known rate, including that no-evidence events are excluded from the + numerator/denominator but still counted in log coverage. +- **Flake split** — both shapes verified (later attempt of the same run passed; + later run on the identical head SHA passed). +- **Ingest path** — verified end-to-end against canned API payloads: annotation + evidence preferred, step-name fallback when a job has no annotations, and + every job outcome recorded including successes. + +Two real bugs were found and fixed by this testing, both of which would have +quietly corrupted the headline number: + +1. A port rule (`:\d{4,5}`) fired on **four-digit line numbers** before the + line/column rule could see them, so `test_nn.py:1234:5` and + `test_nn.py:998:11` fingerprinted differently. In a repo the size of pytorch + this shatters collapse across every large file — it would have understated + the recurrence rate badly. The rule was removed; ports fall through to the + generic int rule, which is equally stable. +2. The absolute-path rule matched the `/src/Foo.kt` tail inside a + **repo-relative** `app/src/Foo.kt`, mangling it to `app/Foo.kt`. + Repo-relative paths are exactly what Check Run annotations carry, and they + are identical on every runner — signal, not noise. Now only absolute paths + collapse. + +## To finish this + +On a machine with a personal PAT: + +```bash +export GITHUB_TOKEN= +uv run fetch.py --repo ankidroid/Anki-Android --since-days 90 +uv run analyze.py --repo ankidroid/Anki-Android --top 50 +``` + +AnkiDroid first: it is small enough to complete in one sitting and gives a real +Tier 0 collapse rate to size Tier 1 against. pytorch needs a scoped crawl +(`--workflow trunk --workflow pull --max-runs 5000`); at ~1000+ completed runs +a day, a full 90-day walk of it is several hours of API calls even with the +15000/hr budget, and that projection should be re-checked against the actual +page-1 `total_count` before committing to it. + +## Open items, in order + +1. Tier 0 collapse rate and log coverage on AnkiDroid — no tokens required. +2. Singleton count from step 1 sizes Tier 1. If singletons run to six figures, + a 20b-class model on one M1 Max will not finish overnight and the split has + to push toward Flash; that call is not worth making before the number exists. +3. Only then build the Tier 1 batch loop and the audit sampler. diff --git a/studies/ci-recurrence/normalize.py b/studies/ci-recurrence/normalize.py new file mode 100644 index 0000000..48b64b8 --- /dev/null +++ b/studies/ci-recurrence/normalize.py @@ -0,0 +1,166 @@ +"""Tier 0: deterministic normalization + fingerprinting of CI failure text. + +No model. Free, reproducible, and hashable -- which an LLM label is not. +Every failure event gets a Tier 0 fingerprint; tiers 1/2 only ever look at +what Tier 0 leaves as a singleton. + +Rules apply in the order listed. Order matters: specific patterns (UUIDs, hex, +paths) must fire before the catch-all number rule, or the number rule eats +their internals and destroys the distinction. + +Two judgment calls worth arguing about, both flagged and both measurable: + + KEEP_EXIT_CODES -- `exit code 137` (OOM-kill) and `exit code 143` (SIGTERM + / timeout) are different failures from `exit code 1` (tests failed). The + brief says strip bare ints; stripping these would merge an infra failure + into a test failure and inflate the recurrence rate. Kept by default, + switchable so we can report the collapse rate both ways. + + KEEP_BASENAME -- `/home/runner/work/pytorch/pytorch/test/test_nn.py` and + `.../test_optim.py` are different failures. The directory prefix is pure + noise (it encodes the runner, not the problem); the basename is signal. + So paths collapse to `/test_nn.py`, not ``. +""" + +from __future__ import annotations + +import hashlib +import re + +TIER0_VERSION = "t0-2026-08-14" + +KEEP_EXIT_CODES = True +KEEP_BASENAME = True +MAX_LEN = 512 + +# Sentinel that shields exit-code digits from the catch-all int rule. Letters +# only: the int rule's lookbehind refuses to match a digit preceded by a word +# character, so EXITCODE137 survives intact and is unwrapped at the end. +_SENTINEL = "EXITCODE" + +_R: list[tuple[str, re.Pattern[str], str]] = [] + + +def _rule(name: str, pattern: str, repl: str, flags: int = 0) -> None: + _R.append((name, re.compile(pattern, flags), repl)) + + +# --- 1. transport noise: escapes, log decoration, GH Actions framing ---------- +_rule("ansi", r"\x1b\[[0-9;?]*[a-zA-Z]", "") +_rule("cr", r"\r", "") +# GH Actions prefixes every raw-log line with an RFC3339 nano timestamp. +_rule("gha_ts", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s*", "", re.M) +_rule("gha_grp", r"##\[(?:group|endgroup|debug|command|section)\][^\n]*", "") +_rule("gha_err", r"##\[(?:error|warning)\]", "") + +# --- 2. identifiers that are unique per run ---------------------------------- +_rule("url", r"https?://[^\s'\"<>)\]]+", "") +_rule("uuid", r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b", "") +_rule("hexlit", r"\b0[xX][0-9a-fA-F]+\b", "") +# Bare hex blobs (git shas, content hashes): >=7 chars, must mix digit+letter +# so ordinary lowercase words don't get swallowed. +_rule("sha", r"\b(?=[0-9a-f]{7,64}\b)(?=[a-f]*[0-9])(?=[0-9]*[a-f])[0-9a-f]{7,64}\b", "") +_rule("ipv4", r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", "") +_rule("email", r"\b[\w.+-]+@[\w-]+\.[\w.]+\b", "") + +# --- 3. runner / container / temp identity ----------------------------------- +_rule("pytest_tmp", r"pytest-of-\w+/pytest-\d+(?:/[\w.-]+)?", "") +_rule("tmpdir", r"/tmp/[^\s:'\"]*", "") +_rule("runner", r"\b(?:i-[0-9a-f]{8,}|ip-\d+-\d+-\d+-\d+|runner-[\w-]+|gh-ci-[\w-]+)\b", "") +# No port rule on purpose. A `:\d{4,5}` rule fires on 4-digit LINE NUMBERS +# (`test_nn.py:1234:5`) before the linecol rule can see them, which shatters +# collapse across every large file in the repo. Ports fall through to the int +# rule and become `:`, which is equally stable -- `` only ever bought +# readability, never a distinct fingerprint. + +# --- 4. time, duration, size -------------------------------------------------- +_rule("iso", r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", "") +_rule("date", r"\b\d{4}-\d{2}-\d{2}\b", "") +_rule("clock", r"\b\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\b", "