diff --git a/studies/ci-recurrence/.gitignore b/studies/ci-recurrence/.gitignore new file mode 100644 index 0000000..63646c9 --- /dev/null +++ b/studies/ci-recurrence/.gitignore @@ -0,0 +1,4 @@ +cache/ +out/ +__pycache__/ +*.pyc diff --git a/studies/ci-recurrence/README.md b/studies/ci-recurrence/README.md new file mode 100644 index 0000000..f8a7f31 --- /dev/null +++ b/studies/ci-recurrence/README.md @@ -0,0 +1,129 @@ +# CI failure recurrence study + +**Question.** Over 90 days on one repo, walk failure events chronologically. A +failure is a RECURRENCE if its fingerprint appeared earlier in the window. +`rate = recurrences / total_failures`. + +**Decision rule.** Below roughly 30% at the most defensible fingerprinting, the +premise is dead. + +## Status + +Tier 0 and the collection/analysis path are built and tested. **No recurrence +rate has been computed yet** — the session this was written in cannot reach the +GitHub Actions API for `pytorch/pytorch` or `ankidroid/Anki-Android` (see +`findings.md`). Run the two commands below on the study machine and the numbers +drop out. + +The Tier 1 clustering loop is deliberately **not built yet**. It should not be +written until the Tier 0 collapse rate says how many singletons it would have +to chew through — that number decides whether Tier 1 is a night of local +inference or a week of it. + +## Run it + +```bash +export GITHUB_TOKEN= + +uv run fetch.py --repo ankidroid/Anki-Android --since-days 90 +uv run analyze.py --repo ankidroid/Anki-Android --top 50 +``` + +### What token + +**A classic PAT with zero scopes checked.** Both targets are public, and every +endpoint this study touches is public-readable: + +| Endpoint | Needs | +|---|---| +| `/repos/{o}/{r}/actions/runs` | nothing, for a public repo | +| `/repos/{o}/{r}/actions/runs/{id}/jobs` | nothing, for a public repo | +| `/repos/{o}/{r}/check-runs/{id}/annotations` | nothing, for a public repo | + +The token is not there for access, it is there for **rate limit**: +unauthenticated is 60 requests/hour, which makes a 90-day crawl impossible; any +valid token raises that to 5,000/hour regardless of its scopes. + +Do **not** tick `public_repo`. That scope grants *write* to every public repo +you can see — push, issues, the lot — and buys this study nothing. A scopeless +token can only read public data, which is exactly the blast radius we want for +something crawling two repos we don't own. + +A fine-grained PAT also works: set the resource owner to your own account and +choose **"Public repositories (read-only)"**. Note you cannot scope a +fine-grained PAT *to* `pytorch/pytorch` — fine-grained tokens only target repos +you own, so the public-read option is the route. The scopeless classic token is +simpler and no less safe. + +`fetch.py` caches every API response under `cache/` keyed by URL hash, so a +re-run costs no API calls and the window can be rebuilt offline. `analyze.py` +touches no network and no model — rerun it freely. + +For pytorch, scope the crawl or it will run for hours: + +```bash +uv run fetch.py --repo pytorch/pytorch --since-days 90 \ + --workflow trunk --workflow pull --max-runs 5000 +``` + +Tests (no network, no model, no tokens): + +```bash +uv run test_pipeline.py +``` + +## The cascade + +| Tier | Who | Handles | Cost | +|---|---|---|---| +| 0 | regex normalizer | everything with a matching fingerprint | free, reproducible, hashable | +| 1 | local model via Ollama | Tier 0 singletons only | electricity | +| 2 | Gemini Flash | low-confidence Tier 1, or when local is the bottleneck | metered | +| 3 | coordinator | audit of 20 sampled labels per batch | attention | + +Tier 0 is the one that matters. An LLM label is not reproducible and not +hashable; a regex fingerprint is both. Every event Tier 0 places in a group of +2+ is an event no worker ever has to look at. + +## Files + +| File | What | +|---|---| +| `schema.sql` | SQLite: queue, cache, and checkpoint in one file | +| `normalize.py` | **Tier 0.** Normalization rules + fingerprint | +| `fetch.py` | GitHub Actions collector, disk-cached, read-only | +| `analyze.py` | Flake split, Tier 0 collapse stats, chronological walk | +| `prompts.py` | **Tier 1 worker prompt.** Version it on every edit | +| `dispatch.py` | The whole harness: one `ask()`, content-hash cache, retry, tokens | +| `test_pipeline.py` | Known-answer tests for the model-free parts | + +## Three decisions worth arguing with + +**Exit codes survive number-stripping.** `exit code 137` is an OOM kill and +`exit code 143` is a timeout; `exit code 1` is a test failing. Stripping them as +"bare ints" merges infrastructure failures into test failures and inflates the +rate. `KEEP_EXIT_CODES` in `normalize.py` toggles it so the collapse rate can be +reported both ways. + +**Path basenames survive, directory prefixes don't.** `test_nn.py` and +`test_optim.py` are different failures; `/home/runner/work/...` vs +`/opt/actions-runner/_work/...` is the same failure on a different machine. So +absolute paths collapse to `/test_nn.py`. Repo-relative paths from Check +Run annotations are left fully intact — they are identical on every runner, so +they are signal, not noise. + +**The worker is told to answer "different" when torn.** Merging is the +destructive direction: every bad merge turns a first-seen failure into a +recurrence and pushes the rate up, toward the 30% line we are testing against. +The reported rate is therefore a lower bound, which is the only version worth +betting on. + +## One correctness trap, documented so nobody re-introduces it + +You cannot find flakes by listing `status=failure` runs. When a failed run is +re-run and passes, GitHub **rewrites the run's conclusion to `success`** — the +flaky run vanishes from the failure list, taking the fail-then-pass evidence +with it. So `fetch.py` lists all *completed* runs and pulls jobs for any run +that is non-success **or** has `run_attempt > 1`, and records every job outcome +including successes. Filtering on failure would silently drop the flakes and +leave a rate that cannot be corrected after the fact. diff --git a/studies/ci-recurrence/analyze.py b/studies/ci-recurrence/analyze.py new file mode 100644 index 0000000..0cfbd07 --- /dev/null +++ b/studies/ci-recurrence/analyze.py @@ -0,0 +1,193 @@ +"""Flake split, Tier 0 collapse stats, and the chronological recurrence walk. + +THE DEFINITION, verbatim from the brief: + Walk failure events chronologically over the window. A failure is a + RECURRENCE if its fingerprint appeared earlier in the window. + rate = recurrences / total_failures. + +Nothing here calls a model. Tier 0 numbers come out of this file alone, which +is the point: they are reproducible and cost nothing. +""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter + +import db + + +# ----------------------------------------------------------------- flake split +def mark_flakes(con, repo: str) -> dict[str, int]: + """Same head SHA, re-run, fail then pass => flake, not recurrence. + + Two shapes, both requiring an empty diff (identical head SHA, so nothing + changed between the fail and the pass): + A. same run_id + job_name, a LATER attempt succeeded (the classic re-run) + B. same head_sha + workflow + job_name, a later run succeeded + """ + con.execute("UPDATE failure_event SET is_flake=0, flake_reason=NULL WHERE repo=?", (repo,)) + + # A: later successful attempt of the same job in the same run + con.execute( + """UPDATE failure_event SET is_flake=1, flake_reason='rerun_attempt_passed' + WHERE repo=?1 AND EXISTS ( + SELECT 1 FROM job_outcome jo + WHERE jo.repo=?1 AND jo.run_id=failure_event.run_id + AND jo.job_name=failure_event.job_name + AND jo.conclusion='success' + AND jo.run_attempt > failure_event.run_attempt)""", + (repo,), + ) + # B: later successful run on the identical head SHA + con.execute( + """UPDATE failure_event SET is_flake=1, flake_reason='same_sha_later_pass' + WHERE repo=?1 AND is_flake=0 AND EXISTS ( + SELECT 1 FROM job_outcome jo + WHERE jo.repo=?1 AND jo.head_sha=failure_event.head_sha + AND jo.job_name=failure_event.job_name + AND IFNULL(jo.workflow,'')=IFNULL(failure_event.workflow,'') + AND jo.conclusion='success' + AND jo.started_at > failure_event.started_at)""", + (repo,), + ) + con.commit() + rows = con.execute( + "SELECT IFNULL(flake_reason,'not_flake') r, COUNT(*) c FROM failure_event WHERE repo=? GROUP BY r", + (repo,), + ).fetchall() + return {r["r"]: r["c"] for r in rows} + + +# ------------------------------------------------------------- recurrence walk +def walk(con, repo: str, *, key: str = "fingerprint", exclude_flakes: bool = False) -> dict: + """Chronological walk. Returns totals and the recurrence rate.""" + sql = ( + f"SELECT {key} AS k, started_at FROM failure_event " + "WHERE repo=? AND fingerprint != '' " + + ("AND is_flake=0 " if exclude_flakes else "") + + "ORDER BY started_at ASC, job_id ASC" + ) + seen: set[str] = set() + total = recurrences = 0 + for row in con.execute(sql, (repo,)): + k = row["k"] + if not k: + continue + total += 1 + if k in seen: + recurrences += 1 + else: + seen.add(k) + return { + "total_failures": total, + "recurrences": recurrences, + "distinct": len(seen), + "rate": (recurrences / total) if total else 0.0, + } + + +# -------------------------------------------------------------- tier 0 metrics +def tier0_stats(con, repo: str) -> dict: + total = con.execute("SELECT COUNT(*) c FROM failure_event WHERE repo=?", (repo,)).fetchone()["c"] + with_ev = con.execute( + "SELECT COUNT(*) c FROM failure_event WHERE repo=? AND fingerprint != ''", (repo,) + ).fetchone()["c"] + by_kind = { + r["evidence_kind"]: r["c"] + for r in con.execute( + "SELECT evidence_kind, COUNT(*) c FROM failure_event WHERE repo=? GROUP BY evidence_kind", + (repo,), + ) + } + fps = [ + r["c"] + for r in con.execute( + "SELECT fingerprint, COUNT(*) c FROM failure_event " + "WHERE repo=? AND fingerprint != '' GROUP BY fingerprint", + (repo,), + ) + ] + distinct = len(fps) + singletons = sum(1 for c in fps if c == 1) + clustered_events = sum(c for c in fps if c > 1) + return { + "failure_events": total, + "events_with_evidence": with_ev, + "log_coverage": (with_ev / total) if total else 0.0, + "evidence_kinds": by_kind, + "distinct_fingerprints": distinct, + # Fraction of events that Tier 0 placed in a group of 2+. This is the + # "Tier 0 handled it" number: those events never need a worker token. + "tier0_collapse": (clustered_events / with_ev) if with_ev else 0.0, + "singleton_fingerprints": singletons, + # What Tier 1 would actually have to chew through. + "tier1_candidates": singletons, + } + + +def top_fingerprints(con, repo: str, n: int = 50) -> list[dict]: + rows = con.execute( + """SELECT fingerprint, COUNT(*) c, + MIN(started_at) first_seen, MAX(started_at) last_seen, + MIN(norm_message) exemplar, MIN(evidence_kind) kind + FROM failure_event WHERE repo=? AND fingerprint != '' + GROUP BY fingerprint ORDER BY c DESC, first_seen ASC LIMIT ?""", + (repo, n), + ).fetchall() + return [dict(r) for r in rows] + + +def report(repo: str, out_json: str | None = None) -> dict: + con = db.connect() + flakes = mark_flakes(con, repo) + stats = tier0_stats(con, repo) + result = { + "repo": repo, + "window_start": db.meta_get(con, f"window_start:{repo}"), + "since_days": db.meta_get(con, f"since_days:{repo}", "90"), + "runs_scanned": db.meta_get(con, f"runs_scanned:{repo}", "0"), + "tier0": stats, + "flake_breakdown": flakes, + "recurrence": { + "tier0_with_flakes": walk(con, repo, key="fingerprint", exclude_flakes=False), + "tier0_without_flakes": walk(con, repo, key="fingerprint", exclude_flakes=True), + }, + "top_fingerprints": top_fingerprints(con, repo, 50), + } + if out_json: + with open(out_json, "w") as f: + json.dump(result, f, indent=2) + return result + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--repo", required=True) + p.add_argument("--json", default=None) + p.add_argument("--top", type=int, default=50) + a = p.parse_args() + r = report(a.repo, a.json) + t, rec = r["tier0"], r["recurrence"] + print(f"repo {r['repo']} window {r['since_days']}d from {r['window_start']}") + print(f"runs scanned {r['runs_scanned']}") + print(f"failure events {t['failure_events']}") + print(f"log coverage {t['log_coverage']:.1%} ({t['evidence_kinds']})") + print(f"distinct fingerprints {t['distinct_fingerprints']}") + print(f"TIER 0 COLLAPSE {t['tier0_collapse']:.1%}") + print(f"tier 1 candidates {t['tier1_candidates']} singletons") + print(f"flakes {r['flake_breakdown']}") + print(f"RECURRENCE with flakes {rec['tier0_with_flakes']['rate']:.1%} " + f"({rec['tier0_with_flakes']['recurrences']}/{rec['tier0_with_flakes']['total_failures']})") + print(f"RECURRENCE without flakes {rec['tier0_without_flakes']['rate']:.1%} " + f"({rec['tier0_without_flakes']['recurrences']}/{rec['tier0_without_flakes']['total_failures']})") + print() + print(f"--- top {a.top} Tier 0 fingerprints ---") + for i, fp in enumerate(r["top_fingerprints"][: a.top], 1): + print(f"{i:3}. {fp['fingerprint']} n={fp['c']:<5} {fp['kind']:<10} {fp['exemplar'][:110]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studies/ci-recurrence/db.py b/studies/ci-recurrence/db.py new file mode 100644 index 0000000..30f138c --- /dev/null +++ b/studies/ci-recurrence/db.py @@ -0,0 +1,32 @@ +"""SQLite handle + tiny helpers. SQLite is the queue, cache, and checkpoint.""" + +from __future__ import annotations + +import pathlib +import sqlite3 +from datetime import datetime, timezone + +ROOT = pathlib.Path(__file__).parent +DB_PATH = ROOT / "out" / "study.db" + + +def now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def connect(path: pathlib.Path | str = DB_PATH) -> sqlite3.Connection: + path = pathlib.Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(path) + con.row_factory = sqlite3.Row + con.executescript((ROOT / "schema.sql").read_text()) + return con + + +def meta_set(con: sqlite3.Connection, key: str, value: str) -> None: + con.execute("INSERT INTO meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value)) + + +def meta_get(con: sqlite3.Connection, key: str, default: str = "") -> str: + row = con.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() + return row["value"] if row else default diff --git a/studies/ci-recurrence/dispatch.py b/studies/ci-recurrence/dispatch.py new file mode 100644 index 0000000..3b32d85 --- /dev/null +++ b/studies/ci-recurrence/dispatch.py @@ -0,0 +1,185 @@ +"""The whole harness: one dispatch function, a content-hash cache, retry, tokens. + +Deliberately dumb. No framework, no provider abstraction beyond the two +providers actually in use, no config file. If this file starts growing a +plugin system, delete the plugin system. + + ask(con, tier, fp_a, text_a, fp_b, text_b) -> (same, confidence) + +Cache first, always. The cache key is a content hash over +(prompt_version, tier, model, text_a, text_b), so re-runs after a crash cost +nothing and a prompt revision correctly invalidates the old labels. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +import urllib.error +import urllib.request + +import db +import prompts + +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434/api/generate") +OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "qwen3:30b") +GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash") +GEMINI_KEY = os.environ.get("GEMINI_API_KEY", "") + +# USD per 1M tokens. VERIFY against current pricing before quoting a total -- +# these are the numbers the cost estimate multiplies, and stale rates make the +# dollar figure confidently wrong. Local inference is metered at zero: the M1 +# is already paid for, and electricity is not what this study is measuring. +PRICING = { + "local": (0.0, 0.0), + GEMINI_MODEL: (0.30, 2.50), +} + + +def cache_key(tier: int, model: str, a: str, b: str) -> str: + h = hashlib.sha256() + for part in (prompts.PROMPT_VERSION, str(tier), model, a, b): + h.update(part.encode("utf-8")) + h.update(b"\x1f") + return h.hexdigest()[:32] + + +def _parse(text: str) -> tuple[bool | None, float]: + """Pull the JSON verdict out of a worker reply. Models add fences and prose.""" + t = (text or "").strip() + if t.startswith("```"): + t = t.strip("`") + t = t.split("\n", 1)[1] if "\n" in t else t + i, j = t.find("{"), t.rfind("}") + if i == -1 or j == -1: + return None, 0.0 + try: + d = json.loads(t[i : j + 1]) + except json.JSONDecodeError: + return None, 0.0 + same = d.get("same") + if isinstance(same, str): + same = same.strip().lower() in ("true", "yes", "same") + if not isinstance(same, bool): + return None, 0.0 + try: + conf = float(d.get("confidence", 0.0)) + except (TypeError, ValueError): + conf = 0.0 + return same, max(0.0, min(1.0, conf)) + + +def _post(url: str, payload: dict, headers: dict, timeout: int) -> dict: + req = urllib.request.Request( + url, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", **headers} + ) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode()) + + +def _call_ollama(prompt: str) -> tuple[str, int, int]: + body = _post( + OLLAMA_URL, + { + "model": OLLAMA_MODEL, + "prompt": prompt, + "system": prompts.SYSTEM, + "stream": False, + "format": "json", + "options": {"temperature": 0, "num_predict": 64}, + }, + {}, + timeout=180, # Ollama stalls; the retry loop above handles the rest. + ) + return body.get("response", ""), body.get("prompt_eval_count", 0), body.get("eval_count", 0) + + +def _call_gemini(prompt: str) -> tuple[str, int, int]: + if not GEMINI_KEY: + raise RuntimeError("GEMINI_API_KEY not set") + url = f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent" + body = _post( + f"{url}?key={GEMINI_KEY}", + { + "systemInstruction": {"parts": [{"text": prompts.SYSTEM}]}, + "contents": [{"role": "user", "parts": [{"text": prompt}]}], + "generationConfig": {"temperature": 0, "maxOutputTokens": 64, "responseMimeType": "application/json"}, + }, + {}, + timeout=120, + ) + cand = (body.get("candidates") or [{}])[0] + text = "".join(p.get("text", "") for p in cand.get("content", {}).get("parts", [])) + u = body.get("usageMetadata", {}) + return text, u.get("promptTokenCount", 0), u.get("candidatesTokenCount", 0) + + +def ask(con, tier: int, fp_a: str, text_a: str, fp_b: str, text_b: str, repo: str = "") -> tuple[bool | None, float]: + """Cache-first worker call. tier 1 = local via Ollama, tier 2 = Gemini Flash.""" + model = "local" if tier == 1 else GEMINI_MODEL + key = cache_key(tier, model, text_a, text_b) + + row = con.execute("SELECT same, confidence FROM worker_call WHERE cache_key=?", (key,)).fetchone() + if row is not None: + return (None if row["same"] is None else bool(row["same"])), (row["confidence"] or 0.0) + + prompt = prompts.build_local(text_a, text_b) if tier == 1 else prompts.build(text_a, text_b) + caller = _call_ollama if tier == 1 else _call_gemini + + same, conf, ptok, otok, err = None, 0.0, 0, 0, None + t0 = time.time() + delay = 2.0 + for attempt in range(4): + try: + raw, ptok, otok = caller(prompt) + same, conf = _parse(raw) + if same is None: + err = f"unparseable: {raw[:120]}" + else: + err = None + break + except urllib.error.HTTPError as e: + err = f"http {e.code}" + if e.code in (429, 500, 502, 503, 504): + time.sleep(delay) + delay *= 2 + continue + break + except Exception as e: # timeouts, connection resets, Ollama stalls + err = f"{type(e).__name__}: {e}" + time.sleep(delay) + delay *= 2 + + con.execute( + """INSERT OR REPLACE INTO worker_call + (cache_key,repo,tier,model,prompt_version,fp_a,fp_b,same,confidence,rationale, + prompt_tokens,output_tokens,latency_ms,error,created_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + (key, repo, tier, model, prompts.PROMPT_VERSION, fp_a, fp_b, + None if same is None else int(same), conf, None, + ptok, otok, int((time.time() - t0) * 1000), err, db.now()), + ) + return same, conf + + +def spend(con) -> dict: + """Running token + dollar totals, by tier. Printed after every batch.""" + out: dict[str, dict] = {} + total = 0.0 + for r in con.execute( + """SELECT tier, model, SUM(prompt_tokens) p, SUM(output_tokens) o, + COUNT(*) n, SUM(error IS NOT NULL) errs + FROM worker_call GROUP BY tier, model""" + ): + pin, pout = PRICING.get(r["model"], (0.0, 0.0)) + usd = (r["p"] or 0) / 1e6 * pin + (r["o"] or 0) / 1e6 * pout + total += usd + out[f"tier{r['tier']}:{r['model']}"] = { + "calls": r["n"], "errors": r["errs"], + "prompt_tokens": r["p"] or 0, "output_tokens": r["o"] or 0, + "usd": round(usd, 4), + } + out["total_usd"] = round(total, 4) + return out diff --git a/studies/ci-recurrence/fetch.py b/studies/ci-recurrence/fetch.py new file mode 100644 index 0000000..8b4a7ad --- /dev/null +++ b/studies/ci-recurrence/fetch.py @@ -0,0 +1,258 @@ +"""Collect CI failure events from the GitHub Actions API. Read-only, disk-cached. + +Every API response is written to cache/ keyed by URL hash, so re-runs never +re-fetch and the 90-day window can be rebuilt offline. + +ONE NON-OBVIOUS CORRECTNESS POINT, and it drives the whole fetch strategy: + + You cannot find flakes by listing `status=failure` runs. When a failed run is + re-run and passes, GitHub REWRITES the run's conclusion to `success`. The + flaky run then disappears from the failure list entirely -- taking with it + exactly the fail-then-pass evidence the flake split depends on. Filtering on + failure would silently drop the flakes and leave a recurrence rate that + cannot be corrected afterwards. + + So we list ALL completed runs, then pull jobs only for runs that are either + non-success or have run_attempt > 1. A first-attempt success has no failed + jobs and is skipped, which keeps the extra cost small. + +Usage: + uv run fetch.py --repo pytorch/pytorch --since-days 90 --max-runs 5000 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +import db +from normalize import fingerprint, normalize + +API = "https://api.github.com" +CACHE = pathlib.Path(__file__).parent / "cache" +UA = "ci-recurrence-study" + + +# ------------------------------------------------------------------ HTTP layer +class Gh: + def __init__(self, token: str, cache: pathlib.Path = CACHE) -> None: + self.token = token + self.cache = cache + self.cache.mkdir(parents=True, exist_ok=True) + self.calls = 0 + self.cache_hits = 0 + + def _cache_path(self, url: str) -> pathlib.Path: + h = hashlib.sha256(url.encode()).hexdigest()[:24] + return self.cache / f"{h}.json" + + def get(self, url: str, *, allow_404: bool = False) -> tuple[dict | list | None, int]: + """Returns (payload, status). status 0 means served from disk cache.""" + cp = self._cache_path(url) + if cp.exists(): + self.cache_hits += 1 + try: + return json.loads(cp.read_text()), 0 + except json.JSONDecodeError: + cp.unlink() # corrupt cache entry; refetch + + delay = 2.0 + for attempt in range(5): + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": UA, + }, + ) + try: + with urllib.request.urlopen(req, timeout=60) as r: + body = json.loads(r.read().decode()) + self.calls += 1 + remaining = int(r.headers.get("X-RateLimit-Remaining", "1000")) + if remaining < 50: + reset = int(r.headers.get("X-RateLimit-Reset", "0")) + nap = max(0, reset - int(time.time())) + 5 + print(f" [rate limit] {remaining} left, sleeping {nap}s", file=sys.stderr) + time.sleep(nap) + cp.write_text(json.dumps(body)) + return body, 200 + except urllib.error.HTTPError as e: + if e.code == 404 and allow_404: + cp.write_text(json.dumps(None)) + return None, 404 + # 403/429 here are secondary rate limits, not policy denials. + if e.code in (403, 429): + reset = e.headers.get("X-RateLimit-Reset") + nap = max(0, int(reset) - int(time.time())) + 5 if reset else delay + print(f" [{e.code}] backing off {nap}s", file=sys.stderr) + time.sleep(min(nap, 900)) + delay *= 2 + continue + if 500 <= e.code < 600: + time.sleep(delay) + delay *= 2 + continue + raise + except (urllib.error.URLError, TimeoutError): + time.sleep(delay) + delay *= 2 + return None, -1 + + +# ------------------------------------------------------------- evidence picking +def pick_evidence(gh: Gh, repo: str, job: dict) -> tuple[str, str]: + """Return (evidence_kind, raw_message). Annotations preferred over logs.""" + job_id = job["id"] + ann, status = gh.get(f"{API}/repos/{repo}/check-runs/{job_id}/annotations", allow_404=True) + if isinstance(ann, list) and ann: + # failure annotations first; a job often carries warnings too + msgs = [a for a in ann if (a.get("annotation_level") or "").lower() == "failure"] or ann + parts = [] + for a in msgs[:5]: + loc = a.get("path") or "" + line = a.get("start_line") + title = (a.get("title") or "").strip() + body = (a.get("message") or "").strip() + parts.append(f"{loc}:{line}: {title} {body}".strip()) + return "annotation", "\n".join(parts) + + # Fallback: the failed step's name. Weak evidence, but it is evidence, and + # marking it as such is what keeps the coverage fraction honest. + for step in job.get("steps") or []: + if step.get("conclusion") == "failure": + return "step_name", f"step failed: {step.get('name')}" + return "none", "" + + +# ------------------------------------------------------------------ collection +def collect(repo: str, since_days: int, max_runs: int, token: str, workflows: set[str] | None) -> None: + con = db.connect() + gh = Gh(token) + since = (datetime.now(timezone.utc) - timedelta(days=since_days)).strftime("%Y-%m-%d") + db.meta_set(con, f"window_start:{repo}", since) + db.meta_set(con, f"since_days:{repo}", str(since_days)) + + seen_runs = 0 + page = 1 + events = 0 + while seen_runs < max_runs: + url = ( + f"{API}/repos/{repo}/actions/runs" + f"?status=completed&created=%3E%3D{since}&per_page=100&page={page}" + ) + payload, status = gh.get(url) + if not payload or not payload.get("workflow_runs"): + break + runs = payload["workflow_runs"] + con.execute( + "INSERT OR REPLACE INTO fetch_log(repo,kind,ref,status,ok,note,fetched_at) VALUES(?,?,?,?,?,?,?)", + (repo, "runs_page", str(page), status, 1, f"{len(runs)} runs", db.now()), + ) + + for run in runs: + seen_runs += 1 + if workflows and run.get("name") not in workflows: + continue + # See module docstring: a first-attempt success cannot contain a + # failed job, but a re-run success can (that is the flake). + if run.get("conclusion") == "success" and run.get("run_attempt", 1) <= 1: + continue + events += ingest_run(gh, con, repo, run) + + con.commit() + print(f" page {page}: {seen_runs} runs scanned, {events} failure events, " + f"{gh.calls} api calls, {gh.cache_hits} cached", file=sys.stderr) + page += 1 + if len(runs) < 100: + break + + db.meta_set(con, f"runs_scanned:{repo}", str(seen_runs)) + con.commit() + print(f"[{repo}] {seen_runs} runs scanned -> {events} failure events " + f"({gh.calls} api calls, {gh.cache_hits} cache hits)") + + +def ingest_run(gh: Gh, con, repo: str, run: dict) -> int: + run_id = run["id"] + # filter=all returns every ATTEMPT's jobs, which is what makes the + # fail-then-pass flake visible. + jobs_payload, status = gh.get( + f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?filter=all&per_page=100" + ) + con.execute( + "INSERT OR REPLACE INTO fetch_log(repo,kind,ref,status,ok,note,fetched_at) VALUES(?,?,?,?,?,?,?)", + (repo, "jobs", str(run_id), status, 1 if jobs_payload else 0, None, db.now()), + ) + if not jobs_payload: + return 0 + + n = 0 + for job in jobs_payload.get("jobs", []): + # Record every job outcome, not just failures: the successes are what + # make a fail-then-pass on one head SHA detectable as a flake. + con.execute( + """INSERT OR REPLACE INTO job_outcome + (repo,run_id,job_id,run_attempt,workflow,job_name,head_sha,conclusion,started_at) + VALUES(?,?,?,?,?,?,?,?,?)""", + ( + repo, run_id, job["id"], job.get("run_attempt", run.get("run_attempt", 1)), + run.get("name"), job.get("name"), run.get("head_sha", ""), + job.get("conclusion"), job.get("started_at") or run.get("created_at"), + ), + ) + if job.get("conclusion") != "failure": + continue + kind, raw = pick_evidence(gh, repo, job) + norm = normalize(raw) + fp = fingerprint(raw) + con.execute( + """INSERT OR REPLACE INTO failure_event + (event_id,repo,run_id,run_attempt,job_id,workflow,job_name,head_sha,head_branch, + trigger_event,started_at,completed_at,failed_step,evidence_kind,raw_message, + norm_message,fingerprint,fetched_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + ( + f"{repo}:{run_id}:{job['id']}", repo, run_id, + job.get("run_attempt", run.get("run_attempt", 1)), job["id"], + run.get("name"), job.get("name"), run.get("head_sha", ""), + run.get("head_branch"), run.get("event"), + job.get("started_at") or run.get("created_at"), job.get("completed_at"), + next((s.get("name") for s in job.get("steps") or [] + if s.get("conclusion") == "failure"), None), + kind, raw, norm, fp, db.now(), + ), + ) + n += 1 + return n + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--repo", required=True, help="owner/name") + p.add_argument("--since-days", type=int, default=90) + p.add_argument("--max-runs", type=int, default=100000) + p.add_argument("--workflow", action="append", default=None, + help="restrict to these workflow names (repeatable); useful for pytorch") + a = p.parse_args() + + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or "" + if not token: + print("set GITHUB_TOKEN (a classic or fine-grained PAT with public_repo)", file=sys.stderr) + return 2 + collect(a.repo, a.since_days, a.max_runs, token, set(a.workflow) if a.workflow else None) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studies/ci-recurrence/findings.md b/studies/ci-recurrence/findings.md new file mode 100644 index 0000000..4f3b72f --- /dev/null +++ b/studies/ci-recurrence/findings.md @@ -0,0 +1,96 @@ +# Findings + +## Verdict: not yet reached + +No recurrence rate has been computed, so the 30% decision rule has **not** been +evaluated. The premise is neither supported nor dead. This file gets the verdict +once `analyze.py` runs against fetched data. + +## Why no number yet + +The session this was built in cannot reach the GitHub Actions API for either +target repo. Both access paths are closed: + +| Path | Result | +|---|---| +| `curl api.github.com/repos/ankidroid/Anki-Android/actions/runs` | `403` — "GitHub access to this repository is not enabled for this session" | +| `curl .../repos/pytorch/pytorch/actions/runs` | `403`, same | +| `mcp__github__actions_list` on either target | `Access denied: not configured for this session. Allowed repositories: chrishonson/metacortex` | +| `add_repo` with API access for a third-party repo | denied by the permission classifier | + +`api.github.com` itself is reachable — `/rate_limit` returns 200 with a +15000/hr budget — so this is per-repository authorization, not a network or +egress block. + +The one repository this session *can* reach, `chrishonson/Metacortex`, has 24 +workflow runs in its entire history and **zero failures** (18 success, 6 +skipped, all from two Claude Code workflows). There is no failure corpus in it +to fingerprint, so it cannot stand in even as a smoke test of the real numbers. + +Per the brief's own instruction — *"if you find yourself three files deep in +infrastructure without a fingerprint computed, stop and tell me"* — collection +stopped here rather than continuing to build the Tier 1 loop on top of an +unmeasured Tier 0. + +## What is built and verified + +Everything up to the API call, tested with known-answer fixtures +(`uv run test_pipeline.py`, 20/20 passing): + +- **Tier 0 normalizer** — verified to collapse the same bug across differing + paths, line numbers, and float values; verified to keep `test_nn.py` distinct + from `test_optim.py`, and `exit code 1` distinct from `exit code 137`; + verified idempotent on its own output. +- **Chronological recurrence walk** — verified against a hand-built sequence + with a known rate, including that no-evidence events are excluded from the + numerator/denominator but still counted in log coverage. +- **Flake split** — both shapes verified (later attempt of the same run passed; + later run on the identical head SHA passed). +- **Ingest path** — verified end-to-end against canned API payloads: annotation + evidence preferred, step-name fallback when a job has no annotations, and + every job outcome recorded including successes. + +Two real bugs were found and fixed by this testing, both of which would have +quietly corrupted the headline number: + +1. A port rule (`:\d{4,5}`) fired on **four-digit line numbers** before the + line/column rule could see them, so `test_nn.py:1234:5` and + `test_nn.py:998:11` fingerprinted differently. In a repo the size of pytorch + this shatters collapse across every large file — it would have understated + the recurrence rate badly. The rule was removed; ports fall through to the + generic int rule, which is equally stable. +2. The absolute-path rule matched the `/src/Foo.kt` tail inside a + **repo-relative** `app/src/Foo.kt`, mangling it to `app/Foo.kt`. + Repo-relative paths are exactly what Check Run annotations carry, and they + are identical on every runner — signal, not noise. Now only absolute paths + collapse. + +## To finish this + +On a machine with a personal PAT: + +```bash +export GITHUB_TOKEN= +uv run fetch.py --repo ankidroid/Anki-Android --since-days 90 +uv run analyze.py --repo ankidroid/Anki-Android --top 50 +``` + +A scopeless classic PAT is sufficient and is the right blast radius: both +targets are public, every endpoint used here is public-readable, and the token +exists only to lift the rate limit from 60/hr to 5,000/hr. `public_repo` would +grant write to every public repo you can see and buys the study nothing. + +AnkiDroid first: it is small enough to complete in one sitting and gives a real +Tier 0 collapse rate to size Tier 1 against. pytorch needs a scoped crawl +(`--workflow trunk --workflow pull --max-runs 5000`); at ~1000+ completed runs +a day, a full 90-day walk of it is several hours of API calls even with the +15000/hr budget, and that projection should be re-checked against the actual +page-1 `total_count` before committing to it. + +## Open items, in order + +1. Tier 0 collapse rate and log coverage on AnkiDroid — no tokens required. +2. Singleton count from step 1 sizes Tier 1. If singletons run to six figures, + a 20b-class model on one M1 Max will not finish overnight and the split has + to push toward Flash; that call is not worth making before the number exists. +3. Only then build the Tier 1 batch loop and the audit sampler. diff --git a/studies/ci-recurrence/normalize.py b/studies/ci-recurrence/normalize.py new file mode 100644 index 0000000..48b64b8 --- /dev/null +++ b/studies/ci-recurrence/normalize.py @@ -0,0 +1,166 @@ +"""Tier 0: deterministic normalization + fingerprinting of CI failure text. + +No model. Free, reproducible, and hashable -- which an LLM label is not. +Every failure event gets a Tier 0 fingerprint; tiers 1/2 only ever look at +what Tier 0 leaves as a singleton. + +Rules apply in the order listed. Order matters: specific patterns (UUIDs, hex, +paths) must fire before the catch-all number rule, or the number rule eats +their internals and destroys the distinction. + +Two judgment calls worth arguing about, both flagged and both measurable: + + KEEP_EXIT_CODES -- `exit code 137` (OOM-kill) and `exit code 143` (SIGTERM + / timeout) are different failures from `exit code 1` (tests failed). The + brief says strip bare ints; stripping these would merge an infra failure + into a test failure and inflate the recurrence rate. Kept by default, + switchable so we can report the collapse rate both ways. + + KEEP_BASENAME -- `/home/runner/work/pytorch/pytorch/test/test_nn.py` and + `.../test_optim.py` are different failures. The directory prefix is pure + noise (it encodes the runner, not the problem); the basename is signal. + So paths collapse to `/test_nn.py`, not ``. +""" + +from __future__ import annotations + +import hashlib +import re + +TIER0_VERSION = "t0-2026-08-14" + +KEEP_EXIT_CODES = True +KEEP_BASENAME = True +MAX_LEN = 512 + +# Sentinel that shields exit-code digits from the catch-all int rule. Letters +# only: the int rule's lookbehind refuses to match a digit preceded by a word +# character, so EXITCODE137 survives intact and is unwrapped at the end. +_SENTINEL = "EXITCODE" + +_R: list[tuple[str, re.Pattern[str], str]] = [] + + +def _rule(name: str, pattern: str, repl: str, flags: int = 0) -> None: + _R.append((name, re.compile(pattern, flags), repl)) + + +# --- 1. transport noise: escapes, log decoration, GH Actions framing ---------- +_rule("ansi", r"\x1b\[[0-9;?]*[a-zA-Z]", "") +_rule("cr", r"\r", "") +# GH Actions prefixes every raw-log line with an RFC3339 nano timestamp. +_rule("gha_ts", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s*", "", re.M) +_rule("gha_grp", r"##\[(?:group|endgroup|debug|command|section)\][^\n]*", "") +_rule("gha_err", r"##\[(?:error|warning)\]", "") + +# --- 2. identifiers that are unique per run ---------------------------------- +_rule("url", r"https?://[^\s'\"<>)\]]+", "") +_rule("uuid", r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b", "") +_rule("hexlit", r"\b0[xX][0-9a-fA-F]+\b", "") +# Bare hex blobs (git shas, content hashes): >=7 chars, must mix digit+letter +# so ordinary lowercase words don't get swallowed. +_rule("sha", r"\b(?=[0-9a-f]{7,64}\b)(?=[a-f]*[0-9])(?=[0-9]*[a-f])[0-9a-f]{7,64}\b", "") +_rule("ipv4", r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", "") +_rule("email", r"\b[\w.+-]+@[\w-]+\.[\w.]+\b", "") + +# --- 3. runner / container / temp identity ----------------------------------- +_rule("pytest_tmp", r"pytest-of-\w+/pytest-\d+(?:/[\w.-]+)?", "") +_rule("tmpdir", r"/tmp/[^\s:'\"]*", "") +_rule("runner", r"\b(?:i-[0-9a-f]{8,}|ip-\d+-\d+-\d+-\d+|runner-[\w-]+|gh-ci-[\w-]+)\b", "") +# No port rule on purpose. A `:\d{4,5}` rule fires on 4-digit LINE NUMBERS +# (`test_nn.py:1234:5`) before the linecol rule can see them, which shatters +# collapse across every large file in the repo. Ports fall through to the int +# rule and become `:`, which is equally stable -- `` only ever bought +# readability, never a distinct fingerprint. + +# --- 4. time, duration, size -------------------------------------------------- +_rule("iso", r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", "") +_rule("date", r"\b\d{4}-\d{2}-\d{2}\b", "") +_rule("clock", r"\b\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\b", "