diff --git a/.github/workflows/forecast-archive.yml b/.github/workflows/forecast-archive.yml index 3b9908e..3dae395 100644 --- a/.github/workflows/forecast-archive.yml +++ b/.github/workflows/forecast-archive.yml @@ -48,5 +48,11 @@ jobs: - name: Validate the archive run: python3 forecasts/archive.py --check - - name: Scorecard is current + - name: Data vintage store is consistent + run: python3 data/fetch.py --check + + - name: No outturn is available but unrecorded + run: python3 forecasts/ingest_outturns.py --check + + - name: Scorecard and page are current run: python3 forecasts/score.py --check diff --git a/forecasts/README.md b/forecasts/README.md index 5702e5d..a7ab42e 100644 --- a/forecasts/README.md +++ b/forecasts/README.md @@ -14,7 +14,7 @@ directory tests the forecasts, in real time, where nobody does. | path | mutable? | what it is | |------|----------|------------| | `rounds//.json` | **no** | A forecast as it stood on a date. Append-only. | -| `outturns.json` | yes | Realised data, versioned by vintage. | +| `outturns.json` | append-only | Realised data, versioned by vintage. Written by `ingest_outturns.py` from `data/`. | | `scorecard.json` | generated | Scores. Written by `score.py`, never by hand. | | `index.html` | partly generated | The public page. Blocks between `` markers are generated. | @@ -28,11 +28,13 @@ python3 forecasts/archive.py # the git timestamp is what makes the forecast falsifiable. git add forecasts/rounds && git commit -m "Archive forecast round " -# 3. When outturns land, add them to outturns.json and rescore. +# 3. When outturns land, pull them from the vintage store and rescore. +python3 data/fetch.py +python3 forecasts/ingest_outturns.py python3 forecasts/score.py ``` -Both scripts take `--check` and are wired into +All three scripts take `--check` and are wired into `.github/workflows/forecast-archive.yml`, which also fails any pull request that modifies, renames or deletes an already-archived round. diff --git a/forecasts/index.html b/forecasts/index.html index 3d9f5b0..20ac1fe 100644 --- a/forecasts/index.html +++ b/forecasts/index.html @@ -78,22 +78,43 @@

01 — where this stands -

The record is one round old.

+

One round in, one result.

- 1 round archived; 0 forecast period(s) scored. - The earliest period that can be scored is 2026Q2, once the ONS first quarterly estimate lands. + 1 round archived; 1 forecast period scored. + The next result due is 2026Q2 (UK real GDP, y/y), which lands when the ONS publishes it.

- There is no headline accuracy figure on this page yet, because one - cannot honestly be computed from zero observations. That is the point of - publishing the record from its first day rather than from the day it - starts to look good: the empty state is part of the evidence, and it is - checkable that nothing was quietly discarded before the record began. + There is no headline accuracy figure on this page, and there will not be + one until the scored count is large enough to carry it. That is the point + of publishing the record from its first day rather than from the day it + starts to look good: the early, near-empty state is part of the evidence, + and it is checkable that nothing was quietly discarded before the record + began.

+ + +
+ + + + + + + + + + + + + + +
Scored forecasts. Error is forecast minus outturn; the band column is the narrowest credible band the outturn fell inside.
PeriodVariableForecastOutturnErrorBandRound
2026Q2UK CPI, y/y2.68%2.80%-0.12pp68%2026-07-21
+
+

What already exists is a pseudo-out-of-sample evaluation — the boe-svar model re-estimated at 49 expanding-window origins and compared @@ -123,13 +144,26 @@

What was claimed, and when.

boe-svar 2026Q1 13 - 0 + 1 2026-07-21/boe-svar.json
+

+ The first round is a retroactive archive, and its lead time was + thin. The 2026-07-21 round was not archived by this + machinery — it was copied out of + papers/boe-svar/figures/current_forecast.json after the fact, + so its provenance rests on that file's git history rather than on the + archive. The commit landed 2026-07-21 14:54 BST; the ONS published the + 2026Q2 CPI figure on the following morning's release. So the forecast did + precede the outturn, by about sixteen hours, and the model was conditioned + only through 2026Q1 — but a sixteen-hour lead is not the same evidence as + a round archived weeks ahead, and it is recorded here rather than left for + a reader to discover. Every round from here is archived before the fact. +

Rounds are append-only. Files under forecasts/rounds/ can be added but never edited or deleted, and a diff --git a/forecasts/ingest_outturns.py b/forecasts/ingest_outturns.py new file mode 100644 index 0000000..b859483 --- /dev/null +++ b/forecasts/ingest_outturns.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Turn the data vintage store into scoreable outturns. + +Run: python3 forecasts/ingest_outturns.py # append newly available outturns + python3 forecasts/ingest_outturns.py --check # exit 1 if outturns.json is behind + +Maps the variables the models forecast onto the ONS series in ``data/``, converts +them to the forecasts' units (year-on-year percent), and appends them to +``outturns.json`` tagged with the data vintage they were read from. + +Appends, never edits. When the ONS revises a quarter, a later run adds a second +observation for that period under the new vintage; ``score.py`` scores against +the newest but every earlier score stays reproducible. Only periods that some +archived round actually forecast are ingested — this file is evidence for the +track record, not a general data mirror. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HERE = Path(__file__).resolve().parent +ROUNDS = HERE / "rounds" +OUTTURNS = HERE / "outturns.json" +DATA = ROOT / "data" / "latest" + + +def yoy_from_levels(observations: list[dict]) -> dict[str, float]: + """Year-on-year percent change from a quarterly level series.""" + by_period = {o["period"]: o["value"] for o in observations} + out = {} + for period, value in by_period.items(): + year, quarter = int(period[:4]), period[4:] + prior = by_period.get(f"{year - 1}{quarter}") + if prior: + out[period] = (value / prior - 1) * 100 + return out + + +def passthrough(observations: list[dict]) -> dict[str, float]: + """The series is already in the forecasts' units.""" + return {o["period"]: o["value"] for o in observations} + + +# Forecast variable -> the series it is scored against, and how to convert it. +MAPPING = { + "gdp": {"series": "uk_gdp_cvm", "transform": yoy_from_levels}, + "cpi": {"series": "uk_cpi_yoy", "transform": passthrough}, +} + + +def forecast_periods() -> dict[str, set[str]]: + """Periods each variable has been forecast for, across all archived rounds.""" + wanted: dict[str, set[str]] = {} + for path in sorted(ROUNDS.glob("*/*.json")): + data = json.loads(path.read_text()) + for period, block in data["forecast"].items(): + for variable in block: + wanted.setdefault(variable, set()).add(period) + return wanted + + +def available() -> list[dict]: + """Every outturn that some archived round forecast and the data now covers.""" + wanted = forecast_periods() + rows = [] + + for variable, periods in sorted(wanted.items()): + spec = MAPPING.get(variable) + if spec is None: + print(f"skip {variable}: no series mapped in MAPPING", file=sys.stderr) + continue + + path = DATA / f"{spec['series']}.json" + if not path.exists(): + print(f"skip {variable}: {path.relative_to(ROOT)} not fetched", file=sys.stderr) + continue + + doc = json.loads(path.read_text()) + values = spec["transform"](doc["observations"]) + for period in sorted(periods & set(values)): + rows.append( + { + "period": period, + "variable": variable, + "value": round(values[period], 6), + "vintage": doc["vintage"], + "series": spec["series"], + "release_updated": doc.get("release_updated"), + } + ) + return rows + + +def merge(existing: list[dict], fresh: list[dict]) -> tuple[list[dict], list[dict]]: + """Append observations not already recorded for the same (period, variable, vintage).""" + seen = {(o["period"], o["variable"], o.get("vintage")) for o in existing} + added = [r for r in fresh if (r["period"], r["variable"], r["vintage"]) not in seen] + return existing + added, added + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--check", action="store_true", help="exit 1 if outturns.json is behind") + args = ap.parse_args() + + doc = json.loads(OUTTURNS.read_text()) + merged, added = merge(doc.get("observations", []), available()) + + if args.check: + if added: + for row in added: + print(f"FAIL missing outturn {row['variable']} {row['period']} " + f"({row['value']:.2f}, vintage {row['vintage']})", file=sys.stderr) + print("run forecasts/ingest_outturns.py", file=sys.stderr) + return 1 + print(f"OK — {len(merged)} outturn(s) recorded, none pending") + return 0 + + if not added: + print("no new outturns available") + return 0 + + doc["observations"] = sorted(merged, key=lambda o: (o["period"], o["variable"], o["vintage"])) + OUTTURNS.write_text(json.dumps(doc, indent=2) + "\n") + + for row in added: + print(f"added {row['variable']:>4} {row['period']} = {row['value']:.2f} " + f"(vintage {row['vintage']})") + print("\nnow run forecasts/score.py") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/forecasts/outturns.json b/forecasts/outturns.json index 244a545..d5f4f80 100644 --- a/forecasts/outturns.json +++ b/forecasts/outturns.json @@ -25,5 +25,14 @@ "typical_release_lag": "~3 weeks after month end" } }, - "observations": [] + "observations": [ + { + "period": "2026Q2", + "variable": "cpi", + "value": 2.8, + "vintage": "2026-07-25", + "series": "uk_cpi_yoy", + "release_updated": "2026-07-21T23:00:00.000Z" + } + ] } diff --git a/forecasts/score.py b/forecasts/score.py index ac332cb..7e201a3 100644 --- a/forecasts/score.py +++ b/forecasts/score.py @@ -120,14 +120,15 @@ def build() -> dict: scored = [score_round(r, outturns) for r in rounds] total_scored = sum(s["periods_scored"] for s in scored) - # The first period any round forecast that is still unscored — i.e. the - # earliest date this track record can start meaning something. + # The earliest period still waiting on any variable. Checked per variable, + # not per period: CPI lands roughly six weeks before the quarterly GDP + # estimate, so a period with one of the two in is not finished. pending = sorted( { period for r in rounds - for period in r["forecast"] - if not any((period, v) in outturns for v in r["forecast"][period]) + for period, block in r["forecast"].items() + if not all((period, v) in outturns for v in block) } ) @@ -136,6 +137,21 @@ def build() -> dict: "rounds": len(rounds), "periods_scored": total_scored, "next_period_to_score": pending[0] if pending else None, + "pending_detail": ( + { + "period": pending[0], + "variables": sorted( + { + v + for r in rounds + for v in r["forecast"].get(pending[0], {}) + if (pending[0], v) not in outturns + } + ), + } + if pending + else None + ), "status": ( "accumulating — no forecast period has an outturn yet" if total_scored == 0 @@ -147,27 +163,29 @@ def build() -> dict: # ---------------------------------------------------------------- page +VARIABLE_LABELS = {"gdp": "UK real GDP, y/y", "cpi": "UK CPI, y/y"} + def esc(s: str) -> str: return str(s).replace("&", "&").replace("<", "<").replace(">", ">") def render_status(card: dict) -> str: """The headline paragraph. Deliberately refuses to state an accuracy figure - while the scored count is zero — see the module docstring.""" + while the scored count is small — see the module docstring.""" rounds = card["rounds"] scored = card["periods_scored"] - plural = "" if rounds == 1 else "s" lines = [ "

", - f" {rounds} round{plural} archived; {scored} forecast " - "period(s) scored.", + f" {rounds} round{'' if rounds == 1 else 's'} archived; " + f"{scored} forecast period{'' if scored == 1 else 's'} scored.", ] - if card["next_period_to_score"]: + pending = card.get("pending_detail") + if pending: + who = ", ".join(f"{VARIABLE_LABELS.get(v, v)}" for v in pending["variables"]) lines.append( - " The earliest period that can be scored is " - f"{esc(card['next_period_to_score'])}, once the " - "ONS first quarterly estimate lands." + f" The next result due is {esc(pending['period'])} " + f"({esc(who)}), which lands when the ONS publishes it." ) else: lines.append(" Every archived period now has an outturn.") @@ -193,13 +211,61 @@ def render_rounds(card: dict) -> str: return "\n".join(rows) +def render_results(card: dict) -> str: + """The scored entries. Absent entirely until something has been scored.""" + rows = [ + (detail, entry) + for detail in card["detail"] + for entry in detail["entries"] + ] + if not rows: + return " " + + body = [] + for detail, e in sorted(rows, key=lambda r: (r[1]["period"], r[1]["variable"])): + band = "68%" if e["in_68"] else ("90%" if e["in_90"] else "outside 90%") + body.append( + " \n" + f" {esc(e['period'])}\n" + f" {esc(VARIABLE_LABELS.get(e['variable'], e['variable']))}\n" + f" {e['forecast']:.2f}%\n" + f" {e['outturn']:.2f}%\n" + f" {e['error']:+.2f}pp\n" + f" {esc(band)}\n" + f" {esc(detail['round_id'])}\n" + " " + ) + + return "\n".join( + [ + "

", + " ", + " ", + " " + "" + "" + "", + " ", + *body, + " ", + "
Scored forecasts. Error is forecast minus outturn; " + "the band column is the narrowest credible band the outturn fell inside.
PeriodVariableForecastOutturnErrorBandRound
", + "
", + ] + ) + + def render_page(html: str, card: dict) -> str: """Inject the generated blocks between their markers. Same contract as validation/figures/make_charts.py: the page is committed, the numbers inside it are generated, and --check fails if they drift apart. """ - blocks = {"scorecard-status": render_status(card), "scorecard-rounds": render_rounds(card)} + blocks = { + "scorecard-status": render_status(card), + "scorecard-results": render_results(card), + "scorecard-rounds": render_rounds(card), + } for marker, body in blocks.items(): pattern = re.compile( rf"(\n).*?()", re.DOTALL diff --git a/forecasts/scorecard.json b/forecasts/scorecard.json index 5ced444..9e29954 100644 --- a/forecasts/scorecard.json +++ b/forecasts/scorecard.json @@ -1,9 +1,15 @@ { "_comment": "Generated by forecasts/score.py \u2014 do not edit by hand.", "rounds": 1, - "periods_scored": 0, + "periods_scored": 1, "next_period_to_score": "2026Q2", - "status": "accumulating \u2014 no forecast period has an outturn yet", + "pending_detail": { + "period": "2026Q2", + "variables": [ + "gdp" + ] + }, + "status": "1 forecast period(s) scored", "detail": [ { "round_id": "2026-07-21", @@ -12,9 +18,29 @@ "archived_utc": "2026-07-25T01:08:19Z", "path": "forecasts/rounds/2026-07-21/boe-svar.json", "periods_forecast": 13, - "periods_scored": 0, - "entries": [], - "by_variable": {} + "periods_scored": 1, + "entries": [ + { + "period": "2026Q2", + "variable": "cpi", + "forecast": 2.6785302332498873, + "outturn": 2.8, + "outturn_vintage": "2026-07-25", + "error": -0.12146976675011256, + "abs_error": 0.12146976675011256, + "in_68": true, + "in_90": true + } + ], + "by_variable": { + "cpi": { + "n": 1, + "mae": 0.12146976675011256, + "bias": -0.12146976675011256, + "coverage_68": 1.0, + "coverage_90": 1.0 + } + } } ] }