feat: scaffold satellite repositories from the game-catalog layout

README.md

@@ -28,6 +28,35 @@ python -m pytest -q
 
 Adding a repository or endpoint is one entry in `machine/repos.json`.
 
+When the set of problems changes, the check comments on the issue and
+@-mentions `notify` from that file — editing an issue body notifies nobody.
+It stays quiet when the same failure recurs (the fingerprint is
+workflow + result, not the run link) and never pings for an all-clear.
+
+## Satellite repositories
+
+Categories that do not belong in TechAPI live in their own repository (games:
+[game-catalog](https://github.com/GetTechAPI/game-catalog)). The split
+criterion is identity, not size — software and websites are tech data and
+stay in TechAPI.
+
+`machine/new_satellite.py` writes a new one from the layout game-catalog
+proved out: a streaming validator, a site build that publishes
+`summary.json` + `history.json` (never a listing of every record), CI, and
+licences.
+
+```bash
+python -m machine.new_satellite --repo game-catalog --category game     --title "Game catalog" --plural games --date-field release_date     --range rating:0:5 --range metacritic:0:100 --out ../game-catalog
+```
+
+It only writes files. Creating the repository changes the organisation, so
+that is left to a person; the remaining steps are printed at the end,
+including adding `main` to the Pages environment's deployment branches —
+without it every deploy fails and leaves no log.
+
+The tests generate a repository and run *its* test suite and validator, so a
+template change that breaks generated repos fails here.
+
 ## Branching
 
 `develop` is the default branch; `main` is the released state. Pull requests

machine/new_satellite.py

@@ -0,0 +1,153 @@
+"""Scaffold a satellite data repository from the proven game-catalog layout.
+
+Writes the files only. Creating the GitHub repository is left to a person —
+it is an org-level change — and the remaining one-off steps are printed at
+the end, including the one that silently broke game-catalog's first deploys.
+
+Example:
+    python -m machine.new_satellite --repo game-catalog --category game \
+        --title "Game catalog" --plural games --date-field release_date \
+        --range rating:0:5 --range metacritic:0:100 --out ../game-catalog
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+TEMPLATE = Path(__file__).with_name("satellite_template")
+PLACEHOLDER = re.compile(r"\{\{(\w+)\}\}")
+# Template files stored under a name git or pytest would otherwise act on.
+RENAMES = {"gitignore": ".gitignore", "tests_test_validate.py": "tests/test_validate.py"}
+
+
+def parse_range(text: str) -> tuple[str, float, float]:
+    field, low, high = text.split(":")
+    return field, float(low), float(high)
+
+
+def render(text: str, values: dict[str, str]) -> str:
+    def replace(match: re.Match[str]) -> str:
+        key = match.group(1)
+        if key not in values:
+            raise KeyError(f"template placeholder {{{{{key}}}}} has no value")
+        return values[key]
+    return PLACEHOLDER.sub(replace, text)
+
+
+def values_from(args: argparse.Namespace) -> dict[str, str]:
+    ranges = {field: (low, high) for field, low, high in args.range}
+    return {
+        "repo": args.repo,
+        "category": args.category,
+        "title": args.title,
+        "plural": args.plural,
+        "description": args.description,
+        "date_fields": repr(tuple(args.date_field)),
+        "ranges": repr({k: (int(a) if a.is_integer() else a, int(b) if b.is_integer() else b)
+                        for k, (a, b) in ranges.items()}),
+    }
+
+
+def scaffold(out: Path, values: dict[str, str]) -> list[Path]:
+    if out.exists() and any(out.iterdir()):
+        raise SystemExit(f"{out} is not empty; refusing to overwrite")
+    written = []
+    for source in sorted(TEMPLATE.rglob("*")):
+        if source.is_dir():
+            continue
+        rel = source.relative_to(TEMPLATE).as_posix()
+        target = out / RENAMES.get(rel, rel)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if source.suffix in {".py", ".md", ".toml", ".yml", ".html", ""} or source.name == "gitignore":
+            target.write_text(render(source.read_text(encoding="utf-8"), values),
+                              encoding="utf-8", newline="\n")
+        else:
+            target.write_bytes(source.read_bytes())
+        written.append(target)
+    (out / "data" / values["category"]).mkdir(parents=True, exist_ok=True)
+    (out / "README.md").write_text(readme(values), encoding="utf-8", newline="\n")
+    written.append(out / "README.md")
+    return written
+
+
+def readme(v: dict[str, str]) -> str:
+    return f"""# {v['repo']}
+
+[![validate-data](https://github.com/GetTechAPI/{v['repo']}/actions/workflows/validate-data.yml/badge.svg)](https://github.com/GetTechAPI/{v['repo']}/actions/workflows/validate-data.yml)
+
+{v['description']}
+
+Code is MIT; the records under `data/` are CC BY-SA 4.0 ([DATA_LICENSE.md](DATA_LICENSE.md)).
+
+## Layout
+
+```
+data/{v['category']}/<bucket>/<slug>.json   # bucket = first two slug characters
+app/validate.py                             # schema / slug / date / range checks
+site/build.py                               # summary.json + history.json
+```
+
+A record needs `slug`, `name`, `source_urls` and `verified`.
+
+## Self-check
+
+```bash
+python -m app.validate
+python -m pytest -q
+```
+
+## Site
+
+`python site/build.py` writes `summary.json` (`{{"count": N}}`) and
+`history.json` (one point per data commit). The TechAPI homepage reads these
+to count this catalog; there is deliberately no listing of every record.
+
+## Branching (git-flow)
+
+`develop` is the default branch; `main` is the released state and deploys the
+site. Pull requests target `develop`; a release is a PR from `develop` to `main`.
+"""
+
+
+def checklist(v: dict[str, str]) -> str:
+    repo = f"GetTechAPI/{v['repo']}"
+    return f"""
+Next steps (not automated — each changes the org):
+
+  1. gh repo create {repo} --public
+  2. push the scaffold to develop, then develop:main
+  3. gh api -X POST repos/{repo}/pages -f build_type=workflow
+  4. gh api -X POST repos/{repo}/environments/github-pages/deployment-branch-policies -f name=main
+     (skip this and every deploy fails with no log — game-catalog, 2026-09-17)
+  5. add {{"name": "{repo}", "branches": ["develop", "main"]}} to TechMachine machine/repos.json
+     and the summary.json URL to its endpoints
+  6. add {{ key: "{v['plural']}", label: "{v['plural']}", base: "https://gettechapi.github.io/{v['repo']}/" }}
+     to SATELLITES in TechAPI site/src/scripts/techapi.js
+"""
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--repo", required=True, help="repository name, e.g. game-catalog")
+    parser.add_argument("--category", required=True, help="data directory, e.g. game")
+    parser.add_argument("--title", required=True, help='page title, e.g. "Game catalog"')
+    parser.add_argument("--plural", required=True, help="count label, e.g. games")
+    parser.add_argument("--description", default="Split out of TechAPI.")
+    parser.add_argument("--date-field", action="append", default=[], help="YYYY-MM-DD field, repeatable")
+    parser.add_argument("--range", action="append", default=[], type=parse_range,
+                        help="field:low:high, repeatable")
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args(argv)
+
+    values = values_from(args)
+    written = scaffold(args.out, values)
+    print(f"wrote {len(written)} files to {args.out}")
+    print(checklist(values))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

machine/satellite_template/.github/workflows/deploy-pages.yml

@@ -0,0 +1,42 @@
+name: deploy-pages
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # history.json replays every data commit
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build summary + history
+        run: python site/build.py
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

machine/satellite_template/.github/workflows/validate-data.yml

@@ -0,0 +1,22 @@
+name: validate-data
+
+on:
+  pull_request:
+  push:
+    branches: [develop, main]
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90  # 962k files; measure a real run before trimming
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Validate {{title}}
+        run: python -m app.validate
+      - name: Tests
+        run: |
+          pip install pytest
+          python -m pytest -q

machine/satellite_template/DATA_LICENSE.md

@@ -0,0 +1,8 @@
+# Data license
+
+JSON records under `data/` are licensed under
+[Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/).
+
+Attribute **"Data from GetTechAPI / {{repo}}"** and share alike.
+
+Validator, site, and workflow code remain MIT (see `LICENSE`).

machine/satellite_template/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GTA Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

machine/satellite_template/app/validate.py

@@ -0,0 +1,106 @@
+"""Validate the {{title}}.
+
+Generated by TechMachine's satellite template. Records are checked one at a
+time rather than loaded into a list first, so the check scales to catalogs of
+any size (game-catalog validates ~1M records in about 90 seconds on CI).
+
+Run with: python -m app.validate
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parent.parent
+DATA = ROOT / "data" / "{{category}}"
+
+REQUIRED = {"slug", "name", "source_urls", "verified"}
+DATE_FIELDS = {{date_fields}}
+RANGES = {{ranges}}  # field -> (low, high), inclusive
+SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
+DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+MAX_ERRORS = 200  # a wall of identical errors helps nobody
+
+
+def _check(rel: str, rec: dict[str, Any], errors: list[str]) -> None:
+    missing = REQUIRED - rec.keys()
+    if missing:
+        errors.append(f"{rel}: missing required field(s) {sorted(missing)}")
+
+    slug = rec.get("slug")
+    if isinstance(slug, str) and not SLUG_RE.match(slug):
+        errors.append(f"{rel}: slug '{slug}' is not kebab-case")
+
+    urls = rec.get("source_urls")
+    if not isinstance(urls, list) or not urls:
+        errors.append(f"{rel}: source_urls must be a non-empty list")
+    elif not all(isinstance(u, str) and u.startswith("http") for u in urls):
+        errors.append(f"{rel}: every source_url must be an http(s) string")
+
+    for field in DATE_FIELDS:
+        value = rec.get(field)
+        if value is not None and not (isinstance(value, str) and DATE_RE.match(value)):
+            errors.append(f"{rel}: {field} '{value}' is not YYYY-MM-DD")
+
+    for field, (low, high) in RANGES.items():
+        value = rec.get(field)
+        if value is None or isinstance(value, bool):
+            continue
+        if not isinstance(value, (int, float)) or not low <= value <= high:
+            errors.append(f"{rel}: {field} {value!r} outside {low}-{high}")
+
+
+def validate(data_dir: Path = DATA) -> list[str]:
+    errors: list[str] = []
+    seen: dict[str, str] = {}
+    count = 0
+
+    for path in sorted(data_dir.rglob("*.json")):
+        rel = path.relative_to(data_dir.parent).as_posix()
+        count += 1
+        try:
+            rec = json.loads(path.read_text(encoding="utf-8-sig"))
+        except json.JSONDecodeError as exc:
+            errors.append(f"{rel}: invalid JSON ({exc})")
+            continue
+        if not isinstance(rec, dict):
+            errors.append(f"{rel}: top level must be an object")
+            continue
+
+        _check(rel, rec, errors)
+
+        slug = rec.get("slug")
+        if isinstance(slug, str):
+            if slug in seen:
+                errors.append(f"{rel}: duplicate slug '{slug}' (first seen in {seen[slug]})")
+            else:
+                seen[slug] = rel
+            if path.stem != slug:
+                errors.append(f"{rel}: filename does not match slug '{slug}'")
+
+        if len(errors) >= MAX_ERRORS:
+            errors.append(f"... stopped after {MAX_ERRORS} errors")
+            break
+
+    if count == 0:
+        errors.append(f"no {{category}} records found under {data_dir}")
+    return errors
+
+
+def main() -> int:
+    errors = validate()
+    for error in errors:
+        print(error)
+    if errors:
+        print(f"FAIL: {len(errors)} problem(s)")
+        return 1
+    print("OK")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())