From ed8fb607d8b45cdae3f27ac67c0c4fe4640fe950 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 18 Sep 2026 03:44:44 +0900 Subject: [PATCH 1/2] feat(seed): take created_at/updated_at from git instead of the clock With ids fixed, a second dump of unchanged data still rewrote every page in 10 of 12 collections. The only difference was the two timestamps: the database is rebuilt for each dump, so they recorded when the dump ran, not when the record changed. brands and gpus were the only stable collections because their schemas do not expose them. One `git log` pass over data/ maps each record to its first and last commit time. Outside a git checkout the map is empty and the model defaults stand. Refs #1 --- app/seed.py | 87 ++++++++++++++++++++++++++----- tests/unit/test_git_timestamps.py | 66 +++++++++++++++++++++++ 2 files changed, 139 insertions(+), 14 deletions(-) create mode 100644 tests/unit/test_git_timestamps.py diff --git a/app/seed.py b/app/seed.py index 0d7e6ca..795cd8b 100644 --- a/app/seed.py +++ b/app/seed.py @@ -17,7 +17,8 @@ import hashlib import json -from datetime import date +import subprocess +from datetime import date, datetime from pathlib import Path from typing import Any @@ -39,6 +40,43 @@ DATA_DIR = get_data_root() +NUL = "\x00" # git log record separator, so a commit line cannot look like a path + + +def _git_timestamps(data_dir: Path) -> dict[str, tuple[datetime, datetime]]: + """Map each record path to (first commit, last commit) times. + + `created_at`/`updated_at` used to be stamped with "now" at seed time, so + every dump rewrote every page with a timestamp that only said when the dump + ran. Git already knows when a record appeared and when it last changed, and + those answers do not move between runs. + + One `git log` pass over the whole data tree; an empty map (no git, shallow + clone) leaves the model defaults in place. + """ + try: + log = subprocess.run( + # --relative keeps paths relative to data_dir, matching __path. + ["git", "log", "--reverse", "--no-renames", "--relative", + "--format=%x00%cI", "--name-only", "--diff-filter=AM", "--", "."], + cwd=data_dir, check=True, capture_output=True, text=True, encoding="utf-8", + ).stdout + except (OSError, subprocess.CalledProcessError): + return {} + + stamps: dict[str, tuple[datetime, datetime]] = {} + when: datetime | None = None + for line in log.splitlines(): + if line.startswith(NUL): + when = datetime.fromisoformat(line[1:]) + continue + if when is None or not line.endswith(".json"): + continue + first, _ = stamps.get(line, (when, when)) + stamps[line] = (first, when) + return stamps + + def _load_dir(subdir: Path) -> list[dict[str, Any]]: if not subdir.exists(): return [] @@ -52,6 +90,7 @@ def _load_dir(subdir: Path) -> list[dict[str, Any]]: for key, value in list(record.items()): if key.endswith("_date") and isinstance(value, str): record[key] = date.fromisoformat(value) + record["__path"] = path.relative_to(subdir.parent).as_posix() items.append(record) return items @@ -82,6 +121,26 @@ def _with_id(obj: Any, taken: set[int]) -> Any: return obj +def _stamp(obj: Any, path: str | None, stamps: dict[str, tuple[datetime, datetime]]) -> Any: + """Replace the seed-time timestamps with the record's git history.""" + times = stamps.get(path or "") + if times: + obj.created_at, obj.updated_at = times + return obj + + +def _row( + model: Any, + record: dict[str, Any], + taken: set[int], + stamps: dict[str, tuple[datetime, datetime]], + **fks: Any, +) -> Any: + """Build one row: resolved FKs, stable id, git-derived timestamps.""" + path = record.pop("__path", None) + return _stamp(_with_id(model(**fks, **record), taken), path, stamps) + + def _existing_slugs(session: Session, model: type[SQLModel]) -> set[str]: rows = session.exec(select(model)).all() return {row.slug for row in rows} # type: ignore[attr-defined] # all data models have slug @@ -91,6 +150,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]: """Idempotently insert seed data. Returns counts of newly inserted rows.""" # Ids assigned in this run; only used to break hash collisions. taken: set[int] = set() + stamps = _git_timestamps(data_dir) counts = { "brands": 0, "socs": 0, @@ -114,7 +174,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]: # `categories` lives in the JSON for browsing/validation only — the Brand # table model does not (yet) carry it, so drop before construction. record.pop("categories", None) - session.add(_with_id(Brand(**record), taken)) + session.add(_row(Brand, record, taken, stamps)) counts["brands"] += 1 session.commit() @@ -131,7 +191,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]: raise ValueError( f"SoC '{record['slug']}' references unknown brand '{manufacturer}'" ) - session.add(_with_id(SoC(manufacturer_id=manufacturer_id, **record), taken)) + session.add(_row(SoC, record, taken, stamps, manufacturer_id=manufacturer_id)) counts["socs"] += 1 session.commit() @@ -154,8 +214,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]: raise ValueError( f"Smartphone '{record['slug']}' references unknown SoC '{soc_slug}'" ) - phone = Smartphone(brand_id=brand_id, soc_id=soc_id, **record) - session.add(_with_id(phone, taken)) + session.add(_row(Smartphone, record, taken, stamps, brand_id=brand_id, soc_id=soc_id)) counts["smartphones"] += 1 session.commit() @@ -178,8 +237,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N f"{subdir.rstrip('s').title()} '{record['slug']}' " f"references unknown SoC '{soc_slug}'" ) - device = model(brand_id=brand_id, soc_id=soc_id, **record) - session.add(_with_id(device, taken)) + session.add(_row(model, record, taken, stamps, brand_id=brand_id, soc_id=soc_id)) counts[count_key] += 1 session.commit() @@ -198,7 +256,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N raise ValueError( f"GPU '{record['slug']}' references unknown brand '{manufacturer}'" ) - session.add(_with_id(DiscreteGPU(manufacturer_id=manufacturer_id, **record), taken)) + session.add(_row(DiscreteGPU, record, taken, stamps, manufacturer_id=manufacturer_id)) counts["gpus"] += 1 session.commit() @@ -213,7 +271,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N raise ValueError( f"CPU '{record['slug']}' references unknown brand '{manufacturer}'" ) - session.add(_with_id(CPU(manufacturer_id=manufacturer_id, **record), taken)) + session.add(_row(CPU, record, taken, stamps, manufacturer_id=manufacturer_id)) counts["cpus"] += 1 session.commit() @@ -242,8 +300,9 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N raise ValueError( f"Laptop '{record['slug']}' references unknown GPU '{gpu_slug}'" ) - laptop = Laptop(brand_id=brand_id, cpu_id=cpu_id, gpu_id=gpu_id, **record) - session.add(_with_id(laptop, taken)) + session.add( + _row(Laptop, record, taken, stamps, brand_id=brand_id, cpu_id=cpu_id, gpu_id=gpu_id) + ) counts["laptops"] += 1 session.commit() @@ -258,7 +317,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N raise ValueError( f"Monitor '{record['slug']}' references unknown brand '{brand_slug}'" ) - session.add(_with_id(Monitor(brand_id=brand_id, **record), taken)) + session.add(_row(Monitor, record, taken, stamps, brand_id=brand_id)) counts["monitors"] += 1 session.commit() @@ -267,7 +326,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N for record in _load_dir(data_dir / "software"): if record["slug"] in software_slugs: continue - session.add(_with_id(Software(**record), taken)) + session.add(_row(Software, record, taken, stamps)) counts["software"] += 1 session.commit() @@ -276,7 +335,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N for record in _load_dir(data_dir / "website"): if record["slug"] in website_slugs: continue - session.add(_with_id(Website(**record), taken)) + session.add(_row(Website, record, taken, stamps)) counts["websites"] += 1 session.commit() diff --git a/tests/unit/test_git_timestamps.py b/tests/unit/test_git_timestamps.py new file mode 100644 index 0000000..40b0249 --- /dev/null +++ b/tests/unit/test_git_timestamps.py @@ -0,0 +1,66 @@ +"""Timestamps come from git, so a re-dump of unchanged data is byte-identical.""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +from app.seed import _git_timestamps + + +def _git(repo: Path, *args: str, when: str | None = None) -> None: + env = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@example.com", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@example.com", + "PATH": os.environ["PATH"], + } + if when: + # _git_timestamps reads committer dates, so both must be pinned. + env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = when + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, env=env) + + +def _repo(tmp_path: Path) -> Path: + data = tmp_path / "data" + (data / "cpu").mkdir(parents=True) + _git(tmp_path, "init", "-q") + return data + + +def _commit(repo: Path, rel: str, payload: dict, when: str) -> None: + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + _git(repo.parent, "add", "-A") + _git(repo.parent, "commit", "-q", "-m", rel, when=when) + + +def test_created_and_updated_track_first_and_last_commit(tmp_path): + data = _repo(tmp_path) + _commit(data, "cpu/a.json", {"slug": "a"}, "2026-01-02T03:04:05+00:00") + _commit(data, "cpu/b.json", {"slug": "b"}, "2026-02-02T03:04:05+00:00") + _commit(data, "cpu/a.json", {"slug": "a", "cores": 8}, "2026-03-02T03:04:05+00:00") + + stamps = _git_timestamps(data) + created_a, updated_a = stamps["cpu/a.json"] + created_b, updated_b = stamps["cpu/b.json"] + + assert created_a < updated_a # edited later + assert created_b == updated_b # written once + assert created_a < created_b # a came first + + +def test_repeated_reads_agree(tmp_path): + data = _repo(tmp_path) + _commit(data, "cpu/a.json", {"slug": "a"}, "2026-01-02T03:04:05+00:00") + assert _git_timestamps(data) == _git_timestamps(data) + + +def test_outside_a_git_repo_returns_nothing(tmp_path): + plain = tmp_path / "plain" + (plain / "cpu").mkdir(parents=True) + assert _git_timestamps(plain) == {} From bde6b737e537cf6587efc7277b28bdc7f7cd4df1 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 18 Sep 2026 03:49:08 +0900 Subject: [PATCH 2/2] fix(seed): only stamp models that have the timestamp fields Brand and DiscreteGPU have no created_at/updated_at, so assigning them raised ValueError on every brand insert. Refs #1 --- app/seed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/seed.py b/app/seed.py index 795cd8b..6ffff7a 100644 --- a/app/seed.py +++ b/app/seed.py @@ -124,7 +124,8 @@ def _with_id(obj: Any, taken: set[int]) -> Any: def _stamp(obj: Any, path: str | None, stamps: dict[str, tuple[datetime, datetime]]) -> Any: """Replace the seed-time timestamps with the record's git history.""" times = stamps.get(path or "") - if times: + # brand and gpu carry no timestamps at all — skip rather than invent fields. + if times and hasattr(obj, "created_at"): obj.created_at, obj.updated_at = times return obj