From c2fb08624a96c99039f0de4abd0383c176242a34 Mon Sep 17 00:00:00 2001 From: Josh Mabry Date: Wed, 19 Aug 2026 19:55:00 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20v0.3.2=20=E2=80=94=20full-fidelity=20ar?= =?UTF-8?q?chive=20export/import=20(the=20migration=20path)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wiki_export grows format="archive": one versioned, slug-keyed, id-free JSON carrying EVERYTHING — pages, revisions, links, ledger (strength / misconceptions / evidence), FSRS card state, review log. New wiki_import restores it VERBATIM (raw inserts, never through record_retrieval — an import is not retrieval, so strength and scheduling land exactly as exported; the ledger invariant holds). mode="merge" adds new slugs and skips collisions (reported); mode="replace" overwrites colliding pages; other pages never touched. Foreign/future-version files refused. Proven live: exported a 10-page/28-link/4-card corpus through the tool on one instance, imported on a fresh second instance — strength 0.15 and all four due cards arrived intact, not reset. 100 host-free tests. Co-Authored-By: Claude Fable 5 --- protoagent.plugin.yaml | 2 +- pyproject.toml | 2 +- store.py | 208 +++++++++++++++++++++++++++++++++++++++++ tests/test_archive.py | 82 ++++++++++++++++ tests/test_tools.py | 1 + tools.py | 31 +++++- uv.lock | 8 ++ 7 files changed, 330 insertions(+), 4 deletions(-) create mode 100644 tests/test_archive.py create mode 100644 uv.lock diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index ff0e6d0..9925dad 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -3,7 +3,7 @@ # (tests/test_packaging.py enforces it). id: learning_wiki name: Learning Wiki -version: 0.3.1 +version: 0.3.2 description: >- An adaptive learning wiki: the agent maintains a persistent, interlinked wiki of concept pages (Karpathy's LLM-wiki pattern) PLUS a learner ledger — per-concept diff --git a/pyproject.toml b/pyproject.toml index 8bf4c23..acc9795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "learning-wiki" -version = "0.3.1" +version = "0.3.2" description = "Adaptive learning-wiki plugin for protoAgent: LLM-maintained wiki + learner ledger + FSRS spaced review." requires-python = ">=3.11" diff --git a/store.py b/store.py index 80e1fe2..92240f8 100644 --- a/store.py +++ b/store.py @@ -470,6 +470,214 @@ def export_markdown(self, out_dir: str | Path) -> int: n += 1 return n + # ── full-fidelity archive (migration between instances) ──────────────── + + ARCHIVE_FORMAT = "learning-wiki-archive" + ARCHIVE_VERSION = 1 + + def export_archive(self, path: str | Path) -> dict: + """Write EVERYTHING — pages, revisions, links, ledger, FSRS card state, + review log — as one versioned, id-free JSON archive keyed by slug, so an + import can rebuild it on any instance. This is the migration format; + ``export_markdown`` stays the human-readable reading copy.""" + import json + + out = Path(path).expanduser() + out.parent.mkdir(parents=True, exist_ok=True) + with self._lock: + pages = [dict(r) for r in self._conn.execute("SELECT * FROM pages ORDER BY slug")] + by_id = {p["id"]: p["slug"] for p in pages} + doc_pages = [] + for pg in pages: + pid = pg["id"] + concept = self._conn.execute("SELECT * FROM concepts WHERE page_id = ?", (pid,)).fetchone() + revisions = [ + dict(r) + for r in self._conn.execute( + "SELECT content_md, change_summary, source_kind, source_ref, created_at " + "FROM revisions WHERE page_id = ? ORDER BY id", + (pid,), + ) + ] + cards = [] + for c in self._conn.execute("SELECT * FROM cards WHERE page_id = ? ORDER BY id", (pid,)): + reviews = [ + dict(r) + for r in self._conn.execute( + "SELECT rating, reviewed_at, interval_days FROM review_log WHERE card_id = ? ORDER BY id", + (c["id"],), + ) + ] + card = { + k: c[k] + for k in ( + "prompt", + "answer", + "origin", + "stability", + "difficulty", + "reps", + "lapses", + "state", + "due", + "last_review", + "suspended", + "created_at", + ) + } + card["reviews"] = reviews + cards.append(card) + doc_pages.append( + { + **{ + k: pg[k] + for k in ("slug", "title", "kind", "summary", "content_md", "created_at", "updated_at") + }, + "concept": ( + {k: concept[k] for k in ("strength", "last_retrieved", "misconceptions", "evidence")} + if concept + else None + ), + "revisions": revisions, + "cards": cards, + } + ) + links = [ + {"from": by_id[r["from_page"]], "to": by_id[r["to_page"]], "rel": r["rel"]} + for r in self._conn.execute("SELECT * FROM links") + if r["from_page"] in by_id and r["to_page"] in by_id + ] + doc = { + "format": self.ARCHIVE_FORMAT, + "version": self.ARCHIVE_VERSION, + "exported_at": _now_iso(), + "pages": doc_pages, + "links": links, + } + out.write_text(json.dumps(doc, indent=1), encoding="utf-8") + return { + "path": str(out), + "pages": len(doc_pages), + "links": len(links), + "cards": sum(len(p["cards"]) for p in doc_pages), + } + + def import_archive(self, path: str | Path, mode: str = "merge") -> dict: + """Restore an archive VERBATIM — raw inserts, never through + ``record_retrieval``/``grade``, so imported strength and FSRS state land + exactly as exported (the ledger invariant: only retrieval moves strength, + and an import is not retrieval). ``mode="merge"`` inserts new slugs and + SKIPS existing ones (reported); ``mode="replace"`` replaces any colliding + page (cascade wipes its old revisions/cards/logs/links) with the archive's + version. Other pages are never touched.""" + import json + + src = Path(path).expanduser() + doc = json.loads(src.read_text(encoding="utf-8")) + if doc.get("format") != self.ARCHIVE_FORMAT: + raise ValueError(f"not a learning-wiki archive: {src}") + if int(doc.get("version", 0)) > self.ARCHIVE_VERSION: + raise ValueError(f"archive version {doc.get('version')} is newer than this plugin understands") + if mode not in ("merge", "replace"): + raise ValueError("mode must be 'merge' or 'replace'") + + imported, skipped = [], [] + with self._lock, self._conn: + for pg in doc.get("pages", []): + slug = str(pg.get("slug") or "").strip() + if not slug: + continue + existing = self._conn.execute("SELECT id FROM pages WHERE slug = ?", (slug,)).fetchone() + if existing is not None: + if mode == "merge": + skipped.append(slug) + continue + self._conn.execute("DELETE FROM pages WHERE id = ?", (existing["id"],)) + cur = self._conn.execute( + "INSERT INTO pages (slug, title, kind, summary, content_md, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + slug, + pg.get("title") or slug, + pg.get("kind") or "concept", + pg.get("summary") or "", + pg.get("content_md") or "", + pg.get("created_at") or _now_iso(), + pg.get("updated_at") or _now_iso(), + ), + ) + pid = cur.lastrowid + c = pg.get("concept") + if c: + self._conn.execute( + "INSERT INTO concepts (page_id, strength, last_retrieved, misconceptions, evidence) " + "VALUES (?, ?, ?, ?, ?)", + ( + pid, + float(c.get("strength") or 0.0), + c.get("last_retrieved"), + c.get("misconceptions") or "[]", + c.get("evidence") or "[]", + ), + ) + for rv in pg.get("revisions", []): + self._conn.execute( + "INSERT INTO revisions (page_id, content_md, change_summary, source_kind, source_ref, created_at) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + pid, + rv.get("content_md") or "", + rv.get("change_summary") or "", + rv.get("source_kind") or "manual", + rv.get("source_ref") or "", + rv.get("created_at") or _now_iso(), + ), + ) + for cd in pg.get("cards", []): + ccur = self._conn.execute( + "INSERT INTO cards (page_id, prompt, answer, origin, stability, difficulty, reps, " + "lapses, state, due, last_review, suspended, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + pid, + cd.get("prompt") or "", + cd.get("answer") or "", + cd.get("origin") or "restatement", + float(cd.get("stability") or 0.0), + float(cd.get("difficulty") or 0.0), + int(cd.get("reps") or 0), + int(cd.get("lapses") or 0), + cd.get("state") or "new", + cd.get("due") or _now_iso(), + cd.get("last_review"), + int(cd.get("suspended") or 0), + cd.get("created_at") or _now_iso(), + ), + ) + for rv in cd.get("reviews", []): + self._conn.execute( + "INSERT INTO review_log (card_id, rating, reviewed_at, interval_days) VALUES (?, ?, ?, ?)", + ( + ccur.lastrowid, + int(rv.get("rating") or 0), + rv.get("reviewed_at") or _now_iso(), + float(rv.get("interval_days") or 0), + ), + ) + imported.append(slug) + # Links resolve by slug AFTER all pages land (both endpoints must exist). + linked = 0 + for ln in doc.get("links", []): + a = self._conn.execute("SELECT id FROM pages WHERE slug = ?", (ln.get("from"),)).fetchone() + b = self._conn.execute("SELECT id FROM pages WHERE slug = ?", (ln.get("to"),)).fetchone() + if a and b: + self._conn.execute( + "INSERT OR IGNORE INTO links (from_page, to_page, rel) VALUES (?, ?, ?)", + (a["id"], b["id"], ln.get("rel") or "related"), + ) + linked += 1 + return {"imported": len(imported), "skipped": skipped, "links": linked, "mode": mode} + def stats(self) -> dict: with self._lock: pages = self._conn.execute("SELECT COUNT(*) AS n FROM pages").fetchone()["n"] diff --git a/tests/test_archive.py b/tests/test_archive.py new file mode 100644 index 0000000..1fd4d70 --- /dev/null +++ b/tests/test_archive.py @@ -0,0 +1,82 @@ +"""Archive export/import: full-fidelity round trip, verbatim state (an import is +NOT retrieval — the ledger invariant), and merge/replace collision semantics.""" + +from __future__ import annotations + +import json + +import pytest + + +def _far_future(): + from datetime import datetime, timezone + + return datetime(2099, 1, 1, tzinfo=timezone.utc) + + +from learning_wiki.store import WikiStore + + +def make_store(path): + return WikiStore(path) + + +def _populated(tmp_path): + s = make_store(tmp_path / "src.db") + s.upsert_page("attention", title="Attention", content_md="# Attention\nQ·K then ·V", summary="core op") + s.upsert_page("softmax", title="Softmax", content_md="# Softmax", summary="normalizer") + s.add_link("softmax", "attention", rel="prerequisite") + s.record_retrieval("attention", outcome="partial", note="Q·K vs Q·V swap") + s.record_retrieval("attention", outcome="success", note="corrected pairing") + card = s.add_card( + "attention", prompt="What do attention weights score?", answer="Q against K", origin="misconception" + ) + s.grade_card(card["id"], rating=3) + return s + + +def test_round_trip_is_verbatim(tmp_path): + src = _populated(tmp_path) + before_page = src.get_page("attention") + res = src.export_archive(tmp_path / "a.json") + assert res["pages"] == 2 and res["links"] == 1 and res["cards"] == 1 + + dst = make_store(tmp_path / "dst.db") + out = dst.import_archive(tmp_path / "a.json") + assert out["imported"] == 2 and out["skipped"] == [] and out["links"] == 1 + + after = dst.get_page("attention") + # strength/FSRS state land EXACTLY — imported, not re-earned + assert after["strength"] == pytest.approx(before_page["strength"]) + assert after["content_md"] == before_page["content_md"] + src_card = src.due_cards(limit=10, now=_far_future())[0] + dst_card = dst.due_cards(limit=10, now=_far_future())[0] + for k in ("prompt", "stability", "difficulty", "reps", "state", "due"): + assert dst_card[k] == src_card[k], k + + +def test_merge_skips_existing_replace_overwrites(tmp_path): + src = _populated(tmp_path) + src.export_archive(tmp_path / "a.json") + + dst = make_store(tmp_path / "dst.db") + dst.upsert_page("attention", title="My attention", content_md="local notes", summary="mine") + out = dst.import_archive(tmp_path / "a.json", mode="merge") + assert out["skipped"] == ["attention"] and out["imported"] == 1 + assert dst.get_page("attention")["content_md"] == "local notes" # merge never clobbers + + out = dst.import_archive(tmp_path / "a.json", mode="replace") + assert "attention" in [s for s in ("attention",) if out["imported"] >= 1] + assert dst.get_page("attention")["content_md"].startswith("# Attention") # replaced + + +def test_import_rejects_foreign_and_future_files(tmp_path): + dst = make_store(tmp_path / "d.db") + bad = tmp_path / "x.json" + bad.write_text(json.dumps({"format": "something-else"})) + with pytest.raises(ValueError): + dst.import_archive(bad) + future = tmp_path / "f.json" + future.write_text(json.dumps({"format": "learning-wiki-archive", "version": 99, "pages": []})) + with pytest.raises(ValueError): + dst.import_archive(future) diff --git a/tests/test_tools.py b/tests/test_tools.py index e5480f9..402e87c 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -19,6 +19,7 @@ "review_grade", "wiki_map", "wiki_export", + "wiki_import", "wiki_research", } diff --git a/tools.py b/tools.py index a8ee261..8bc4ca0 100644 --- a/tools.py +++ b/tools.py @@ -9,6 +9,7 @@ import json import logging +from pathlib import Path import shutil import subprocess @@ -222,17 +223,42 @@ def wiki_map() -> str: return _err(e) @tool - def wiki_export(out_dir: str = "") -> str: - """Export every wiki page as a markdown file with front-matter (title, kind, strength). Default target: /export. Returns the directory and file count.""" + def wiki_export(out_dir: str = "", format: str = "markdown") -> str: + """Export the wiki. format="markdown" (default) writes one readable .md per + page (front-matter: title/kind/strength) — a reading copy. format="archive" + writes wiki-archive.json: EVERYTHING (pages, revisions, links, ledger, + FSRS card state, review log) as one versioned, slug-keyed file that + wiki_import can rebuild on another instance — the migration path. + Default target: /export.""" try: from . import _data_dir + if format == "archive": + target = out_dir or str(_data_dir(cfg) / "export") + res = get_store().export_archive(Path(target) / "wiki-archive.json") + return _ok(**res) + if format != "markdown": + return _err("format must be 'markdown' or 'archive'") target = out_dir or str(_data_dir(cfg) / "export") n = get_store().export_markdown(target) return _ok(dir=target, files=n) except Exception as e: # noqa: BLE001 return _err(e) + @tool + def wiki_import(path: str, mode: str = "merge") -> str: + """Import a wiki-archive.json produced by wiki_export(format="archive") — + the migration path between instances. State lands VERBATIM (strength, FSRS + scheduling, review history are restored, not re-earned — an import is not + retrieval). mode="merge" (default) adds new pages and SKIPS slugs that + already exist here (reported); mode="replace" overwrites colliding pages + with the archive's version. Other pages are never touched.""" + try: + res = get_store().import_archive(path, mode=mode) + return _ok(**res) + except Exception as e: # noqa: BLE001 + return _err(e) + @tool def wiki_research(topic: str) -> str: """Research a topic via the rabbit-hole `rh` CLI (deep web research) and return the report markdown for filing. Requires learning_wiki.rh_enabled=true and `rh` on PATH; after reading the report, file the concepts with wiki_file (source_kind=research).""" @@ -269,5 +295,6 @@ def wiki_research(topic: str) -> str: review_grade, wiki_map, wiki_export, + wiki_import, wiki_research, ] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..42f5b90 --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "learning-wiki" +version = "0.3.2" +source = { virtual = "." }