feat: v0.3.2 — full-fidelity archive export/import (the migration path)

protoagent.plugin.yaml

@@ -3,7 +3,7 @@
 # (tests/test_packaging.py enforces it).
 id: learning_wiki
 name: Learning Wiki
-version: 0.3.1
+version: 0.3.2
 description: >-
   An adaptive learning wiki: the agent maintains a persistent, interlinked wiki
   of concept pages (Karpathy's LLM-wiki pattern) PLUS a learner ledger — per-concept

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "learning-wiki"
-version = "0.3.1"
+version = "0.3.2"
 description = "Adaptive learning-wiki plugin for protoAgent: LLM-maintained wiki + learner ledger + FSRS spaced review."
 requires-python = ">=3.11"
 

store.py

@@ -470,6 +470,214 @@ def export_markdown(self, out_dir: str | Path) -> int:
             n += 1
         return n
 
+    # ── full-fidelity archive (migration between instances) ────────────────
+
+    ARCHIVE_FORMAT = "learning-wiki-archive"
+    ARCHIVE_VERSION = 1
+
+    def export_archive(self, path: str | Path) -> dict:
+        """Write EVERYTHING — pages, revisions, links, ledger, FSRS card state,
+        review log — as one versioned, id-free JSON archive keyed by slug, so an
+        import can rebuild it on any instance. This is the migration format;
+        ``export_markdown`` stays the human-readable reading copy."""
+        import json
+
+        out = Path(path).expanduser()
+        out.parent.mkdir(parents=True, exist_ok=True)
+        with self._lock:
+            pages = [dict(r) for r in self._conn.execute("SELECT * FROM pages ORDER BY slug")]
+            by_id = {p["id"]: p["slug"] for p in pages}
+            doc_pages = []
+            for pg in pages:
+                pid = pg["id"]
+                concept = self._conn.execute("SELECT * FROM concepts WHERE page_id = ?", (pid,)).fetchone()
+                revisions = [
+                    dict(r)
+                    for r in self._conn.execute(
+                        "SELECT content_md, change_summary, source_kind, source_ref, created_at "
+                        "FROM revisions WHERE page_id = ? ORDER BY id",
+                        (pid,),
+                    )
+                ]
+                cards = []
+                for c in self._conn.execute("SELECT * FROM cards WHERE page_id = ? ORDER BY id", (pid,)):
+                    reviews = [
+                        dict(r)
+                        for r in self._conn.execute(
+                            "SELECT rating, reviewed_at, interval_days FROM review_log WHERE card_id = ? ORDER BY id",
+                            (c["id"],),
+                        )
+                    ]
+                    card = {
+                        k: c[k]
+                        for k in (
+                            "prompt",
+                            "answer",
+                            "origin",
+                            "stability",
+                            "difficulty",
+                            "reps",
+                            "lapses",
+                            "state",
+                            "due",
+                            "last_review",
+                            "suspended",
+                            "created_at",
+                        )
+                    }
+                    card["reviews"] = reviews
+                    cards.append(card)
+                doc_pages.append(
+                    {
+                        **{
+                            k: pg[k]
+                            for k in ("slug", "title", "kind", "summary", "content_md", "created_at", "updated_at")
+                        },
+                        "concept": (
+                            {k: concept[k] for k in ("strength", "last_retrieved", "misconceptions", "evidence")}
+                            if concept
+                            else None
+                        ),
+                        "revisions": revisions,
+                        "cards": cards,
+                    }
+                )
+            links = [
+                {"from": by_id[r["from_page"]], "to": by_id[r["to_page"]], "rel": r["rel"]}
+                for r in self._conn.execute("SELECT * FROM links")
+                if r["from_page"] in by_id and r["to_page"] in by_id
+            ]
+        doc = {
+            "format": self.ARCHIVE_FORMAT,
+            "version": self.ARCHIVE_VERSION,
+            "exported_at": _now_iso(),
+            "pages": doc_pages,
+            "links": links,
+        }
+        out.write_text(json.dumps(doc, indent=1), encoding="utf-8")
+        return {
+            "path": str(out),
+            "pages": len(doc_pages),
+            "links": len(links),
+            "cards": sum(len(p["cards"]) for p in doc_pages),
+        }
+
+    def import_archive(self, path: str | Path, mode: str = "merge") -> dict:
+        """Restore an archive VERBATIM — raw inserts, never through
+        ``record_retrieval``/``grade``, so imported strength and FSRS state land
+        exactly as exported (the ledger invariant: only retrieval moves strength,
+        and an import is not retrieval). ``mode="merge"`` inserts new slugs and
+        SKIPS existing ones (reported); ``mode="replace"`` replaces any colliding
+        page (cascade wipes its old revisions/cards/logs/links) with the archive's
+        version. Other pages are never touched."""
+        import json
+
+        src = Path(path).expanduser()
+        doc = json.loads(src.read_text(encoding="utf-8"))
+        if doc.get("format") != self.ARCHIVE_FORMAT:
+            raise ValueError(f"not a learning-wiki archive: {src}")
+        if int(doc.get("version", 0)) > self.ARCHIVE_VERSION:
+            raise ValueError(f"archive version {doc.get('version')} is newer than this plugin understands")
+        if mode not in ("merge", "replace"):
+            raise ValueError("mode must be 'merge' or 'replace'")
+
+        imported, skipped = [], []
+        with self._lock, self._conn:
+            for pg in doc.get("pages", []):
+                slug = str(pg.get("slug") or "").strip()
+                if not slug:
+                    continue
+                existing = self._conn.execute("SELECT id FROM pages WHERE slug = ?", (slug,)).fetchone()
+                if existing is not None:
+                    if mode == "merge":
+                        skipped.append(slug)
+                        continue
+                    self._conn.execute("DELETE FROM pages WHERE id = ?", (existing["id"],))
+                cur = self._conn.execute(
+                    "INSERT INTO pages (slug, title, kind, summary, content_md, created_at, updated_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        slug,
+                        pg.get("title") or slug,
+                        pg.get("kind") or "concept",
+                        pg.get("summary") or "",
+                        pg.get("content_md") or "",
+                        pg.get("created_at") or _now_iso(),
+                        pg.get("updated_at") or _now_iso(),
+                    ),
+                )
+                pid = cur.lastrowid
+                c = pg.get("concept")
+                if c:
+                    self._conn.execute(
+                        "INSERT INTO concepts (page_id, strength, last_retrieved, misconceptions, evidence) "
+                        "VALUES (?, ?, ?, ?, ?)",
+                        (
+                            pid,
+                            float(c.get("strength") or 0.0),
+                            c.get("last_retrieved"),
+                            c.get("misconceptions") or "[]",
+                            c.get("evidence") or "[]",
+                        ),
+                    )
+                for rv in pg.get("revisions", []):
+                    self._conn.execute(
+                        "INSERT INTO revisions (page_id, content_md, change_summary, source_kind, source_ref, created_at) "
+                        "VALUES (?, ?, ?, ?, ?, ?)",
+                        (
+                            pid,
+                            rv.get("content_md") or "",
+                            rv.get("change_summary") or "",
+                            rv.get("source_kind") or "manual",
+                            rv.get("source_ref") or "",
+                            rv.get("created_at") or _now_iso(),
+                        ),
+                    )
+                for cd in pg.get("cards", []):
+                    ccur = self._conn.execute(
+                        "INSERT INTO cards (page_id, prompt, answer, origin, stability, difficulty, reps, "
+                        "lapses, state, due, last_review, suspended, created_at) "
+                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                        (
+                            pid,
+                            cd.get("prompt") or "",
+                            cd.get("answer") or "",
+                            cd.get("origin") or "restatement",
+                            float(cd.get("stability") or 0.0),
+                            float(cd.get("difficulty") or 0.0),
+                            int(cd.get("reps") or 0),
+                            int(cd.get("lapses") or 0),
+                            cd.get("state") or "new",
+                            cd.get("due") or _now_iso(),
+                            cd.get("last_review"),
+                            int(cd.get("suspended") or 0),
+                            cd.get("created_at") or _now_iso(),
+                        ),
+                    )
+                    for rv in cd.get("reviews", []):
+                        self._conn.execute(
+                            "INSERT INTO review_log (card_id, rating, reviewed_at, interval_days) VALUES (?, ?, ?, ?)",
+                            (
+                                ccur.lastrowid,
+                                int(rv.get("rating") or 0),
+                                rv.get("reviewed_at") or _now_iso(),
+                                float(rv.get("interval_days") or 0),
+                            ),
+                        )
+                imported.append(slug)
+            # Links resolve by slug AFTER all pages land (both endpoints must exist).
+            linked = 0
+            for ln in doc.get("links", []):
+                a = self._conn.execute("SELECT id FROM pages WHERE slug = ?", (ln.get("from"),)).fetchone()
+                b = self._conn.execute("SELECT id FROM pages WHERE slug = ?", (ln.get("to"),)).fetchone()
+                if a and b:
+                    self._conn.execute(
+                        "INSERT OR IGNORE INTO links (from_page, to_page, rel) VALUES (?, ?, ?)",
+                        (a["id"], b["id"], ln.get("rel") or "related"),
+                    )
+                    linked += 1
+        return {"imported": len(imported), "skipped": skipped, "links": linked, "mode": mode}
+
     def stats(self) -> dict:
         with self._lock:
             pages = self._conn.execute("SELECT COUNT(*) AS n FROM pages").fetchone()["n"]

tests/test_archive.py

@@ -0,0 +1,82 @@
+"""Archive export/import: full-fidelity round trip, verbatim state (an import is
+NOT retrieval — the ledger invariant), and merge/replace collision semantics."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+
+def _far_future():
+    from datetime import datetime, timezone
+
+    return datetime(2099, 1, 1, tzinfo=timezone.utc)
+
+
+from learning_wiki.store import WikiStore
+
+
+def make_store(path):
+    return WikiStore(path)
+
+
+def _populated(tmp_path):
+    s = make_store(tmp_path / "src.db")
+    s.upsert_page("attention", title="Attention", content_md="# Attention\nQ·K then ·V", summary="core op")
+    s.upsert_page("softmax", title="Softmax", content_md="# Softmax", summary="normalizer")
+    s.add_link("softmax", "attention", rel="prerequisite")
+    s.record_retrieval("attention", outcome="partial", note="Q·K vs Q·V swap")
+    s.record_retrieval("attention", outcome="success", note="corrected pairing")
+    card = s.add_card(
+        "attention", prompt="What do attention weights score?", answer="Q against K", origin="misconception"
+    )
+    s.grade_card(card["id"], rating=3)
+    return s
+
+
+def test_round_trip_is_verbatim(tmp_path):
+    src = _populated(tmp_path)
+    before_page = src.get_page("attention")
+    res = src.export_archive(tmp_path / "a.json")
+    assert res["pages"] == 2 and res["links"] == 1 and res["cards"] == 1
+
+    dst = make_store(tmp_path / "dst.db")
+    out = dst.import_archive(tmp_path / "a.json")
+    assert out["imported"] == 2 and out["skipped"] == [] and out["links"] == 1
+
+    after = dst.get_page("attention")
+    # strength/FSRS state land EXACTLY — imported, not re-earned
+    assert after["strength"] == pytest.approx(before_page["strength"])
+    assert after["content_md"] == before_page["content_md"]
+    src_card = src.due_cards(limit=10, now=_far_future())[0]
+    dst_card = dst.due_cards(limit=10, now=_far_future())[0]
+    for k in ("prompt", "stability", "difficulty", "reps", "state", "due"):
+        assert dst_card[k] == src_card[k], k
+
+
+def test_merge_skips_existing_replace_overwrites(tmp_path):
+    src = _populated(tmp_path)
+    src.export_archive(tmp_path / "a.json")
+
+    dst = make_store(tmp_path / "dst.db")
+    dst.upsert_page("attention", title="My attention", content_md="local notes", summary="mine")
+    out = dst.import_archive(tmp_path / "a.json", mode="merge")
+    assert out["skipped"] == ["attention"] and out["imported"] == 1
+    assert dst.get_page("attention")["content_md"] == "local notes"  # merge never clobbers
+
+    out = dst.import_archive(tmp_path / "a.json", mode="replace")
+    assert "attention" in [s for s in ("attention",) if out["imported"] >= 1]
+    assert dst.get_page("attention")["content_md"].startswith("# Attention")  # replaced
+
+
+def test_import_rejects_foreign_and_future_files(tmp_path):
+    dst = make_store(tmp_path / "d.db")
+    bad = tmp_path / "x.json"
+    bad.write_text(json.dumps({"format": "something-else"}))
+    with pytest.raises(ValueError):
+        dst.import_archive(bad)
+    future = tmp_path / "f.json"
+    future.write_text(json.dumps({"format": "learning-wiki-archive", "version": 99, "pages": []}))
+    with pytest.raises(ValueError):
+        dst.import_archive(future)

tests/test_tools.py

@@ -19,6 +19,7 @@
     "review_grade",
     "wiki_map",
     "wiki_export",
+    "wiki_import",
     "wiki_research",
 }
 

tools.py

@@ -9,6 +9,7 @@
 
 import json
 import logging
+from pathlib import Path
 import shutil
 import subprocess
 
@@ -222,17 +223,42 @@ def wiki_map() -> str:
             return _err(e)
 
     @tool
-    def wiki_export(out_dir: str = "") -> str:
-        """Export every wiki page as a markdown file with front-matter (title, kind, strength). Default target: <data_dir>/export. Returns the directory and file count."""
+    def wiki_export(out_dir: str = "", format: str = "markdown") -> str:
+        """Export the wiki. format="markdown" (default) writes one readable .md per
+        page (front-matter: title/kind/strength) — a reading copy. format="archive"
+        writes wiki-archive.json: EVERYTHING (pages, revisions, links, ledger,
+        FSRS card state, review log) as one versioned, slug-keyed file that
+        wiki_import can rebuild on another instance — the migration path.
+        Default target: <data_dir>/export."""
         try:
             from . import _data_dir
 
+            if format == "archive":
+                target = out_dir or str(_data_dir(cfg) / "export")
+                res = get_store().export_archive(Path(target) / "wiki-archive.json")
+                return _ok(**res)
+            if format != "markdown":
+                return _err("format must be 'markdown' or 'archive'")
             target = out_dir or str(_data_dir(cfg) / "export")
             n = get_store().export_markdown(target)
             return _ok(dir=target, files=n)
         except Exception as e:  # noqa: BLE001
             return _err(e)
 
+    @tool
+    def wiki_import(path: str, mode: str = "merge") -> str:
+        """Import a wiki-archive.json produced by wiki_export(format="archive") —
+        the migration path between instances. State lands VERBATIM (strength, FSRS
+        scheduling, review history are restored, not re-earned — an import is not
+        retrieval). mode="merge" (default) adds new pages and SKIPS slugs that
+        already exist here (reported); mode="replace" overwrites colliding pages
+        with the archive's version. Other pages are never touched."""
+        try:
+            res = get_store().import_archive(path, mode=mode)
+            return _ok(**res)
+        except Exception as e:  # noqa: BLE001
+            return _err(e)
+
     @tool
     def wiki_research(topic: str) -> str:
         """Research a topic via the rabbit-hole `rh` CLI (deep web research) and return the report markdown for filing. Requires learning_wiki.rh_enabled=true and `rh` on PATH; after reading the report, file the concepts with wiki_file (source_kind=research)."""
@@ -269,5 +295,6 @@ def wiki_research(topic: str) -> str:
         review_grade,
         wiki_map,
         wiki_export,
+        wiki_import,
         wiki_research,
     ]