Data Liberation: add portable website capture

packages/data-liberation-agent/.gitignore

@@ -10,6 +10,7 @@ node_modules/
 # must live in git.
 dist/*
 !dist/mcp-server.bundle.mjs
+!dist/capture-engine.bundle.mjs
 !dist/scripts/
 docs/plans/
 docs/superpowers/

packages/data-liberation-agent/README.md

@@ -103,9 +103,9 @@ npm run mcp
 > npm run setup:browser
 > ```
 
-It exposes **35 tools**. The ones you'll call directly for a deterministic extract → QA → import flow:
+It exposes MCP tools for deterministic capture and extract → QA → import workflows. The ones you'll call directly:
 
-`liberate_detect`, `liberate_discover`, `liberate_inspect`, `liberate_extract`, `liberate_screenshot`, `liberate_status`, `liberate_qa`, `liberate_verify`, `liberate_setup`, and `liberate_import` — plus `liberate_paths` (resolve the output directory) and `liberate_probe` / `liberate_map_apis` (browser-based diagnostics). The remaining tools drive the design/reconstruction phase and are orchestrated by the skills rather than called by hand. Full reference with parameters: [docs/mcp.md](./docs/mcp.md).
+`liberate_capture`, `liberate_detect`, `liberate_discover`, `liberate_inspect`, `liberate_extract`, `liberate_screenshot`, `liberate_status`, `liberate_qa`, `liberate_verify`, `liberate_setup`, and `liberate_import` — plus `liberate_paths` (resolve the output directory) and `liberate_probe` / `liberate_map_apis` (browser-based diagnostics). `liberate_capture` writes a canonical `artifact.json`, replayable `website/` tree, capture receipt, and diagnostics. The remaining tools drive the design/reconstruction phase and are orchestrated by the skills rather than called by hand. Full reference with parameters: [docs/mcp.md](./docs/mcp.md).
 
 ## Output
 

packages/data-liberation-agent/dist/scripts/carry-reconstruct-drive.mjs

@@ -3,7 +3,7 @@ const require = __bundleCreateRequire(import.meta.url);
 import{a as $,b as y,c as w,d as S}from"./chunk-ENWLFTGA.mjs";import"./chunk-4TKBPMF4.mjs";import{writeFileSync as g,existsSync as P,mkdirSync as O,copyFileSync as D}from"node:fs";import{join as r}from"node:path";function N(s){console.log(`pages: ${s.pages.length} (${s.pages.filter(o=>o.postType==="post").length} posts, home=${s.pages.some(o=>o.isHome)})`),s.excluded.length&&console.log(`excluded (${s.excluded.length}): ${s.excluded.map(o=>o.slug).join(", ")}`),s.skipped.length&&console.log(`skipped \u2014 no captured html (${s.skipped.length}): ${s.skipped.map(o=>`${o.postType}:${o.postName}`).join(", ")}`)}async function k(){let s=process.argv.slice(2),o=s.includes("--list"),m=s.includes("--slim"),[n,d,_="Liberated (Carry)"]=s.filter(t=>t!=="--list"&&t!=="--slim");if((!n||!o&&!m&&!d)&&(console.error(`usage:
   node scripts/run.mjs carry-reconstruct-drive <outputDir> --slim   # slim output.wxr for provisioning (run BEFORE liberate_preview)
   node scripts/run.mjs carry-reconstruct-drive <outputDir> --list   # inspect the carry page list (no handler)
-  node scripts/run.mjs carry-reconstruct-drive <outputDir> <studioSitePath> "<Theme Name>"`),process.exit(2)),m){let{dropped:t,flipped:C}=S(n);console.log(`slimmed: dropped ${t} attachment items, flipped ${C} draft->publish; full WXR preserved at output.wxr.full`);return}let b=(process.env.EXCLUDE??"").split(",").map(t=>t.trim()).filter(Boolean),c=$(n,{exclude:b});if(g(r(n,"carry-pages.json"),JSON.stringify(c.pages,null,2)),N(c),o){console.log(`wrote ${r(n,"carry-pages.json")}`);return}let{reconstructPagesCarryHandler:x}=await import("./chunk-Q5FWHB6M.mjs"),E={textResult:t=>({_data:t}),errorResult:t=>{throw new Error(`handler error: ${t}`)}},f=!/^(0|false)$/i.test(process.env.EDITABLE_ISLANDS??""),e=(await x({outputDir:n,studioSitePath:d,themeName:_,pages:c.pages,editableIslands:f},E))._data;console.log(`theme: ${e.themeSlug}  media: ${e.mediaInstalled}  mediaErrors: ${e.mediaErrors.length}  fetchErrors: ${e.fetchErrors.length}  missingMediaDownloaded: ${e.missingMediaDownloaded??0}  fontsLocalized: ${e.fontsLocalized??0}`),e.missingMediaFailed?.length&&console.log("  missingMediaFailed:",JSON.stringify(e.missingMediaFailed)),e.fontsFailed?.length&&console.log("  fontsFailed:",JSON.stringify(e.fontsFailed)),f&&console.log(`editable islands: converted=${e.islandsConverted??0}  plugin=${e.editableHtmlPluginSlug??"(none)"}`);let p=e.residualCdnAssets??0;console.log(`self-host audit: residualCdnAssets=${p}${p?`  byHost=${JSON.stringify(e.residualCdnByHost)}`:" \u2713"}`),p&&console.log("  residualCdnSamples:",JSON.stringify(e.residualCdnSamples)),e.mediaErrors.length&&console.log("  mediaErrors sample:",JSON.stringify(e.mediaErrors.slice(0,3))),e.fetchErrors.length&&console.log("  fetchErrors:",JSON.stringify(e.fetchErrors));let i=r(d,"wp-content/uploads/_carry-islands");O(i,{recursive:!0});let u=e.pages.map(t=>t.slug),l=w(i,u);l.length&&console.log(`  removed ${l.length} stale island(s): ${l.slice(0,8).join(", ")}${l.length>8?" \u2026":""}`);for(let t of e.pages)g(r(i,`${t.slug}.html`),t.postContent);let v=`<?php
+  node scripts/run.mjs carry-reconstruct-drive <outputDir> <studioSitePath> "<Theme Name>"`),process.exit(2)),m){let{dropped:t,flipped:C}=S(n);console.log(`slimmed: dropped ${t} attachment items, flipped ${C} draft->publish; full WXR preserved at output.wxr.full`);return}let b=(process.env.EXCLUDE??"").split(",").map(t=>t.trim()).filter(Boolean),c=$(n,{exclude:b});if(g(r(n,"carry-pages.json"),JSON.stringify(c.pages,null,2)),N(c),o){console.log(`wrote ${r(n,"carry-pages.json")}`);return}let{reconstructPagesCarryHandler:x}=await import("./chunk-NMSHK7HD.mjs"),E={textResult:t=>({_data:t}),errorResult:t=>{throw new Error(`handler error: ${t}`)}},f=!/^(0|false)$/i.test(process.env.EDITABLE_ISLANDS??""),e=(await x({outputDir:n,studioSitePath:d,themeName:_,pages:c.pages,editableIslands:f},E))._data;console.log(`theme: ${e.themeSlug}  media: ${e.mediaInstalled}  mediaErrors: ${e.mediaErrors.length}  fetchErrors: ${e.fetchErrors.length}  missingMediaDownloaded: ${e.missingMediaDownloaded??0}  fontsLocalized: ${e.fontsLocalized??0}`),e.missingMediaFailed?.length&&console.log("  missingMediaFailed:",JSON.stringify(e.missingMediaFailed)),e.fontsFailed?.length&&console.log("  fontsFailed:",JSON.stringify(e.fontsFailed)),f&&console.log(`editable islands: converted=${e.islandsConverted??0}  plugin=${e.editableHtmlPluginSlug??"(none)"}`);let p=e.residualCdnAssets??0;console.log(`self-host audit: residualCdnAssets=${p}${p?`  byHost=${JSON.stringify(e.residualCdnByHost)}`:" \u2713"}`),p&&console.log("  residualCdnSamples:",JSON.stringify(e.residualCdnSamples)),e.mediaErrors.length&&console.log("  mediaErrors sample:",JSON.stringify(e.mediaErrors.slice(0,3))),e.fetchErrors.length&&console.log("  fetchErrors:",JSON.stringify(e.fetchErrors));let i=r(d,"wp-content/uploads/_carry-islands");O(i,{recursive:!0});let u=e.pages.map(t=>t.slug),l=w(i,u);l.length&&console.log(`  removed ${l.length} stale island(s): ${l.slice(0,8).join(", ")}${l.length>8?" \u2026":""}`);for(let t of e.pages)g(r(i,`${t.slug}.html`),t.postContent);let v=`<?php
 $dir = '/wordpress/wp-content/uploads/_carry-islands';
 $slugs = ${JSON.stringify(u)};
 foreach ($slugs as $slug) {

packages/data-liberation-agent/dist/scripts/carry-replica-shots.mjs

@@ -1,3 +1,3 @@
 import { createRequire as __bundleCreateRequire } from 'node:module';
 const require = __bundleCreateRequire(import.meta.url);
-import{b as y}from"./chunk-6FIZQDM2.mjs";import{a as b}from"./chunk-6FA2POBN.mjs";import"./chunk-4TKBPMF4.mjs";import{chromium as R}from"playwright";import{readFileSync as w,writeFileSync as U,mkdirSync as O,existsSync as $}from"node:fs";import{join as s}from"node:path";var[p,N,c]=process.argv.slice(2);(!p||!N||!c)&&(console.error("usage: node scripts/run.mjs carry-replica-shots <originDir> <carryBaseUrl> <replicaDir>"),process.exit(2));var v=JSON.parse(w(s(p,"screenshots/manifest.json"),"utf8")),j=v.entries||v,d=null,S=s(p,"carry-pages.json");if($(S)){let r=JSON.parse(w(S,"utf8"));d=new Set(r.map(i=>i.htmlSlug).filter(i=>!!i))}var T=new Map,P=s(p,"redirect-map.json");if($(P)){let r=JSON.parse(w(P,"utf8"));for(let i of r)T.set(i.from.replace(/\/$/,""),i.to)}var k=r=>T.get(r.replace(/\/$/,""))??r;O(s(c,"desktop"),{recursive:!0});O(s(c,"mobile"),{recursive:!0});async function A(r){await r.evaluate(async()=>{let l=Math.min(document.body.scrollHeight,window.innerHeight*2);for(let t=0;t<l;t+=600)window.scrollTo(0,t),await new Promise(f=>setTimeout(f,60));window.scrollTo(0,0);let o=Array.from(document.images).filter(t=>t.getBoundingClientRect().top<window.innerHeight*2),m=Promise.all(o.map(t=>t.complete?Promise.resolve():t.decode().catch(()=>{})));await Promise.race([m,new Promise(t=>setTimeout(t,4e3))])});let i=await r.evaluate(()=>!!document.querySelector(".lib-carry-mobile-frame"));await r.waitForTimeout(i?2500:400)}var M=Math.max(1,Math.min(12,Number(process.argv[5]??process.env.CONCURRENCY??6)||6)),F="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",E=[{vp:{width:1440,height:900},scale:.7,dir:"desktop",mobile:!1},{vp:{width:390,height:844},scale:1,dir:"mobile",mobile:!0}];async function x(){let r=N.replace(/\/$/,""),i=await R.launch(),l=Object.entries(j).filter(([,e])=>e.slug&&(!d||d.has(e.slug))),o=new Map;for(let[e,a]of l)o.set(e,{slug:a.slug});let m=[];for(let[e,a]of l)for(let u of E)m.push({url:e,slug:a.slug,...u});let t=0;await b(m,M,async e=>{let a=new URL(e.url).pathname,u=await i.newContext(e.mobile?{viewport:e.vp,deviceScaleFactor:e.scale,isMobile:!0,hasTouch:!0,userAgent:F}:{viewport:e.vp,deviceScaleFactor:e.scale}),n=await u.newPage();await y(n);try{for(let h=1;;h++)try{await n.goto(r+k(a),{waitUntil:"load",timeout:45e3});break}catch(C){if(h>=3)throw C;await n.waitForTimeout(1500*h)}await A(n);let g=`${e.dir}/${e.slug}.png`;await n.screenshot({path:s(c,g),fullPage:!1}),o.get(e.url)[e.dir]=g}catch(g){t++,console.warn(`  fail ${e.dir} ${a}: ${g.message.slice(0,60)}`)}await n.close(),await u.close()});let f={version:1,entries:Object.fromEntries(o)};U(s(c,"manifest.json"),JSON.stringify(f,null,2)),console.log(`captured ${o.size} urls (${t} viewport failures, concurrency ${M}) -> ${c}`),await i.close()}x().catch(r=>{console.error(r),process.exit(1)});
+import{b as y}from"./chunk-6FIZQDM2.mjs";import{a as b}from"./chunk-6FA2POBN.mjs";import"./chunk-4TKBPMF4.mjs";import{chromium as C}from"playwright";import{readFileSync as f,writeFileSync as R,mkdirSync as $,existsSync as O}from"node:fs";import{join as n}from"node:path";var[u,N,m]=process.argv.slice(2);(!u||!N||!m)&&(console.error("usage: node scripts/run.mjs carry-replica-shots <originDir> <carryBaseUrl> <replicaDir>"),process.exit(2));var v=JSON.parse(f(n(u,"screenshots/manifest.json"),"utf8")),j=v.entries||v,w=null,S=n(u,"carry-pages.json");if(O(S)){let t=JSON.parse(f(S,"utf8"));w=new Set(t.map(i=>i.htmlSlug).filter(i=>!!i))}var T=new Map,P=n(u,"redirect-map.json");if(O(P)){let t=JSON.parse(f(P,"utf8"));for(let i of t)T.set(i.from.replace(/\/$/,""),i.to)}var k=t=>T.get(t.replace(/\/$/,""))??t;$(n(m,"desktop"),{recursive:!0});$(n(m,"mobile"),{recursive:!0});async function A(t){await t.evaluate(async()=>{let s=Math.min(document.body.scrollHeight,window.innerHeight*2);for(let r=0;r<s;r+=600)window.scrollTo(0,r),await new Promise(h=>setTimeout(h,60));window.scrollTo(0,0);let l=Array.from(document.images).filter(r=>r.getBoundingClientRect().top<window.innerHeight*2),g=Promise.all(l.map(r=>r.complete?Promise.resolve():r.decode().catch(()=>{})));await Promise.race([g,new Promise(r=>setTimeout(r,4e3))])});let i=await t.evaluate(()=>!!document.querySelector(".lib-carry-mobile-frame"));await t.waitForTimeout(i?2500:400)}var M=Math.max(1,Math.min(12,Number(process.argv[5]??process.env.CONCURRENCY??6)||6)),E="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",F=[{vp:{width:1440,height:900},scale:.7,dir:"desktop",mobile:!1},{vp:{width:390,height:844},scale:1,dir:"mobile",mobile:!0}];function x(t){try{let s=f(n(u,"screenshots/mobile",`${t}.png`)).readUInt32BE(16);if(s>=240&&s<=1024)return s}catch{}return 390}async function B(){let t=N.replace(/\/$/,""),i=await C.launch(),s=Object.entries(j).filter(([,e])=>e.slug&&(!w||w.has(e.slug))),l=new Map;for(let[e,a]of s)l.set(e,{slug:a.slug});let g=[];for(let[e,a]of s)for(let c of F){let o=c.mobile?{...c.vp,width:x(a.slug)}:c.vp;g.push({url:e,slug:a.slug,...c,vp:o})}let r=0;await b(g,M,async e=>{let a=new URL(e.url).pathname,c=await i.newContext(e.mobile?{viewport:e.vp,deviceScaleFactor:e.scale,isMobile:!0,hasTouch:!0,userAgent:E}:{viewport:e.vp,deviceScaleFactor:e.scale}),o=await c.newPage();await y(o);try{for(let d=1;;d++)try{await o.goto(t+k(a),{waitUntil:"load",timeout:45e3});break}catch(U){if(d>=3)throw U;await o.waitForTimeout(1500*d)}await A(o);let p=`${e.dir}/${e.slug}.png`;await o.screenshot({path:n(m,p),fullPage:!0}),l.get(e.url)[e.dir]=p}catch(p){r++,console.warn(`  fail ${e.dir} ${a}: ${p.message.slice(0,60)}`)}await o.close(),await c.close()});let h={version:1,entries:Object.fromEntries(l)};R(n(m,"manifest.json"),JSON.stringify(h,null,2)),console.log(`captured ${l.size} urls (${r} viewport failures, concurrency ${M}) -> ${m}`),await i.close()}B().catch(t=>{console.error(t),process.exit(1)});