diff --git a/apps/web/app/api/pages/[pageId]/export/route.ts b/apps/web/app/api/pages/[pageId]/export/route.ts index 0498d572..1177999a 100644 --- a/apps/web/app/api/pages/[pageId]/export/route.ts +++ b/apps/web/app/api/pages/[pageId]/export/route.ts @@ -2,7 +2,10 @@ import { getToken } from "@/lib/auth/server"; import { getServerConvexClient } from "@/lib/convex/server"; import { getFiles } from "@/lib/files/server"; import { assertStoredChecksum, type PageExportAsset } from "./page-export"; -import { iterableSource, readSource } from "@baseblocks/anydoc/sources"; +import { + iterableSource, + readSource, +} from "@baseblocks/anydoc-contracts/sources"; import { api } from "@baseblocks/backend"; import { projectBaseBlocksDocumentForPortableExport } from "@baseblocks/openeditor-contracts"; import { baseBlocksBlockRegistry } from "@baseblocks/openeditor-contracts/block-registry"; diff --git a/apps/web/components/file-viewer/anydoc-preview-client.tsx b/apps/web/components/file-viewer/anydoc-preview-client.tsx index 43075826..990e832a 100644 --- a/apps/web/components/file-viewer/anydoc-preview-client.tsx +++ b/apps/web/components/file-viewer/anydoc-preview-client.tsx @@ -1,12 +1,12 @@ "use client"; import type { PreviewFile } from "@/components/file-viewer/file-viewer"; -import { isSafeExternalUrl } from "@baseblocks/anydoc"; +import { isSafeExternalUrl } from "@baseblocks/anydoc-contracts"; import { AnyDocumentViewer, type ViewerControls, type ViewerError, -} from "@baseblocks/anydoc/react"; +} from "@baseblocks/anydoc-viewer/react"; import { Spinner } from "@baseblocks/ui/spinner"; import { useMemo } from "react"; import { UnifiedViewerControls } from "./anydoc-viewer-controls"; diff --git a/apps/web/components/file-viewer/anydoc-viewer-controls.tsx b/apps/web/components/file-viewer/anydoc-viewer-controls.tsx index 9376298e..b55d4d1b 100644 --- a/apps/web/components/file-viewer/anydoc-viewer-controls.tsx +++ b/apps/web/components/file-viewer/anydoc-viewer-controls.tsx @@ -16,7 +16,7 @@ import { TextWrapIcon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; -import type { ViewerControls } from "@baseblocks/anydoc/react"; +import type { ViewerControls } from "@baseblocks/anydoc-viewer/react"; import { Button } from "@baseblocks/ui/button"; import { Input } from "@baseblocks/ui/input"; import { cn } from "@baseblocks/ui/lib/utils"; diff --git a/apps/web/components/file-viewer/file-viewer.tsx b/apps/web/components/file-viewer/file-viewer.tsx index eff4997f..a1a352a4 100644 --- a/apps/web/components/file-viewer/file-viewer.tsx +++ b/apps/web/components/file-viewer/file-viewer.tsx @@ -22,7 +22,7 @@ import { useState, } from "react"; import { createPortal } from "react-dom"; -import { detectViewerFormat } from "@baseblocks/anydoc/react"; +import { detectViewerFormat } from "@baseblocks/anydoc-viewer/react"; const AnyDocPreview = dynamic(() => import("./anydoc-preview-client"), { ssr: false, diff --git a/apps/web/package.json b/apps/web/package.json index 8d9cf1a9..db6ae314 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -15,9 +15,10 @@ "@aws-sdk/client-s3": "^3.1084.0", "@aws-sdk/s3-presigned-post": "^3.1084.0", "@aws-sdk/s3-request-presigner": "^3.1084.0", + "@baseblocks/anydoc-contracts": "0.2.0-alpha.3", + "@baseblocks/anydoc-viewer": "0.2.0-alpha.3", "@baseblocks/backend": "workspace:*", "@baseblocks/custom-blocks": "workspace:*", - "@baseblocks/anydoc": "0.1.0-alpha.15", "@baseblocks/domain": "workspace:*", "@baseblocks/i18n": "workspace:*", "@baseblocks/openeditor-contracts": "workspace:*", @@ -29,13 +30,13 @@ "@flags-sdk/vercel": "^1.4.6", "@hugeicons/core-free-icons": "^4.2.3", "@hugeicons/react": "^1.1.9", - "@openeditor/workspace": "0.0.48", "@openeditor/core": "0.0.48", "@openeditor/custom-block": "0.0.48", "@openeditor/emoji": "0.0.48", "@openeditor/exporters": "0.0.48", "@openeditor/react": "0.0.48", "@openeditor/ui": "0.0.48", + "@openeditor/workspace": "0.0.48", "@vercel/analytics": "^2.0.1", "@vercel/sdk": "^1.28.8", "ai": "7.0.49", @@ -51,8 +52,8 @@ "next": "16.3.0", "next-intl": "4.13.6", "next-themes": "^0.4.6", - "react": "^19.2.7", - "react-dom": "^19.2.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", "react-dropzone": "^15.0.0", "server-only": "^0.0.1", "sonner": "^2.0.7", diff --git a/bun.lock b/bun.lock index 3284febd..78e0e606 100644 --- a/bun.lock +++ b/bun.lock @@ -19,7 +19,8 @@ "@aws-sdk/client-s3": "^3.1084.0", "@aws-sdk/s3-presigned-post": "^3.1084.0", "@aws-sdk/s3-request-presigner": "^3.1084.0", - "@baseblocks/anydoc": "0.1.0-alpha.15", + "@baseblocks/anydoc-contracts": "0.2.0-alpha.3", + "@baseblocks/anydoc-viewer": "0.2.0-alpha.3", "@baseblocks/backend": "workspace:*", "@baseblocks/custom-blocks": "workspace:*", "@baseblocks/domain": "workspace:*", @@ -55,8 +56,8 @@ "next": "16.3.0", "next-intl": "4.13.6", "next-themes": "^0.4.6", - "react": "^19.2.7", - "react-dom": "^19.2.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", "react-dropzone": "^15.0.0", "server-only": "^0.0.1", "sonner": "^2.0.7", @@ -82,13 +83,14 @@ "@aws-sdk/client-s3": "^3.1084.0", "@aws-sdk/s3-presigned-post": "^3.1084.0", "@aws-sdk/s3-request-presigner": "^3.1084.0", - "@baseblocks/anydoc-convex": "0.1.0-alpha.15", + "@baseblocks/anydoc-contracts": "0.2.0-alpha.3", "@baseblocks/custom-blocks": "workspace:*", "@baseblocks/domain": "workspace:*", "@baseblocks/openeditor-contracts": "workspace:*", "@convex-dev/better-auth": "^0.12.5", "@convex-dev/workflow": "^0.4.4", "@convex-dev/workpool": "^0.4.9", + "@firecrawl/anydoc": "0.1.9", "@noble/hashes": "2.0.1", "@openeditor/core": "0.0.48", "@openeditor/custom-block": "0.0.48", @@ -215,15 +217,45 @@ }, }, "overrides": { - "@convex-dev/better-auth": "0.12.5", - "@radix-ui/react-presence": "1.1.6", - "better-auth": "1.6.23", - "convex": "^1.42.1", - "defu": "6.1.7", - "picomatch": "4.0.5", - "postcss": "8.5.23", - "react": "^19.2.7", - "react-dom": "^19.2.7", + "@tiptap/core": "3.29.2", + "@tiptap/extension-blockquote": "3.29.2", + "@tiptap/extension-bold": "3.29.2", + "@tiptap/extension-bubble-menu": "3.29.2", + "@tiptap/extension-bullet-list": "3.29.2", + "@tiptap/extension-code": "3.29.2", + "@tiptap/extension-code-block": "3.29.2", + "@tiptap/extension-document": "3.29.2", + "@tiptap/extension-dropcursor": "3.29.2", + "@tiptap/extension-floating-menu": "3.29.2", + "@tiptap/extension-gapcursor": "3.29.2", + "@tiptap/extension-hard-break": "3.29.2", + "@tiptap/extension-heading": "3.29.2", + "@tiptap/extension-horizontal-rule": "3.29.2", + "@tiptap/extension-image": "3.29.2", + "@tiptap/extension-italic": "3.29.2", + "@tiptap/extension-link": "3.29.2", + "@tiptap/extension-list": "3.29.2", + "@tiptap/extension-list-item": "3.29.2", + "@tiptap/extension-list-keymap": "3.29.2", + "@tiptap/extension-ordered-list": "3.29.2", + "@tiptap/extension-paragraph": "3.29.2", + "@tiptap/extension-placeholder": "3.29.2", + "@tiptap/extension-strike": "3.29.2", + "@tiptap/extension-table": "3.29.2", + "@tiptap/extension-task-item": "3.29.2", + "@tiptap/extension-task-list": "3.29.2", + "@tiptap/extension-text": "3.29.2", + "@tiptap/extension-text-align": "3.29.2", + "@tiptap/extension-underline": "3.29.2", + "@tiptap/extension-unique-id": "3.29.2", + "@tiptap/extensions": "3.29.2", + "@tiptap/pm": "3.29.2", + "@tiptap/react": "3.29.2", + "@tiptap/starter-kit": "3.29.2", + "@tiptap/suggestion": "3.29.2", + "convex": "1.43.0", + "react": "19.2.8", + "react-dom": "19.2.8", }, "packages": { "@ai-sdk/gateway": ["@ai-sdk/gateway@4.0.38", "", { "dependencies": { "@ai-sdk/provider": "4.0.4", "@ai-sdk/provider-utils": "5.0.18", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-BMxiGGfXlmUQeDCCW/7XsxBTVPBlw4gyk63RGM1qVyTt5DrMeQGeCxkscS/EnUzCvmLCcFCl8f8Eos61PIag9g=="], @@ -288,23 +320,9 @@ "@base-ui/utils": ["@base-ui/utils@0.3.2", "", { "dependencies": { "@babel/runtime": "^7.29.2", "@floating-ui/utils": "^0.2.12", "reselect": "^5.2.0", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "@types/react": "^17 || ^18 || ^19", "react": "^17 || ^18 || ^19", "react-dom": "^17 || ^18 || ^19" }, "optionalPeers": ["@types/react"] }, "sha512-oWy1aq/I2GmYjpl4PhEAhzflF8VPGKgZeq0xAWTbfD5KBWyxcN0ZP2+WHSUm/5Z6lVMBDLReLcoXwSYoRc/zNQ=="], - "@baseblocks/anydoc": ["@baseblocks/anydoc@0.1.0-alpha.15", "", { "dependencies": { "@baseblocks/anydoc-ingestion": "0.1.0-alpha.15" }, "optionalDependencies": { "@baseblocks/anydoc-presentation-viewer": "0.1.0-alpha.15", "@baseblocks/anydoc-react-viewer": "0.1.0-alpha.15", "@baseblocks/anydoc-spreadsheet-engine": "0.1.0-alpha.15", "@baseblocks/anydoc-spreadsheet-viewer": "0.1.0-alpha.15", "@firecrawl/anydoc-wasm": "0.1.8" }, "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"] }, "sha512-M+J+7idkKfQDt3XzazEqqIrIdvzgYjvKMcHOqBI8amI8iWTwHG57lZQ9LPmrHu46FYIQvMJykOBDOk2O34trhw=="], + "@baseblocks/anydoc-contracts": ["@baseblocks/anydoc-contracts@0.2.0-alpha.3", "", {}, "sha512-q4Uw+FQRxMn32NHAB7XWJjeWZepvum/8wNDdZPx59E3U09Frsd+B1MUx/wooLv5PodcqYpkPUTrbPyBei1r2nA=="], - "@baseblocks/anydoc-contracts": ["@baseblocks/anydoc-contracts@0.1.0-alpha.15", "", {}, "sha512-uZMH4yOzcaCwMq8GAa8U7nm+DLFg1Rch1XY/vexuUeGJe/m+T3sYIO51+HewcvIT7CVzshx42dMYWWTCFHccFw=="], - - "@baseblocks/anydoc-convex": ["@baseblocks/anydoc-convex@0.1.0-alpha.15", "", { "dependencies": { "@baseblocks/anydoc-ingestion": "0.1.0-alpha.15", "@convex-dev/workpool": "^0.4.9" }, "peerDependencies": { "convex": ">=1.36.1" } }, "sha512-97h2GFYlcypx/KUpW06X0GX5zZfIYuLgcMQ8p3wBZ2HlHkZGkogR7u/Dij5kYtp2O+8HsbWrcs4uw609a4Xnaw=="], - - "@baseblocks/anydoc-ingestion": ["@baseblocks/anydoc-ingestion@0.1.0-alpha.15", "", { "dependencies": { "@baseblocks/anydoc-contracts": "0.1.0-alpha.15", "@firecrawl/anydoc": "0.1.8" } }, "sha512-7fHdFxaL/F8FLDYrKRlPbPrVFAqk7pKvSOewKgu0/+Q9GAIVIMX9cN9lebOUbLUGC/ufibCPVMbwpWqMZACmCA=="], - - "@baseblocks/anydoc-presentation-viewer": ["@baseblocks/anydoc-presentation-viewer@0.1.0-alpha.15", "", { "dependencies": { "@aiden0z/pptx-renderer": "1.2.4", "@baseblocks/anydoc-contracts": "0.1.0-alpha.15", "@baseblocks/anydoc-viewer-ui": "0.1.0-alpha.15" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" } }, "sha512-W36it1CfpCzEw0Q57Itxf3aJoZDJ+ii3gN0nuSnTvznfGPaEtvrqxNSaEF6WCz+wPdPN4DZHEqCwHKU9KjmVxg=="], - - "@baseblocks/anydoc-react-viewer": ["@baseblocks/anydoc-react-viewer@0.1.0-alpha.15", "", { "dependencies": { "@baseblocks/anydoc-contracts": "0.1.0-alpha.15", "@baseblocks/anydoc-viewer-ui": "0.1.0-alpha.15", "@tanstack/react-virtual": "^3.13.12", "@zip.js/zip.js": "2.8.31", "docx-preview": "^0.3.7", "github-slugger": "^2.0.0", "pdfjs-dist": "6.2.108", "react-markdown": "^10.1.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "rehype-slug": "^6.0.0", "remark-gfm": "^4.0.1" }, "peerDependencies": { "@baseblocks/anydoc-presentation-viewer": "0.1.0-alpha.15", "@baseblocks/anydoc-spreadsheet-viewer": "0.1.0-alpha.15", "react": ">=18.2", "react-dom": ">=18.2" }, "optionalPeers": ["@baseblocks/anydoc-presentation-viewer", "@baseblocks/anydoc-spreadsheet-viewer"] }, "sha512-Kze/WlQmh0yl9CXbkXfXksOi3HCOrWoNrMUd0/g8cTCUniVBIQ0nnrWN5r4b0BSmWt+/kLnw1sWghVKYxpsPxg=="], - - "@baseblocks/anydoc-spreadsheet-engine": ["@baseblocks/anydoc-spreadsheet-engine@0.1.0-alpha.15", "", { "dependencies": { "@baseblocks/anydoc-contracts": "0.1.0-alpha.15", "@zip.js/zip.js": "2.8.31", "saxes": "5.0.1" } }, "sha512-LBMtgTDh6r9OGwjeMdpHTTfdtBNZ4AM0jcSsnaCve8/xsRgeBl3Df5WjZnVt9Lc2HPYu+4ZuWjt3aQmtqC2xeA=="], - - "@baseblocks/anydoc-spreadsheet-viewer": ["@baseblocks/anydoc-spreadsheet-viewer@0.1.0-alpha.15", "", { "dependencies": { "@baseblocks/anydoc-contracts": "0.1.0-alpha.15", "@baseblocks/anydoc-spreadsheet-engine": "0.1.0-alpha.15", "@baseblocks/anydoc-viewer-ui": "0.1.0-alpha.15", "echarts": "6.1.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" } }, "sha512-j2tsycw0Ozh3hiiL56wN+3/mNwxxNCUwNaFwmgFmDUCvuq5mcAH2JZP92M+20ambqhm8VGkd7Q6dpt/+M2AImg=="], - - "@baseblocks/anydoc-viewer-ui": ["@baseblocks/anydoc-viewer-ui@0.1.0-alpha.15", "", { "dependencies": { "@pierre/icons": "0.7.1" }, "peerDependencies": { "react": ">=18.2", "react-dom": ">=18.2" } }, "sha512-VZJ1RPW8Y67Hc+/Cm1fxg32BbrCCQf0FBZrUl9++LdGAxLKXkuRKHimIqcugI3VbI1rkGPCDshdX1F69mn7JbQ=="], + "@baseblocks/anydoc-viewer": ["@baseblocks/anydoc-viewer@0.2.0-alpha.3", "", { "dependencies": { "@aiden0z/pptx-renderer": "1.2.4", "@baseblocks/anydoc-contracts": "0.2.0-alpha.3", "@tanstack/react-virtual": "^3.13.12", "@zip.js/zip.js": "2.8.31", "docx-preview": "^0.3.7", "echarts": "6.1.0", "github-slugger": "^2.0.0", "pdfjs-dist": "6.2.108", "react-markdown": "^10.1.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "rehype-slug": "^6.0.0", "remark-gfm": "^4.0.1", "saxes": "5.0.1" }, "peerDependencies": { "react": ">=18.2", "react-dom": ">=18.2" }, "optionalPeers": ["react", "react-dom"] }, "sha512-S585w5qamUuBVloIivhVAq3Vl/5KNUHM7pyuMP3WDLdcojFHQFaSxrfLhjbCvdDJO7ZP3zCOydmKvSS+k0/sEA=="], "@baseblocks/backend": ["@baseblocks/backend@workspace:packages/backend"], @@ -452,23 +470,21 @@ "@fastify/busboy": ["@fastify/busboy@2.1.1", "", {}, "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA=="], - "@firecrawl/anydoc": ["@firecrawl/anydoc@0.1.8", "", { "optionalDependencies": { "@firecrawl/anydoc-darwin-arm64": "0.1.8", "@firecrawl/anydoc-darwin-x64": "0.1.8", "@firecrawl/anydoc-linux-arm64-gnu": "0.1.8", "@firecrawl/anydoc-linux-arm64-musl": "0.1.8", "@firecrawl/anydoc-linux-x64-gnu": "0.1.8", "@firecrawl/anydoc-linux-x64-musl": "0.1.8", "@firecrawl/anydoc-win32-x64-msvc": "0.1.8" }, "bin": { "anydoc": "cli.js" } }, "sha512-DJXcCdEL1CcIyRRO/2C6SAcvn/SgM0nnpeGLOyZxJwCZJlx9s0HBUnsNtvQd5UHFUY/vZOURMOUf/h4I3HLjvQ=="], - - "@firecrawl/anydoc-darwin-arm64": ["@firecrawl/anydoc-darwin-arm64@0.1.8", "", { "os": "darwin", "cpu": "arm64" }, "sha512-B+30kDFGW8RUt1S6nOwdRT3ZTf8pDztid1ws8XL95UuAkz2d49ktg00hfEIW0xCgM1HByUTfQkQtcJsfF2O35g=="], + "@firecrawl/anydoc": ["@firecrawl/anydoc@0.1.9", "", { "optionalDependencies": { "@firecrawl/anydoc-darwin-arm64": "0.1.9", "@firecrawl/anydoc-darwin-x64": "0.1.9", "@firecrawl/anydoc-linux-arm64-gnu": "0.1.9", "@firecrawl/anydoc-linux-arm64-musl": "0.1.9", "@firecrawl/anydoc-linux-x64-gnu": "0.1.9", "@firecrawl/anydoc-linux-x64-musl": "0.1.9", "@firecrawl/anydoc-win32-x64-msvc": "0.1.9" }, "bin": { "anydoc": "cli.js" } }, "sha512-WT4PiXAYWsTOk65pH6MKmM8yB27Vj5NyAnc29pVAZtJSf9DO5QBCDKOU+iFai4t11N1Qjs1QuCLrp9DMIeRITQ=="], - "@firecrawl/anydoc-darwin-x64": ["@firecrawl/anydoc-darwin-x64@0.1.8", "", { "os": "darwin", "cpu": "x64" }, "sha512-PqUQFRv7I86ou2sZ0qE6bQFASi2Ja5yfpsU1vhngnqTsC31q2iSLDE5MWWGGRMoNZrWDh5ICTN7yQBoepkSokw=="], + "@firecrawl/anydoc-darwin-arm64": ["@firecrawl/anydoc-darwin-arm64@0.1.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-r7LQfF8nLArQmeMIeTWOeItqe3ktLO0DsuJWU/LvscGqGY+OPhdxRRvUU2vYOEbZUEqWW+FJWzV7Nni9JHLapw=="], - "@firecrawl/anydoc-linux-arm64-gnu": ["@firecrawl/anydoc-linux-arm64-gnu@0.1.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-lFuS2GdkKexYCCfBGb47si3vE4jA8Y+w9F377JJPeQc1/HQilqi+RjOhQrEgHiGlNygeOJwL6bKOQr+R+CWL9Q=="], + "@firecrawl/anydoc-darwin-x64": ["@firecrawl/anydoc-darwin-x64@0.1.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-/9jNFk4Waxm88PgxnrFO7vRyjqTPQvr37ABPQZKuBLAI+fhhVAYbyVhFvSrF2694yVXg84qdg0kws+5DtTzCog=="], - "@firecrawl/anydoc-linux-arm64-musl": ["@firecrawl/anydoc-linux-arm64-musl@0.1.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-MT+24XbTQ5Jh+SDmbQz5F46sbz/gE2SeWf/5RyFriuOJHtiYSb0rym1AOa7HZH12AJvnfPcU6xTqyj1gKxRF0g=="], + "@firecrawl/anydoc-linux-arm64-gnu": ["@firecrawl/anydoc-linux-arm64-gnu@0.1.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-FtPDamesoHW3NUDQVWWsjtt5H6oqObtcOwCwREnKRjNqnEVkIE9eW3/vElbGrZlrqlcPz4OLLAI4fGJUUKh3kQ=="], - "@firecrawl/anydoc-linux-x64-gnu": ["@firecrawl/anydoc-linux-x64-gnu@0.1.8", "", { "os": "linux", "cpu": "x64" }, "sha512-Row1/D9/3Zf5fFKaIyZqlMbz/Bxzl9FULFH7Zxlx/h4HTZT3S4Yl5d5A5Gl6T1Rps3qwrsJerXfL+9nLrj4s3Q=="], + "@firecrawl/anydoc-linux-arm64-musl": ["@firecrawl/anydoc-linux-arm64-musl@0.1.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-fvhEB45eo7xxfev3h+c3+gKhNrUvgMOKBrZol3R2Sljp97oVL2HyC4Mwqq+nqLW0yOH9qGikDgwlLLESD3gYSQ=="], - "@firecrawl/anydoc-linux-x64-musl": ["@firecrawl/anydoc-linux-x64-musl@0.1.8", "", { "os": "linux", "cpu": "x64" }, "sha512-PovgyJR229FbxYsPGFhHFio1B+zBn2WKgNMmon/aPZanSG5a+xGkv+ywdrYJnZtBF1+LHgfbiQPljv5BqdbUMg=="], + "@firecrawl/anydoc-linux-x64-gnu": ["@firecrawl/anydoc-linux-x64-gnu@0.1.9", "", { "os": "linux", "cpu": "x64" }, "sha512-12Fbby9ZxXNdGVuObBSIJTmJhmjm6MFjZinunjRR3j92cFLJiYeXYRHk8iYReSEXbgat0i34AfkY15uTjXoAng=="], - "@firecrawl/anydoc-wasm": ["@firecrawl/anydoc-wasm@0.1.8", "", {}, "sha512-RKqM9/Oh6NTRPeJgOSq50Kn+vNdBxeF/lFL7ceUJlfuxnFjbVwQ03AZfj3LBSyKZ0WO9WyE+H4/qTZ3M8uc/zQ=="], + "@firecrawl/anydoc-linux-x64-musl": ["@firecrawl/anydoc-linux-x64-musl@0.1.9", "", { "os": "linux", "cpu": "x64" }, "sha512-Y8VhjazuAAqBuuOZPEw5QwB0u7WX6EIKP8E3q/LGTKTIJsjl0tK6Hjcgxdg52SiF/aa1fB4oHx2UK2ojB1V2rA=="], - "@firecrawl/anydoc-win32-x64-msvc": ["@firecrawl/anydoc-win32-x64-msvc@0.1.8", "", { "os": "win32", "cpu": "x64" }, "sha512-WwCy76FKeNLMFpfQkegaGZh6Oh05XMTQkGCVizh5IDZXTtYfn2j+KqOQbx8XT1y+dewBYQIvq/8OJFpeHuckKA=="], + "@firecrawl/anydoc-win32-x64-msvc": ["@firecrawl/anydoc-win32-x64-msvc@0.1.9", "", { "os": "win32", "cpu": "x64" }, "sha512-SRX+GrFn+R/hTwSxJvJzKW0S+zn2vxJnolI9RTrn8HtNlyQUnvxB8PulzxvgBVV82kA3/ZTvGUKokH/SQe8sqg=="], "@flags-sdk/vercel": ["@flags-sdk/vercel@1.4.6", "", { "dependencies": { "@vercel/flags-core": "1.7.1" }, "peerDependencies": { "flags": "*" } }, "sha512-S0hkhjdzFmOTvpfI5kQMF6fyTNkx3V0dov3DOi3C6M/EdXlieZhGeMv21npW7zzj5IyUYUmYHihextWLe7ib5Q=="], @@ -814,8 +830,6 @@ "@paulirish/trace_engine": ["@paulirish/trace_engine@0.0.61", "", { "dependencies": { "legacy-javascript": "latest", "third-party-web": "latest" } }, "sha512-/O08DwmUqIlJjUSPSZbNF8lWnlxaMsIOV6sS+uDKCxBd5i1psAmjEoG3JAqR6+nHD8X+YY474NW7SxUH/K+/kQ=="], - "@pierre/icons": ["@pierre/icons@0.7.1", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-GzuvLVd8Hp86DuENXvQNNqlVDW7JQ3H4y7KNTK1bP42NA7+WeM8BuagFTq50jnYhrfG7I1YUju7e2k3exw/0fQ=="], - "@polar-sh/sdk": ["@polar-sh/sdk@0.49.0", "", { "dependencies": { "standardwebhooks": "^1.0.0", "zod": "^3.25.65 || ^4.0.0" } }, "sha512-9UYb70iKjJCtWYlu0OF5HLYBLmkxHwqr2RlXwuxXQgRGqq56IQWlVG+NO7e1YJ7I5GW0CBHhGIjRbQ9hYM6ycQ=="], "@preact/signals-core": ["@preact/signals-core@1.14.4", "", {}, "sha512-HNB6HYeYKhQbJ1aKl+YRjrS4+QWHLKX6qKoUsfS/m0vqzsVaEBiZiaKbG/e+NKk2ch5ALQr/ihWaMHxiCuuWHA=="], @@ -872,7 +886,7 @@ "@radix-ui/react-portal": ["@radix-ui/react-portal@1.1.17", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.10", "@radix-ui/react-use-layout-effect": "1.1.4" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ=="], - "@radix-ui/react-presence": ["@radix-ui/react-presence@1.1.6", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-zdTk4PlUO0E18HnZ3wYbW0KkJJxWCdiNYp6g6X1PtONFhxVkg01vliTJAmwIszU6mHiyBOoW9P0rAugl5/hULQ=="], + "@radix-ui/react-presence": ["@radix-ui/react-presence@1.1.10", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.4" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw=="], "@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.10", "", { "dependencies": { "@radix-ui/react-slot": "1.3.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg=="], @@ -2608,8 +2622,6 @@ "@paulirish/trace_engine/third-party-web": ["third-party-web@0.29.2", "", {}, "sha512-fegtha91tq2DHphyoiBXVHjVi2YG9zFaRnboT9C28tO1en9Y3wJsfspuy40F+u5wl3hHVbw7cnd1b67kEGHb8g=="], - "@radix-ui/react-presence/@radix-ui/react-use-layout-effect": ["@radix-ui/react-use-layout-effect@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jrBWOxZITuGcnjRCM2t2U5ZPkCLxD+Ym6DjfssS5haTj2iiak/DOb64JeN6OdLfLgptb6/e2kKR+ZuTrGoZTPA=="], - "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.3", "tslib": "^2.4.0" }, "bundled": true }, "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg=="], @@ -2746,6 +2758,8 @@ "micro/raw-body": ["raw-body@2.4.1", "", { "dependencies": { "bytes": "3.1.0", "http-errors": "1.7.3", "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, "sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA=="], + "micromatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], + "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], "path-scurry/lru-cache": ["lru-cache@11.5.2", "", {}, "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g=="], diff --git a/bunfig.toml b/bunfig.toml index ec0fca1c..27582102 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -7,15 +7,8 @@ minimumReleaseAgeExcludes = [ "@baseblocks/domain", "@baseblocks/ui", "@baseblocks/web", - "@baseblocks/anydoc", - "@baseblocks/anydoc-ingestion", - "@baseblocks/anydoc-convex", "@baseblocks/anydoc-contracts", - "@baseblocks/anydoc-presentation-viewer", - "@baseblocks/anydoc-react-viewer", - "@baseblocks/anydoc-spreadsheet-engine", - "@baseblocks/anydoc-spreadsheet-viewer", - "@baseblocks/anydoc-viewer-ui", + "@baseblocks/anydoc-viewer", "@firecrawl/anydoc", "@firecrawl/anydoc-darwin-arm64", "@firecrawl/anydoc-darwin-x64", diff --git a/package.json b/package.json index 2d99c655..d5d25e9c 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,8 @@ "defu": "6.1.7", "picomatch": "4.0.5", "postcss": "8.5.23", - "react": "^19.2.7", - "react-dom": "^19.2.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", "convex": "^1.42.1" }, "engines": { @@ -40,5 +40,46 @@ "apps/*", "packages/*", "tooling/*" - ] + ], + "overrides": { + "convex": "1.43.0", + "react": "19.2.8", + "react-dom": "19.2.8", + "@tiptap/core": "3.29.2", + "@tiptap/extension-blockquote": "3.29.2", + "@tiptap/extension-bold": "3.29.2", + "@tiptap/extension-bubble-menu": "3.29.2", + "@tiptap/extension-bullet-list": "3.29.2", + "@tiptap/extension-code": "3.29.2", + "@tiptap/extension-code-block": "3.29.2", + "@tiptap/extension-document": "3.29.2", + "@tiptap/extension-dropcursor": "3.29.2", + "@tiptap/extension-floating-menu": "3.29.2", + "@tiptap/extension-gapcursor": "3.29.2", + "@tiptap/extension-hard-break": "3.29.2", + "@tiptap/extension-heading": "3.29.2", + "@tiptap/extension-horizontal-rule": "3.29.2", + "@tiptap/extension-image": "3.29.2", + "@tiptap/extension-italic": "3.29.2", + "@tiptap/extension-link": "3.29.2", + "@tiptap/extension-list": "3.29.2", + "@tiptap/extension-list-item": "3.29.2", + "@tiptap/extension-list-keymap": "3.29.2", + "@tiptap/extension-ordered-list": "3.29.2", + "@tiptap/extension-paragraph": "3.29.2", + "@tiptap/extension-placeholder": "3.29.2", + "@tiptap/extension-strike": "3.29.2", + "@tiptap/extension-table": "3.29.2", + "@tiptap/extension-task-item": "3.29.2", + "@tiptap/extension-task-list": "3.29.2", + "@tiptap/extension-text": "3.29.2", + "@tiptap/extension-text-align": "3.29.2", + "@tiptap/extension-underline": "3.29.2", + "@tiptap/extension-unique-id": "3.29.2", + "@tiptap/extensions": "3.29.2", + "@tiptap/pm": "3.29.2", + "@tiptap/react": "3.29.2", + "@tiptap/starter-kit": "3.29.2", + "@tiptap/suggestion": "3.29.2" + } } diff --git a/packages/backend/README.md b/packages/backend/README.md index b7557b08..328fb5c0 100644 --- a/packages/backend/README.md +++ b/packages/backend/README.md @@ -30,11 +30,16 @@ bunx convex run deploymentPreflight:checkFileExtractionEnvironment ``` The check returns only variable names and non-secret configuration. Convex -validates and installs the AnyDoc adapter's Node dependency closure when the -functions are deployed. - -`convex.json` externalizes only `@baseblocks/anydoc-convex`; keep that adapter -pinned to the exact registry version verified by the application lockfile. +validates and installs the parser's Node dependency closure (`@firecrawl/anydoc`) +when the functions are deployed. + +`convex.json` externalizes `@firecrawl/anydoc`, whose native NAPI binary must +stay out of the Convex bundle; keep it pinned to the exact registry version +verified by the application lockfile. `@baseblocks/anydoc-contracts` is pure +TypeScript and bundles normally. The extraction queue and its failure encoding +live in `convex/fileExtractionQueue.ts`; bounded parsing and failure +classification live in `convex/fileExtractionParser.ts`; the Node action handler +is in `convex/fileExtractionAction.ts`. Uploads enqueue extraction immediately. Operators retry a terminal extraction through the product's file-extraction retry action; there is no polling cron or permanent backfill API. diff --git a/packages/backend/convex.json b/packages/backend/convex.json index 2c9f10fa..dae50c24 100644 --- a/packages/backend/convex.json +++ b/packages/backend/convex.json @@ -2,6 +2,6 @@ "$schema": "./node_modules/convex/schemas/convex.schema.json", "functions": "convex/", "node": { - "externalPackages": ["@baseblocks/anydoc-convex"] + "externalPackages": ["@firecrawl/anydoc"] } } diff --git a/packages/backend/convex/_generated/api.d.ts b/packages/backend/convex/_generated/api.d.ts index 444b4600..5b191b33 100644 --- a/packages/backend/convex/_generated/api.d.ts +++ b/packages/backend/convex/_generated/api.d.ts @@ -25,6 +25,8 @@ import type * as draftRestores from "../draftRestores.js"; import type * as editorWorkspace from "../editorWorkspace.js"; import type * as fileExtraction from "../fileExtraction.js"; import type * as fileExtractionAction from "../fileExtractionAction.js"; +import type * as fileExtractionParser from "../fileExtractionParser.js"; +import type * as fileExtractionQueue from "../fileExtractionQueue.js"; import type * as files from "../files.js"; import type * as http from "../http.js"; import type * as integrationAccess from "../integrationAccess.js"; @@ -112,6 +114,8 @@ declare const fullApi: ApiFromModules<{ editorWorkspace: typeof editorWorkspace; fileExtraction: typeof fileExtraction; fileExtractionAction: typeof fileExtractionAction; + fileExtractionParser: typeof fileExtractionParser; + fileExtractionQueue: typeof fileExtractionQueue; files: typeof files; http: typeof http; integrationAccess: typeof integrationAccess; diff --git a/packages/backend/convex/fileExtraction.state.test.ts b/packages/backend/convex/fileExtraction.state.test.ts index 43af3210..c7046764 100644 --- a/packages/backend/convex/fileExtraction.state.test.ts +++ b/packages/backend/convex/fileExtraction.state.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import type { WorkId } from "@baseblocks/anydoc-convex"; +import type { WorkId } from "@convex-dev/workpool"; import { completed, fileIngestion, diff --git a/packages/backend/convex/fileExtraction.ts b/packages/backend/convex/fileExtraction.ts index 609f9862..d30520b5 100644 --- a/packages/backend/convex/fileExtraction.ts +++ b/packages/backend/convex/fileExtraction.ts @@ -1,11 +1,4 @@ -import { - ConvexIngestionQueue, - decodeConvexIngestionFailure, - type ConvexIngestionJob, - type ConvexIngestionReceipt, - type DurableIngestionBinding, - type WorkId, -} from "@baseblocks/anydoc-convex"; +import type { WorkId } from "@convex-dev/workpool"; import { vOnCompleteArgs } from "@convex-dev/workpool"; import { ConvexError, v } from "convex/values"; import { components, internal } from "./_generated/api"; @@ -17,6 +10,13 @@ import { query, type QueryCtx, } from "./_generated/server"; +import { + ConvexIngestionQueue, + decodeConvexIngestionFailure, + type ConvexIngestionJob, + type ConvexIngestionReceipt, + type DurableIngestionBinding, +} from "./fileExtractionQueue"; import { assertDraftReadable, touchSiteDraft } from "./model/draft"; import { FILE_EXTRACTION_LIMITS, diff --git a/packages/backend/convex/fileExtractionAction.ts b/packages/backend/convex/fileExtractionAction.ts index 8ebccec0..7a240007 100644 --- a/packages/backend/convex/fileExtractionAction.ts +++ b/packages/backend/convex/fileExtractionAction.ts @@ -1,13 +1,19 @@ "use node"; -import { - createConvexIngestionHandler, - iterableSource, -} from "@baseblocks/anydoc-convex/node"; +import { iterableSource } from "@baseblocks/anydoc-contracts/sources"; +import { NonRetryableError } from "@convex-dev/workpool"; import { v } from "convex/values"; import { internal } from "./_generated/api"; -import { internalAction, type ActionCtx } from "./_generated/server"; +import { internalAction } from "./_generated/server"; import type { FileIngestionJob, FileIngestionResult } from "./fileExtraction"; +import { + classifyIngestionFailure, + createAttempt, + DEFAULT_ATTEMPT_TIMEOUT_MS, + ingestStoredDocument, + validTimeout, +} from "./fileExtractionParser"; +import { encodeConvexIngestionFailure } from "./fileExtractionQueue"; import { FILE_EXTRACTION_LIMITS, validateStoredSourceMetadata, @@ -40,66 +46,116 @@ function jobIdentity(job: FileIngestionJob) { }; } -const ingestionHandler = createConvexIngestionHandler< - ActionCtx, - FileIngestionJob ->({ - ingestion: { maxTextBytes: FILE_EXTRACTION_LIMITS.maxOutputBytes }, - resolveSource: async (ctx, job, attempt) => { - const source = await ctx.runMutation( - internal.fileExtraction.markProcessing, - jobIdentity(job), +export const process = internalAction({ + args: jobArgs, + handler: async (ctx, job): Promise => { + const timeout = validTimeout( + job.attemptTimeoutMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS, ); - if (!source) { - throw Object.assign(new Error("The file source changed"), { - code: "source-changed", - retryable: false, + const attempt = createAttempt(timeout, Date.now); + try { + const resolveSource = ctx.runMutation( + internal.fileExtraction.markProcessing, + jobIdentity(job), + ); + const source = await Promise.race([resolveSource, attempt.aborted]); + if (!source) { + throw Object.assign(new Error("The file source changed"), { + code: "source-changed", + retryable: false, + }); + } + const storage = getStorage(); + const metadata = await storage.head(source.objectKey, { + retries: FILE_EXTRACTION_LIMITS.storageRetries, + signal: attempt.value.signal, + timeout: FILE_EXTRACTION_LIMITS.storageTimeoutMs, }); - } - const storage = getStorage(); - const metadata = await storage.head(source.objectKey, { - retries: FILE_EXTRACTION_LIMITS.storageRetries, - signal: attempt.signal, - timeout: FILE_EXTRACTION_LIMITS.storageTimeoutMs, - }); - const failure = validateStoredSourceMetadata(source, metadata); - if (failure) { - throw Object.assign(new Error(failure.message), { - ...failure, - code: "source-changed", - retryable: false, + const failure = validateStoredSourceMetadata(source, metadata); + if (failure) { + throw Object.assign(new Error(failure.message), { + ...failure, + code: "source-changed", + retryable: false, + }); + } + const documentSource = iterableSource( + async ({ signal }) => { + const stored = await storage.download(source.objectKey, { + as: "stream", + retries: FILE_EXTRACTION_LIMITS.storageRetries, + signal, + timeout: FILE_EXTRACTION_LIMITS.storageTimeoutMs, + }); + return stored.stream(); + }, + { + contentType: source.contentType, + etag: metadata.etag, + filename: source.filename, + id: source.objectKey, + size: source.size, + }, + ); + + const resultPromise = ingestStoredDocument(documentSource, { + ...(job.contentType === undefined + ? {} + : { contentType: job.contentType }), + deadline: attempt.value.deadline, + ...(job.expectedSha256 === undefined + ? {} + : { expectedSha256: job.expectedSha256 }), + ...(job.expectedSize === undefined + ? {} + : { expectedSize: job.expectedSize }), + ...(job.filename === undefined ? {} : { filename: job.filename }), + ...(job.format === undefined ? {} : { format: job.format }), + ...(job.maxBytes === undefined ? {} : { maxBytes: job.maxBytes }), + maxTextBytes: FILE_EXTRACTION_LIMITS.maxOutputBytes, + signal: attempt.value.signal, }); + const result = await Promise.race([resultPromise, attempt.aborted]); + + const write = await Promise.race([ + ctx.runMutation(internal.fileExtraction.storeResult, { + format: result.format, + inputBytes: result.source.byteLength, + ...jobIdentity(job), + text: result.markdown, + }), + attempt.aborted, + ]); + return { + byteLength: result.source.byteLength, + format: result.format, + sha256: result.source.sha256, + status: write.status, + }; + } catch (cause) { + const { code, terminal } = classifyIngestionFailure(cause); + const failure = Object.assign( + new Error( + cause instanceof Error ? cause.message : "AnyDoc ingestion failed.", + { cause }, + ), + { + code, + retryable: !terminal, + ...(job.format === undefined ? {} : { format: job.format }), + ...(job.maxBytes === undefined ? {} : { maxBytes: job.maxBytes }), + ...(job.expectedSize === undefined + ? {} + : { expectedSize: job.expectedSize }), + }, + ); + const encoded = encodeConvexIngestionFailure(failure); + if (terminal) { + throw new NonRetryableError(encoded, { cause }); + } + throw new Error(encoded, { cause }); + } finally { + attempt.dispose(); } - return iterableSource( - async ({ signal }) => { - const stored = await storage.download(source.objectKey, { - as: "stream", - retries: FILE_EXTRACTION_LIMITS.storageRetries, - signal, - timeout: FILE_EXTRACTION_LIMITS.storageTimeoutMs, - }); - return stored.stream(); - }, - { - contentType: source.contentType, - etag: metadata.etag, - filename: source.filename, - id: source.objectKey, - size: source.size, - }, - ); }, - writeResult: (ctx, job, result) => - ctx.runMutation(internal.fileExtraction.storeResult, { - format: result.format, - inputBytes: result.source.byteLength, - ...jobIdentity(job), - text: result.markdown, - }), -}); - -export const process = internalAction({ - args: jobArgs, - handler: async (ctx, job): Promise => - ingestionHandler(ctx, job), }); diff --git a/packages/backend/convex/fileExtractionParser.test.ts b/packages/backend/convex/fileExtractionParser.test.ts new file mode 100644 index 00000000..5bb7b948 --- /dev/null +++ b/packages/backend/convex/fileExtractionParser.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, test } from "bun:test"; +import { + classifyIngestionFailure, + createAttempt, + formatFromContentType, + formatFromFilename, + ingestStoredDocument, + TERMINAL_ERROR_CODES, + validTimeout, +} from "./fileExtractionParser"; +import { + bytesSource, + iterableSource, +} from "@baseblocks/anydoc-contracts/sources"; + +const encoder = new TextEncoder(); + +describe("ingestion failure classification", () => { + test("explicit retryable=false is terminal regardless of code", () => { + const classified = classifyIngestionFailure( + Object.assign(new Error("nope"), { + code: "fetch-failed", + retryable: false, + }), + ); + expect(classified.terminal).toBe(true); + }); + + test("known terminal codes are terminal without an explicit flag", () => { + for (const code of TERMINAL_ERROR_CODES) { + expect(classifyIngestionFailure({ code }).terminal).toBe(true); + } + }); + + test("retryable codes stay retryable and default to processing-failed", () => { + expect( + classifyIngestionFailure({ code: "deadline-exceeded" }).terminal, + ).toBe(false); + expect(classifyIngestionFailure({}).code).toBe("processing-failed"); + expect(classifyIngestionFailure(undefined).code).toBe("processing-failed"); + }); +}); + +describe("attempt timeout", () => { + test("rejects invalid timeouts", () => { + expect(() => validTimeout(0)).toThrow(); + expect(() => validTimeout(Number.NaN)).toThrow(); + expect(validTimeout(1_000)).toBe(1_000); + }); + + test("aborts with a retryable deadline error after the timeout", async () => { + const attempt = createAttempt(10, Date.now); + let reason: unknown; + try { + await attempt.aborted; + } catch (cause) { + reason = cause; + } + expect(reason).toMatchObject({ + code: "deadline-exceeded", + retryable: true, + }); + attempt.dispose(); + }); +}); + +describe("format detection", () => { + test("text extensions win over signature detection", () => { + expect(formatFromFilename("notes.md")).toBe("markdown"); + expect(formatFromFilename("notes.MDOWN")).toBe("markdown"); + expect(formatFromFilename("notes.txt")).toBe("text"); + }); + + test("binary extensions resolve through the parser", () => { + expect(formatFromFilename("report.docx")).toBe("docx"); + expect(formatFromFilename("slides.pptx")).toBe("pptx"); + expect(formatFromFilename("sheet.xlsx")).toBe("xlsx"); + expect(formatFromFilename("no-extension")).toBeUndefined(); + }); + + test("content types map markdown, text, and csv only", () => { + expect(formatFromContentType("text/markdown; charset=utf-8")).toBe( + "markdown", + ); + expect(formatFromContentType("application/pdf")).toBeUndefined(); + }); +}); + +describe("stored document ingestion", () => { + function sourceFor(text: string) { + return bytesSource(encoder.encode(text), { filename: "note.txt" }); + } + + test("converts plain text passthrough with source metrics", async () => { + const result = await ingestStoredDocument(sourceFor("hello verification"), { + filename: "note.txt", + }); + expect(result.format).toBe("text"); + expect(result.markdown).toBe("hello verification"); + expect(result.source.byteLength).toBe(18); + expect(result.source.sha256).toMatch(/^[a-f\d]{64}$/); + }); + + test("enforces the text ceiling during the bounded read", async () => { + // Text hints cap input at the text ceiling, so oversized text fails in + // readSource before any conversion work happens. + await expect( + ingestStoredDocument(sourceFor("x".repeat(64)), { + filename: "note.txt", + maxTextBytes: 16, + }), + ).rejects.toMatchObject({ code: "too-large" }); + }); + + test("rejects non-UTF-8 text as terminal", async () => { + await expect( + ingestStoredDocument( + bytesSource(new Uint8Array([0xff, 0xfe, 0xfd]), { + filename: "note.txt", + }), + { + filename: "note.txt", + }, + ), + ).rejects.toMatchObject({ code: "invalid-text" }); + }); + + test("verifies expected checksums before parsing", async () => { + await expect( + ingestStoredDocument(sourceFor("hello"), { + filename: "note.txt", + expectedSha256: "0".repeat(64), + }), + ).rejects.toMatchObject({ code: "integrity-failed" }); + }); + + test("detects CSV by extension hint without signature detection", async () => { + const csv = "a,b\n1,2\n"; + const result = await ingestStoredDocument( + iterableSource( + async () => + (async function* () { + yield encoder.encode(csv); + })(), + { size: csv.length }, + ), + { format: "csv" }, + ); + expect(result.format).toBe("csv"); + // CSV converts to a GFM table. + expect(result.markdown).toContain("| a | b |"); + expect(result.markdown).toContain("| 1 | 2 |"); + }); + + test("fails terminally when no format can be detected", async () => { + await expect( + ingestStoredDocument(bytesSource(encoder.encode("???")), {}), + ).rejects.toMatchObject({ code: "invalid-source" }); + }); +}); diff --git a/packages/backend/convex/fileExtractionParser.ts b/packages/backend/convex/fileExtractionParser.ts new file mode 100644 index 00000000..b65e79ac --- /dev/null +++ b/packages/backend/convex/fileExtractionParser.ts @@ -0,0 +1,288 @@ +"use node"; + +// Bounded document parsing for file extraction. Pure logic with no Convex +// runtime dependencies so it can be unit-tested directly; the extraction +// action in fileExtractionAction.ts supplies storage and mutation plumbing. + +import { + defaultDocumentLimits, + DocumentPlatformError, +} from "@baseblocks/anydoc-contracts"; +import { + type iterableSource, + readSource, +} from "@baseblocks/anydoc-contracts/sources"; +import * as anydocModule from "@firecrawl/anydoc"; + +/** + * Minimal surface of the native AnyDoc parser used by extraction. + */ +export type AnyDocFormat = + | "csv" + | "doc" + | "docx" + | "epub" + | "odp" + | "ods" + | "odt" + | "pdf" + | "ppt" + | "pptx" + | "rtf" + | "xlsx"; + +interface AnyDocParser { + formatFromBytes(bytes: Uint8Array): AnyDocFormat | null; + formatFromExtension(extension: string): AnyDocFormat | null; + toMarkdownBytes( + bytes: Uint8Array, + format?: AnyDocFormat | null, + ): Promise; +} + +const anydoc = anydocModule as unknown as AnyDocParser; + +const TEXT_FORMATS = new Set(["markdown", "text"]); + +/** Error codes that must never be retried; anything else may be retried. */ +export const TERMINAL_ERROR_CODES: ReadonlySet = new Set([ + "encrypted", + "integrity-failed", + "invalid-source", + "invalid-text", + "malformed", + "missing-part", + "output-too-large", + "processing-failed", + "resource-limit", + "source-changed", + "too-large", + "too-many-cells", + "too-many-pages", + "too-many-slides", + "unsupported", +]); + +export function normalizeFormat( + value: string | undefined | null, +): string | undefined { + if (value === undefined || value === null || value === "") return undefined; + return String(value).toLowerCase().replace(/^\./, ""); +} + +export function formatFromFilename( + filename: string | undefined, +): string | undefined { + if (!filename) return undefined; + const extension = ( + filename.includes(".") + ? filename.slice(filename.lastIndexOf(".")) + : filename + ).toLowerCase(); + if ( + extension === ".md" || + extension === ".mdown" || + extension === ".markdown" + ) { + return "markdown"; + } + if (extension === ".txt" || extension === ".text") return "text"; + return anydoc.formatFromExtension(extension) ?? undefined; +} + +export function formatFromContentType( + contentType: string | undefined, +): string | undefined { + const type = contentType?.split(";", 1)[0]?.trim().toLowerCase(); + if (type === "text/markdown" || type === "text/x-markdown") return "markdown"; + if (type === "text/plain") return "text"; + if (type === "text/csv") return "csv"; + return undefined; +} + +function decodeTextContent(bytes: Uint8Array, format: string): string { + try { + // Strict UTF-8 decoding; invalid text is a terminal failure, not mojibake. + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (cause) { + throw new DocumentPlatformError("The document is not valid UTF-8 text.", { + cause, + code: "invalid-text", + format, + }); + } +} + +export interface StoredDocumentIngestOptions { + readonly contentType?: string; + readonly deadline?: number; + readonly expectedSha256?: string; + readonly expectedSize?: number; + readonly filename?: string; + readonly format?: string; + readonly maxBytes?: number; + readonly maxTextBytes?: number; + readonly signal?: AbortSignal; +} + +/** + * Bounded one-shot conversion: read the untrusted source under hard limits, + * detect the format from content signatures, and convert to Markdown with the + * native AnyDoc parser. Ported from the former @baseblocks/anydoc-ingestion + * pipeline, reduced to what file extraction needs. + */ +export async function ingestStoredDocument( + source: ReturnType, + options: StoredDocumentIngestOptions = {}, +) { + const maxTextBytes = + options.maxTextBytes ?? defaultDocumentLimits.maxTextBytes; + const hintedFormat = + normalizeFormat(options.format) ?? + formatFromFilename(options.filename) ?? + formatFromContentType(options.contentType); + const read = await readSource(source, { + calculateSha256: true, + ...(options.deadline === undefined ? {} : { deadline: options.deadline }), + ...(options.expectedSha256 === undefined + ? {} + : { expectedSha256: options.expectedSha256 }), + ...(options.expectedSize === undefined + ? {} + : { expectedSize: options.expectedSize }), + maxBytes: + options.maxBytes ?? + (TEXT_FORMATS.has(hintedFormat ?? "") + ? maxTextBytes + : defaultDocumentLimits.maxBytes), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + + const skipSignatureDetection = + TEXT_FORMATS.has(hintedFormat ?? "") || hintedFormat === "csv"; + const detectedFormat = skipSignatureDetection + ? undefined + : normalizeFormat(anydoc.formatFromBytes(read.bytes)); + const format = + detectedFormat ?? + hintedFormat ?? + formatFromFilename(read.filename) ?? + formatFromContentType(read.contentType); + + if (!format) { + throw new DocumentPlatformError( + "The document format could not be detected. Pass format for signature-less input such as CSV or plain text.", + { code: "invalid-source" }, + ); + } + + let markdown: string; + if (TEXT_FORMATS.has(format)) { + markdown = decodeTextContent(read.bytes, format); + } else { + try { + // Text passthrough formats never reach this branch, so `format` is + // always one of the native parser formats here. + markdown = await anydoc.toMarkdownBytes( + read.bytes, + format as AnyDocFormat, + ); + } catch (cause) { + throw new DocumentPlatformError( + cause instanceof Error ? cause.message : "Document conversion failed.", + { cause, code: "processing-failed", retryable: false }, + ); + } + } + if (typeof markdown !== "string") { + throw new DocumentPlatformError("The parser returned no Markdown.", { + code: "processing-failed", + retryable: false, + }); + } + if (new TextEncoder().encode(markdown).byteLength > maxTextBytes) { + throw new DocumentPlatformError( + `Extracted Markdown exceeds the ${maxTextBytes.toLocaleString()} byte limit.`, + { code: "output-too-large", retryable: false }, + ); + } + return { + format, + markdown, + source: { + byteLength: read.byteLength, + sha256: read.sha256, + }, + }; +} + +export const DEFAULT_ATTEMPT_TIMEOUT_MS = 120_000; + +export function validTimeout(value: number): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error("attemptTimeoutMs must be a positive integer."); + } + return value; +} + +export interface IngestAttempt { + readonly deadline: number; + readonly signal: AbortSignal; +} + +export function createAttempt( + timeout: number, + now: () => number, +): { + aborted: Promise; + dispose: () => void; + value: IngestAttempt; +} { + const controller = new AbortController(); + const deadline = now() + timeout; + const failure = Object.assign( + new Error("The ingestion attempt exceeded its deadline."), + { + code: "deadline-exceeded", + retryable: true, + }, + ); + const timer = setTimeout(() => controller.abort(failure), timeout); + const aborted = new Promise((_resolve, reject) => { + controller.signal.addEventListener( + "abort", + () => reject(controller.signal.reason), + { + once: true, + }, + ); + }); + return { + aborted, + dispose: () => clearTimeout(timer), + value: { deadline, signal: controller.signal } satisfies IngestAttempt, + }; +} + +export interface ClassifiedIngestionFailure { + readonly code: string; + readonly terminal: boolean; +} + +/** Decide whether a thrown cause ends the retry loop and under which code. */ +export function classifyIngestionFailure( + cause: unknown, +): ClassifiedIngestionFailure { + const code = + cause && typeof cause === "object" && "code" in cause + ? String(cause.code) + : undefined; + const retryable = + cause && typeof cause === "object" && "retryable" in cause + ? cause.retryable === true + : undefined; + const terminal = + retryable === false || + (code !== undefined && TERMINAL_ERROR_CODES.has(code)); + return { code: code ?? "processing-failed", terminal }; +} diff --git a/packages/backend/convex/fileExtractionQueue.test.ts b/packages/backend/convex/fileExtractionQueue.test.ts new file mode 100644 index 00000000..72c20150 --- /dev/null +++ b/packages/backend/convex/fileExtractionQueue.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { + decodeConvexIngestionFailure, + encodeConvexIngestionFailure, +} from "./fileExtractionQueue"; + +describe("Convex ingestion failure codec", () => { + test("round-trips code, message, retryability, and format", () => { + const cause = Object.assign(new Error("The document is encrypted."), { + code: "encrypted", + retryable: false, + format: "pdf", + }); + const decoded = decodeConvexIngestionFailure( + encodeConvexIngestionFailure(cause), + ); + expect(decoded).toMatchObject({ + version: 1, + kind: "anydoc-ingestion-failure", + code: "encrypted", + message: "The document is encrypted.", + retryable: false, + format: "pdf", + }); + }); + + test("decodes Workpool failed results", () => { + const encoded = encodeConvexIngestionFailure( + Object.assign(new Error("boom"), { code: "too-large", retryable: false }), + ); + const decoded = decodeConvexIngestionFailure({ + kind: "failed", + error: encoded, + }); + expect(decoded?.code).toBe("too-large"); + }); + + test("collects limit fields through the cause chain", () => { + const inner = Object.assign(new Error("limit"), { maxBytes: 100 }); + const outer = new Error("wrap", { cause: inner }); + const decoded = decodeConvexIngestionFailure( + encodeConvexIngestionFailure( + Object.assign(outer, { code: "resource-limit", retryable: false }), + ), + ); + expect(decoded?.limits).toMatchObject({ maxBytes: 100 }); + }); + + test("ignores non-failure strings and malformed payloads", () => { + expect(decodeConvexIngestionFailure("plain error text")).toBeUndefined(); + expect(decodeConvexIngestionFailure(undefined)).toBeUndefined(); + expect(decodeConvexIngestionFailure({ kind: "canceled" })).toBeUndefined(); + const forged = + "ANYDOC_FAILURE_V1:" + encodeURIComponent(JSON.stringify({ version: 9 })); + expect(decodeConvexIngestionFailure(forged)).toBeUndefined(); + expect( + decodeConvexIngestionFailure("ANYDOC_FAILURE_V1:%7Bnot-json"), + ).toBeUndefined(); + }); + + test("bounds oversized messages and unknown limit keys are dropped", () => { + const cause = Object.assign(new Error("x".repeat(10_000)), { + code: "processing-failed", + retryable: true, + notALimitKey: 1, + }); + const decoded = decodeConvexIngestionFailure( + encodeConvexIngestionFailure(cause), + ); + expect(decoded).toBeDefined(); + expect(Array.from(decoded!.message).length).toBeLessThanOrEqual(2_048); + expect(decoded?.limits).toBeUndefined(); + }); +}); diff --git a/packages/backend/convex/fileExtractionQueue.ts b/packages/backend/convex/fileExtractionQueue.ts new file mode 100644 index 00000000..620f56fc --- /dev/null +++ b/packages/backend/convex/fileExtractionQueue.ts @@ -0,0 +1,379 @@ +// Durable file-extraction job plumbing owned by BaseBlocks. +// Ported from the former @baseblocks/anydoc-convex package: Workpool owns +// scheduling, concurrency, retry state, cancellation, and status; this module +// adds idempotent binding of work IDs to extraction rows and structured +// failure encoding across the Workpool result channel. + +import { + Workpool, + type OnCompleteArgs, + type RetryBehavior, + type WorkId, + type WorkpoolComponent, + type WorkpoolOptions, +} from "@convex-dev/workpool"; +import type { + DefaultFunctionArgs, + FunctionReference, + FunctionVisibility, +} from "convex/server"; + +export type WorkpoolMutationContext = Parameters[0]; +export type WorkpoolQueryContext = Parameters[0]; + +export interface ConvexIngestionJob + extends DefaultFunctionArgs { + readonly entityId: string; + readonly sourceVersion: string; + /** Monotonically increases whenever source or cancellation state changes. */ + readonly generation: number; + readonly idempotencyKey: string; + readonly source: Source; + readonly metadata?: Metadata; + readonly format?: string; + readonly filename?: string; + readonly contentType?: string; + readonly expectedSize?: number; + readonly expectedSha256?: string; + readonly maxBytes?: number; + /** Maximum wall time for each Workpool attempt. A fresh deadline is created on every retry. */ + readonly attemptTimeoutMs?: number; +} + +export interface ConvexIngestionReceipt { + readonly entityId: string; + readonly sourceVersion: string; + readonly generation: number; + readonly idempotencyKey: string; + readonly workId: WorkId; +} + +export interface DurableIngestionBinding< + Args extends ConvexIngestionJob, + MutationContext extends WorkpoolMutationContext = WorkpoolMutationContext, + QueryContext extends WorkpoolQueryContext = WorkpoolQueryContext, + State = unknown, +> { + /** Atomically keep the first workId for this idempotency key/generation and return the winner. */ + bind(ctx: MutationContext, job: Args, candidate: WorkId): Promise; + /** Atomically invalidate the generation before returning true. */ + cancel( + ctx: MutationContext, + receipt: ConvexIngestionReceipt, + ): Promise; + status(ctx: QueryContext, receipt: ConvexIngestionReceipt): Promise; +} + +export interface ConvexIngestionQueueOptions< + Args extends ConvexIngestionJob, + MutationContext extends WorkpoolMutationContext = WorkpoolMutationContext, + QueryContext extends WorkpoolQueryContext = WorkpoolQueryContext, + BindingState = unknown, +> extends WorkpoolOptions { + /** Retries are safe only when the result writer is idempotent by idempotencyKey. */ + readonly retry?: boolean | RetryBehavior; + readonly binding: DurableIngestionBinding< + Args, + MutationContext, + QueryContext, + BindingState + >; + readonly onComplete?: FunctionReference< + "mutation", + FunctionVisibility, + OnCompleteArgs + >; + readonly completionContext?: (job: Args) => unknown; +} + +export class ConvexIngestionQueue< + Args extends ConvexIngestionJob, + ReturnValue = unknown, + MutationContext extends WorkpoolMutationContext = WorkpoolMutationContext, + QueryContext extends WorkpoolQueryContext = WorkpoolQueryContext, + BindingState = unknown, +> { + readonly #action: FunctionReference< + "action", + FunctionVisibility, + Args, + ReturnValue + >; + readonly #pool: Workpool; + readonly #retry: boolean | RetryBehavior; + readonly #binding: DurableIngestionBinding< + Args, + MutationContext, + QueryContext, + BindingState + >; + readonly #onComplete: + | FunctionReference<"mutation", FunctionVisibility, OnCompleteArgs> + | undefined; + readonly #completionContext: ((job: Args) => unknown) | undefined; + + constructor( + component: WorkpoolComponent, + action: FunctionReference<"action", FunctionVisibility, Args, ReturnValue>, + options: ConvexIngestionQueueOptions< + Args, + MutationContext, + QueryContext, + BindingState + >, + ) { + const { + binding, + completionContext, + onComplete, + retry = true, + ...poolOptions + } = options; + this.#action = action; + this.#retry = retry; + this.#binding = binding; + this.#onComplete = onComplete; + this.#completionContext = completionContext; + this.#pool = new Workpool(component, { + defaultRetryBehavior: { + base: 2, + initialBackoffMs: 1_000, + maxAttempts: 4, + }, + maxParallelism: 4, + ...poolOptions, + retryActionsByDefault: false, + }); + } + + async enqueue( + ctx: MutationContext, + job: Args, + ): Promise { + if (!job.idempotencyKey || job.idempotencyKey.length > 512) { + throw new Error( + "AnyDoc Convex jobs require an idempotencyKey of at most 512 characters.", + ); + } + if ( + !Number.isSafeInteger(job.generation) || + job.generation < 0 || + !job.entityId || + !job.sourceVersion + ) { + throw new Error( + "AnyDoc Convex jobs require entityId, sourceVersion, and a non-negative generation.", + ); + } + if ( + job.attemptTimeoutMs !== undefined && + (!Number.isSafeInteger(job.attemptTimeoutMs) || job.attemptTimeoutMs <= 0) + ) { + throw new Error( + "attemptTimeoutMs must be a positive integer when provided.", + ); + } + const candidate = await this.#pool.enqueueAction(ctx, this.#action, job, { + name: "anydoc:ingest", + retry: this.#retry, + ...(this.#onComplete + ? { + onComplete: this.#onComplete, + context: this.#completionContext?.(job), + } + : {}), + }); + let workId: WorkId; + try { + workId = await this.#binding.bind(ctx, job, candidate); + } catch (cause) { + // Enqueue happens before the application can atomically bind its workId. + // Best-effort cancellation prevents an unbound candidate from becoming + // an invisible orphan; the binding error remains authoritative. + await this.#pool.cancel(ctx, candidate).catch(() => undefined); + throw cause; + } + if (workId !== candidate) await this.#pool.cancel(ctx, candidate); + return { + entityId: job.entityId, + generation: job.generation, + idempotencyKey: job.idempotencyKey, + sourceVersion: job.sourceVersion, + workId, + }; + } + + async cancel( + ctx: MutationContext, + receipt: ConvexIngestionReceipt, + ): Promise { + const cancelled = await this.#binding.cancel(ctx, receipt); + if (cancelled) await this.#pool.cancel(ctx, receipt.workId); + return cancelled; + } + + async status(ctx: QueryContext, receipt: ConvexIngestionReceipt) { + const [workpool, binding] = await Promise.all([ + this.#pool.status(ctx, receipt.workId), + this.#binding.status(ctx, receipt), + ]); + return { binding, workpool }; + } +} + +const FAILURE_MARKER = "ANYDOC_FAILURE_V1:"; +const MAX_FAILURE_MESSAGE_CHARACTERS = 2_048; +const MAX_FAILURE_FIELD_CHARACTERS = 128; +const LIMIT_KEYS = new Set([ + "actualBytes", + "actualSize", + "expectedSize", + "limit", + "maxBytes", + "maxCells", + "maxDocumentBytes", + "maxPages", + "maxSlides", + "maxTextBytes", + "maximum", +]); + +export interface ConvexIngestionFailure { + readonly version: 1; + readonly kind: "anydoc-ingestion-failure"; + readonly code: string; + readonly message: string; + readonly retryable: boolean; + readonly format?: string; + readonly status?: number; + readonly limits?: Readonly>; +} + +function field(cause: unknown, name: string): unknown { + return cause && typeof cause === "object" && name in cause + ? (cause as Record)[name] + : undefined; +} + +function boundedString(value: string, maximum: number): string { + return Array.from(value).slice(0, maximum).join(""); +} + +function collectLimits( + cause: unknown, +): Record | undefined { + const limits: Record = {}; + let current = cause; + for ( + let depth = 0; + depth < 4 && current && typeof current === "object"; + depth += 1 + ) { + for (const key of LIMIT_KEYS) { + const value = field(current, key); + if ( + (typeof value === "number" && Number.isFinite(value)) || + typeof value === "string" + ) { + limits[key] ??= + typeof value === "string" + ? boundedString(value, MAX_FAILURE_FIELD_CHARACTERS) + : value; + } + } + current = field(current, "cause"); + } + return Object.keys(limits).length === 0 ? undefined : limits; +} + +/** Encodes a failure into the Workpool string channel without losing stable machine fields. */ +export function encodeConvexIngestionFailure(cause: unknown): string { + const code = field(cause, "code"); + const retryable = field(cause, "retryable"); + const format = field(cause, "format"); + const status = field(cause, "status"); + const limits = collectLimits(cause); + const failure: ConvexIngestionFailure = { + version: 1, + kind: "anydoc-ingestion-failure", + code: + typeof code === "string" + ? boundedString(code, MAX_FAILURE_FIELD_CHARACTERS) + : "processing-failed", + message: boundedString( + cause instanceof Error ? cause.message : "AnyDoc ingestion failed.", + MAX_FAILURE_MESSAGE_CHARACTERS, + ), + retryable: retryable === true, + ...(typeof format === "string" + ? { format: boundedString(format, MAX_FAILURE_FIELD_CHARACTERS) } + : {}), + ...(typeof status === "number" && Number.isFinite(status) + ? { status } + : {}), + ...(limits ? { limits } : {}), + }; + return `${FAILURE_MARKER}${encodeURIComponent(JSON.stringify(failure))}`; +} + +/** Accepts either Workpool's complete result or its error string. */ +export function decodeConvexIngestionFailure( + value: OnCompleteArgs["result"] | string | unknown, +): ConvexIngestionFailure | undefined { + const text = + typeof value === "string" + ? value + : value && + typeof value === "object" && + "kind" in value && + value.kind === "failed" && + "error" in value + ? String(value.error) + : undefined; + if (!text) return undefined; + const start = text.indexOf(FAILURE_MARKER); + if (start < 0) return undefined; + const encoded = text + .slice(start + FAILURE_MARKER.length) + .match(/^[A-Za-z0-9%_.!~*'()-]+/)?.[0]; + if (!encoded) return undefined; + try { + const parsed = JSON.parse( + decodeURIComponent(encoded), + ) as Partial; + if ( + parsed.version !== 1 || + parsed.kind !== "anydoc-ingestion-failure" || + typeof parsed.code !== "string" || + Array.from(parsed.code).length > MAX_FAILURE_FIELD_CHARACTERS || + typeof parsed.message !== "string" || + Array.from(parsed.message).length > MAX_FAILURE_MESSAGE_CHARACTERS || + typeof parsed.retryable !== "boolean" || + (parsed.format !== undefined && + (typeof parsed.format !== "string" || + Array.from(parsed.format).length > MAX_FAILURE_FIELD_CHARACTERS)) || + (parsed.status !== undefined && + (typeof parsed.status !== "number" || + !Number.isFinite(parsed.status))) || + (parsed.limits !== undefined && + (!parsed.limits || + typeof parsed.limits !== "object" || + Array.isArray(parsed.limits) || + Object.keys(parsed.limits).length > LIMIT_KEYS.size || + Object.entries(parsed.limits).some( + ([key, item]) => + !LIMIT_KEYS.has(key) || + !( + (typeof item === "number" && Number.isFinite(item)) || + (typeof item === "string" && + Array.from(item).length <= MAX_FAILURE_FIELD_CHARACTERS) + ), + ))) + ) { + return undefined; + } + return parsed as ConvexIngestionFailure; + } catch { + return undefined; + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index e4ea7941..489911ad 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,13 +34,14 @@ "@aws-sdk/client-s3": "^3.1084.0", "@aws-sdk/s3-presigned-post": "^3.1084.0", "@aws-sdk/s3-request-presigner": "^3.1084.0", - "@baseblocks/anydoc-convex": "0.1.0-alpha.15", + "@baseblocks/anydoc-contracts": "0.2.0-alpha.3", "@baseblocks/custom-blocks": "workspace:*", "@baseblocks/domain": "workspace:*", "@baseblocks/openeditor-contracts": "workspace:*", "@convex-dev/better-auth": "^0.12.5", "@convex-dev/workflow": "^0.4.4", "@convex-dev/workpool": "^0.4.9", + "@firecrawl/anydoc": "0.1.9", "@noble/hashes": "2.0.1", "@openeditor/core": "0.0.48", "@openeditor/custom-block": "0.0.48",