From 6f63222a6494bcff9b2b7befe6c3d483d1f52866 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 18:36:04 +0100 Subject: [PATCH 01/29] fix(routing): bundle CommonJS dependencies of compiled-binary API routes A compiled binary cannot load a CommonJS file from disk at runtime, so the route build now bundles a project's CommonJS dependencies with esbuild's Node resolution and interop instead of loading them through a hand-written runtime CommonJS loader. ESM dependencies still load from file:// URLs. - built-ins a bundled module requires are imported statically and bound to the module-level require; any other id throws MODULE_NOT_FOUND - __dirname/__filename are defined per bundled CommonJS file - native addons fail the build with a clear message - bundled dependency files pass the same project boundary and import scan as route code Refs veryfront/veryfront-issue-inbox#4177, veryfront/veryfront-code#5471 --- .../compiled-cjs-bundling.test.ts | 172 ++++++++++++++++++ .../module-loader/compiled-cjs-bundling.ts | 115 ++++++++++++ src/routing/api/module-loader/loader.ts | 97 +++++++--- .../integration/compiled-binary-e2e.suite.ts | 78 ++++++++ 4 files changed, 434 insertions(+), 28 deletions(-) create mode 100644 src/routing/api/module-loader/compiled-cjs-bundling.test.ts create mode 100644 src/routing/api/module-loader/compiled-cjs-bundling.ts diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts new file mode 100644 index 0000000000..28a54fa1ee --- /dev/null +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -0,0 +1,172 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import * as esbuild from "veryfront/extensions/bundler"; +import type { Plugin } from "veryfront/extensions/bundler"; +import { + builtinModuleName, + commonJSPathPrelude, + createCompiledBuiltinPlugin, + prependBuiltinRequire, +} from "./compiled-cjs-bundling.ts"; + +async function writeFiles(root: string, files: Record): Promise { + for (const [relative, contents] of Object.entries(files)) { + const path = `${root}/${relative}`; + await Deno.mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true }); + await Deno.writeTextFile(path, contents); + } +} + +/** Defines __dirname/__filename for node_modules files, as the route build does. */ +const pathPreludePlugin: Plugin = { + name: "test-path-prelude", + setup(build) { + build.onLoad({ filter: /node_modules[\\/].*\.c?js$/ }, async (args) => { + const contents = await Deno.readTextFile(args.path); + return { contents: commonJSPathPrelude(contents, args.path) + contents, loader: "js" }; + }); + }, +}; + +/** Bundles a route the way the compiled binary does and imports the result. */ +async function bundleAndImport( + projectDir: string, + route: string, +): Promise> { + const requiredBuiltins = new Set(); + const result = await esbuild.build({ + bundle: true, + write: false, + format: "esm", + platform: "neutral", + target: "es2022", + mainFields: ["main", "module"], + conditions: ["node"], + resolveExtensions: [".ts", ".js", ".mjs", ".json"], + logLevel: "silent", + stdin: { + contents: route, + loader: "ts", + resolveDir: projectDir, + sourcefile: `${projectDir}/route.ts`, + }, + plugins: [createCompiledBuiltinPlugin(requiredBuiltins), pathPreludePlugin], + }); + const code = prependBuiltinRequire(result.outputFiles?.[0]?.text ?? "", requiredBuiltins); + const file = `${projectDir}/handler.mjs`; + await Deno.writeTextFile(file, code); + return await import(`file://${file}`); +} + +describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { + describe("builtinModuleName()", () => { + it("names built-ins with or without the node: prefix, including subpaths", () => { + assertEquals(builtinModuleName("fs"), "fs"); + assertEquals(builtinModuleName("node:fs"), "fs"); + assertEquals(builtinModuleName("fs/promises"), "fs/promises"); + assertEquals(builtinModuleName("left-pad"), undefined); + }); + + it("treats prefix-only built-ins as npm packages without the prefix", () => { + assertEquals(builtinModuleName("test"), undefined); + assertEquals(builtinModuleName("node:test"), "test"); + }); + }); + + describe("commonJSPathPrelude()", () => { + it("declares the file's own directory and path when the file reads them", () => { + assertEquals( + commonJSPathPrelude("module.exports = __dirname;", "/app/node_modules/a/index.js"), + 'var __dirname = "/app/node_modules/a", __filename = "/app/node_modules/a/index.js";\n', + ); + }); + + it("adds nothing to a file that does not read them", () => { + assertEquals(commonJSPathPrelude("module.exports = 1;", "/app/node_modules/a/i.js"), ""); + }); + + it("leaves a file that declares its own binding alone", () => { + const esm = 'import { dirname } from "path";\nconst __dirname = dirname("x");'; + assertEquals(commonJSPathPrelude(esm, "/app/node_modules/a/i.js"), ""); + }); + }); + + describe("prependBuiltinRequire()", () => { + it("keeps a hashbang on the first line", () => { + const code = prependBuiltinRequire("#!/usr/bin/env node\nexport {};\n", ["path"]); + assert(code.startsWith("#!/usr/bin/env node\nimport * as "), code); + }); + + it("picks helper names the bundle does not already use", () => { + const code = prependBuiltinRequire("const __vf_builtin0 = 1;\n", ["path"]); + assert(code.includes("import * as __vf_builtin_0 from"), code); + }); + }); + + describe("bundling a route's CommonJS dependency", () => { + it("runs exports maps, built-in and JSON requires, __dirname and optional requires", async () => { + const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); + try { + await writeFiles(projectDir, { + "node_modules/legacy/package.json": JSON.stringify({ + name: "legacy", + // The node condition must win over default, as it does in Node. + exports: { ".": { node: "./lib/index.js", default: "./lib/missing.js" } }, + }), + "node_modules/legacy/lib/index.js": [ + '"use strict";', + 'var fs = require("fs");', + 'var path = require("node:path");', + 'var data = require("./data");', + "var optional;", + 'try { optional = require("not-installed"); } catch (error) { optional = error.code; }', + "exports.name = data.name;", + "exports.ownFile = path.basename(__filename);", + 'exports.readsSibling = fs.existsSync(path.join(__dirname, "data.json"));', + "exports.optional = optional;", + ].join("\n"), + "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), + }); + const mod = await bundleAndImport( + projectDir, + 'const require = (id: string) => "route:" + id;\n' + + 'export const routeRequire = require("x");\n' + + 'import legacy from "legacy";\nexport { legacy };\n', + ); + assertEquals(mod.legacy, { + name: "legacy-data", + ownFile: "index.js", + readsSibling: true, + optional: "MODULE_NOT_FOUND", + }); + // esbuild renames a route's own `require`, so the prologue cannot clash. + assertEquals(mod.routeRequire, "route:x"); + } finally { + await esbuild.stop(); + await Deno.remove(projectDir, { recursive: true }); + } + }); + + it("rejects a native addon at build time", async () => { + const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); + try { + await writeFiles(projectDir, { + "node_modules/native/package.json": JSON.stringify({ name: "native", main: "index.js" }), + "node_modules/native/index.js": 'module.exports = require("./build/binding.node");', + "node_modules/native/build/binding.node": "not a real addon", + }); + const error = await assertRejects(() => + bundleAndImport(projectDir, 'import native from "native";\nexport { native };\n') + ); + assert( + String((error as Error).message).includes("Native addons are not supported"), + String(error), + ); + } finally { + await esbuild.stop(); + await Deno.remove(projectDir, { recursive: true }); + } + }); + }); +}); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts new file mode 100644 index 0000000000..129c127bf1 --- /dev/null +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -0,0 +1,115 @@ +/** + * CommonJS dependencies of an API route in the compiled binary. + * + * A compiled binary cannot load a CommonJS file from disk at runtime: Deno's + * `require` there reads only the files embedded at compile time, and importing + * a `.cjs` file evaluates it as ESM. So the route build bundles a project's + * CommonJS dependencies into the route module itself, and esbuild supplies Node + * resolution and interop. Three pieces keep the bundled code runnable: + * + * - A `require()` of a built-in stays external. esbuild emits it as a call to + * the module's free `require`, which {@link prependBuiltinRequire} binds to + * built-ins imported statically, so nothing loads `node:module` at runtime. + * - `__dirname` and `__filename` are not defined in an ES module, so + * {@link commonJSPathPrelude} defines them for each bundled file that uses + * them, as Node's CommonJS wrapper would. + * - A native addon cannot be bundled or loaded; the build rejects it with a + * clear message rather than an esbuild loader error. + * + * @module routing/api/module-loader/compiled-cjs-bundling + */ +import { builtinModules } from "node:module"; +import type { Plugin } from "veryfront/extensions/bundler"; + +// Names Node accepts only with the `node:` prefix. A bare `test` is an npm package. +const PREFIX_ONLY_BUILTINS = new Set(["test", "test/reporters", "sqlite", "sea"]); +const BUILTINS = new Set(builtinModules.map((name) => name.replace(/^node:/, ""))); + +/** The built-in module a specifier names, without its `node:` prefix. */ +export function builtinModuleName(specifier: string): string | undefined { + if (specifier.startsWith("node:")) { + const name = specifier.slice(5); + return BUILTINS.has(name) || PREFIX_ONLY_BUILTINS.has(name) ? name : undefined; + } + if (PREFIX_ONLY_BUILTINS.has(specifier)) return undefined; + return BUILTINS.has(specifier) ? specifier : undefined; +} + +/** + * Keeps built-ins external and records each one a bundled module `require`s, + * so {@link prependBuiltinRequire} can bind it. Must run before any plugin + * that resolves built-ins itself. + */ +export function createCompiledBuiltinPlugin(requiredBuiltins: Set): Plugin { + return { + name: "vf-compiled-builtins", + setup(build) { + build.onResolve({ filter: /.*/ }, (args) => { + const name = builtinModuleName(args.path); + if (name === undefined) return undefined; + if (args.kind === "require-call") { + requiredBuiltins.add(args.path); + return { path: args.path, external: true }; + } + return { path: `node:${name}`, external: true }; + }); + build.onResolve({ filter: /\.node$/ }, (args) => ({ + errors: [{ + text: `Native addons are not supported in API routes: ${JSON.stringify(args.path)}`, + }], + })); + }, + }; +} + +const DECLARES_PATH_BINDING = + /\b(?:var|let|const|function|class)\s+(?:__dirname|__filename)\b|\bimport\s[^;]*\b(?:__dirname|__filename)\b/; + +/** + * Declarations of `__dirname` and `__filename` for a bundled dependency file + * that reads them without declaring them, or "" when it needs none. esbuild + * wraps each CommonJS file in its own function, so the values stay per file. + */ +export function commonJSPathPrelude(contents: string, filePath: string): string { + const usesDirname = /\b__dirname\b/.test(contents); + const usesFilename = /\b__filename\b/.test(contents); + if (!usesDirname && !usesFilename) return ""; + if (DECLARES_PATH_BINDING.test(contents)) return ""; + const dir = filePath.slice(0, Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\"))); + return `var __dirname = ${JSON.stringify(dir)}, __filename = ${JSON.stringify(filePath)};\n`; +} + +/** + * Binds the module-level `require` that esbuild's output calls for an external + * `require()` to the built-ins recorded during the build. Any other id throws + * the error Node gives for a missing module, so optional dependencies probed + * in a `try` still fall back. esbuild renames route bindings named `require`, + * so the prologue cannot collide with route code. + */ +export function prependBuiltinRequire(code: string, requiredBuiltins: Iterable): string { + const ids = [...requiredBuiltins].sort(); + let prefix = "__vf_builtin"; + while (code.includes(prefix)) prefix += "_"; + + const lines = ids.map((id, index) => + `import * as ${prefix}${index} from ${JSON.stringify(`node:${builtinModuleName(id)}`)};` + ); + const entries = ids.map((id, index) => + `${JSON.stringify(id)}: ${prefix}${index}.default ?? ${prefix}${index}` + ); + lines.push( + `const ${prefix}s = { __proto__: null${entries.map((entry) => `, ${entry}`).join("")} };`, + "const require = (id) => {", + // The table has no prototype, so `in` reads only its own keys and does + // not depend on built-ins route code could replace. + ` if (typeof id === "string" && id in ${prefix}s) return ${prefix}s[id];`, + ' const error = new Error("Cannot find module " + JSON.stringify(String(id)));', + ' error.code = "MODULE_NOT_FOUND";', + " throw error;", + "};", + ); + + // A hashbang is only valid as the first line. + const hashbangEnd = code.startsWith("#!") ? code.indexOf("\n") + 1 : 0; + return code.slice(0, hashbangEnd) + lines.join("\n") + "\n" + code.slice(hashbangEnd); +} diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index b707eb8509..f28d72de40 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -43,11 +43,16 @@ import { withSpan } from "#veryfront/observability/tracing/otlp-setup.ts"; import { wrapWithCurrentContext } from "#veryfront/platform/adapters/fs/veryfront/multi-project-adapter.ts"; import { isWithinDirectory } from "#veryfront/security/path-validation.ts"; import { - generateCompiledBinaryRequireShim, NODE_BUILTINS, readProjectDependencies, + resolveEsmUserDependencies as resolveEsmUserDependencyLocations, rewriteExternalImports, } from "./external-import-rewriter.ts"; +import { + commonJSPathPrelude, + createCompiledBuiltinPlugin, + prependBuiltinRequire, +} from "./compiled-cjs-bundling.ts"; import { MAX_WORKER_MODULE_SOURCE_BYTES, type PreparedWorkerModule, @@ -1864,6 +1869,7 @@ function createProjectBoundaryPlugin( routeModulePath: string, allowedHosts: string[], workerImportMap: DenoImportMap | null, + definesCommonJSPaths = false, ): Plugin { return { name: "vf-project-boundary", @@ -1872,10 +1878,11 @@ function createProjectBoundaryPlugin( try { const source = await sourceSnapshot.read(args.path); const executableModule = !isJSONModulePath(source.logicalPath); + const commonJS = mayBeCommonJSDependency(source.logicalPath); // JSON is parsed as data, never executed; see the note above. if (executableModule) { const scan = await validateHTTPImports(source.contents, allowedHosts, { - commonJS: mayBeCommonJSDependency(source.logicalPath), + commonJS, path: projectModuleLabel(source.logicalPath, projectDir), }); await validateBundledLocalWorkerEntries({ @@ -1888,9 +1895,12 @@ function createProjectBoundaryPlugin( importMap: workerImportMap, }); } + const pathPrelude = executableModule && commonJS && definesCommonJSPaths + ? commonJSPathPrelude(source.contents, source.logicalPath) + : ""; return { contents: executableModule - ? await rewriteBundledImportMetaUrl( + ? pathPrelude + await rewriteBundledImportMetaUrl( source.contents, pathHelper.toFileUrl(source.logicalPath).href, workerImportMap, @@ -2069,22 +2079,25 @@ function buildTranspiledModuleSource( // rewritten to a resolvable specifier. Excluding it left a bare `import "zod"` // in the temp handler that Deno cannot resolve → "not a dependency and not in // import map" → 500. (The Node path already always-resolves zod via - // getNodeExternalPackagesToResolve.) On the compiled binary, keeping zod in - // userDeps routes it through rewriteCompiledBinaryUserDependencyImports like - // any other npm package, so its import resolves from the project's - // node_modules via the createRequire shim. zod is still force-externalized - // below, never bundled inline. See veryfront-issue-inbox#217. + // getNodeExternalPackagesToResolve.) On the compiled binary, zod is handled + // like any other npm package: imported from node_modules when it is ESM, + // bundled when it is CommonJS. Elsewhere it is force-externalized below, + // never bundled inline. See veryfront-issue-inbox#217. const userDeps = getUserDependencies(allDeps); + const compiled = isDeno && isCompiledBinary(); - // Always externalize user npm dependencies. The bundled handler is loaded - // from a temp file and user deps are resolved at runtime: + // User npm dependencies the bundled handler resolves at runtime: // - Node.js: via file:// URLs pointing to node_modules - // - Deno (compiled or not): via createRequire or npm: specifiers - // Bundling CJS deps inline (especially complex ones like pdf-parse/pdf.js) - // breaks their internal global state management during esbuild's CJS→ESM - // conversion. + // - Deno source run: via npm: specifiers + // - Compiled binary: ESM dependencies via file:// URLs. A compiled + // binary cannot load a CommonJS file from disk at runtime, so its + // CommonJS dependencies are bundled into the handler instead, with + // esbuild's Node resolution and interop (see compiled-cjs-bundling.ts). + const runtimeDeps = compiled + ? await externalEsmUserDependencies(projectDir, sourceSnapshot, userDeps) + : userDeps; const userExternals: string[] = []; - for (const name of userDeps.keys()) { + for (const name of runtimeDeps.keys()) { userExternals.push(name, `${name}/*`); } @@ -2094,13 +2107,14 @@ function buildTranspiledModuleSource( // When esbuild bundles CJS into ESM output, these become __require() shims that // fail at runtime. Inject a createRequire-based shim so require() works in ESM. // Use projectDir as the resolve base so require() finds the project's node_modules. + // The compiled binary binds `require` after the build instead, from the + // built-ins the bundle actually requires. const safeProjectDir = JSON.stringify(projectDir + "/package.json"); - const requireShim = isDeno && isCompiledBinary() - ? generateCompiledBinaryRequireShim(projectDir) - : [ - 'import { createRequire as __vf_createRequire } from "node:module";', - `var require = __vf_createRequire(${safeProjectDir});`, - ].join("\n"); + const requireShim = compiled ? "" : [ + 'import { createRequire as __vf_createRequire } from "node:module";', + `var require = __vf_createRequire(${safeProjectDir});`, + ].join("\n"); + const requiredBuiltins = new Set(); const routeSourceUrl = pathHelper.toFileUrl(resolvedPath).href; const workerShim = [ `const __vf_routeSourceUrl = ${JSON.stringify(routeSourceUrl)};`, @@ -2113,7 +2127,7 @@ function buildTranspiledModuleSource( const routeRuntimeShim = [ requireShim, workerShim, - "export { require, Worker };", + compiled ? "export { Worker };" : "export { require, Worker };", ].join("\n"); const buildOptions = typescriptDecoratorOptions @@ -2128,13 +2142,18 @@ function buildTranspiledModuleSource( target: "es2022", jsx: "automatic", jsxImportSource: "react", - resolveExtensions: [".ts", ".tsx", ".js", ".jsx", ".mjs"], + resolveExtensions: compiled + ? [".ts", ".tsx", ".js", ".jsx", ".mjs", ".json"] + : [".ts", ".tsx", ".js", ".jsx", ".mjs"], + // Bundled CommonJS packages resolve as Node resolves them for require(). + ...(compiled ? { mainFields: ["main", "module"], conditions: ["node"] } : {}), // Injection binds only free identifiers and lets esbuild rename either // side of a collision. A raw banner would redeclare a route import such // as `import { Worker } from "worker-package"` and emit invalid ESM. inject: [ROUTE_RUNTIME_SHIM_SPECIFIER], external: [ - "zod", + // The compiled binary bundles a CommonJS zod like any other package. + ...(compiled ? [] : ["zod"]), "node:*", ...NODE_BUILTINS, "veryfront", @@ -2149,6 +2168,7 @@ function buildTranspiledModuleSource( sourcefile: resolvedPath, }, plugins: [ + ...(compiled ? [createCompiledBuiltinPlugin(requiredBuiltins)] : []), createRouteRuntimeShimPlugin(routeRuntimeShim), createImportMapPlugin( projectDir, @@ -2185,6 +2205,7 @@ function buildTranspiledModuleSource( resolvedPath, allowedHosts, workerImportMap, + compiled, ), ], // Only the opt-in decorator transform needs a working directory: adding @@ -2202,10 +2223,13 @@ function buildTranspiledModuleSource( if (result.errors?.length) throw buildFailure(result.errors, projectDir, workingDir); logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); - const js = result.outputFiles?.[0]?.text ?? "export {}"; + const output = result.outputFiles?.[0]?.text ?? "export {}"; + const js = compiled ? prependBuiltinRequire(output, requiredBuiltins) : output; logger.debug(`transpiled size ${js.length} bytes`); - return await rewriteExternalImports(js, projectDir, sourceSnapshot, userDeps); + // Only dependencies left external are rewritten: bundled package code + // may contain text that looks like an import of a bundled dependency. + return await rewriteExternalImports(js, projectDir, sourceSnapshot, runtimeDeps); }, { "api.modulePath": modulePath, "api.projectDir": projectDir }, ); @@ -2251,6 +2275,23 @@ async function readFileWithExtensions( ); } +/** + * The user dependencies a compiled-binary route imports at runtime: the ESM + * ones. The build bundles the rest, which are CommonJS. + */ +async function externalEsmUserDependencies( + projectDir: string, + sourceSnapshot: ProjectSourceSnapshot, + userDeps: Map, +): Promise> { + const esmDeps = await resolveEsmUserDependencyLocations(projectDir, sourceSnapshot, userDeps); + const external = new Map(); + for (const [name, version] of userDeps) { + if (esmDeps.has(name)) external.set(name, version); + } + return external; +} + export function getUserDependencies( allDeps: ReadonlyMap, ): Map { @@ -2401,8 +2442,8 @@ async function importModuleFromCode( } } - // Note: user npm dependencies are externalized and loaded at runtime via - // a custom CJS loader (see generateCompiledBinaryRequireShim), no shims needed. + // Note: ESM user dependencies load from file:// URLs and CommonJS ones are + // bundled into the handler (see compiled-cjs-bundling.ts), no shims needed. } await fs.writeTextFile(tempFile, transformedCode); diff --git a/tests/integration/compiled-binary-e2e.suite.ts b/tests/integration/compiled-binary-e2e.suite.ts index 20444175dd..483011cfa7 100644 --- a/tests/integration/compiled-binary-e2e.suite.ts +++ b/tests/integration/compiled-binary-e2e.suite.ts @@ -780,6 +780,84 @@ export function GET() { }); }); + // veryfront-issue-inbox#4177: a compiled binary cannot load a CommonJS file + // from disk at runtime, so the route build bundles CommonJS dependencies. + it("should bundle CommonJS dependencies of compiled-binary API routes", async () => { + const projectDir = await createTestProject( + "api-cjs-deps-test", + ` +export default function Home() { + return
Home Page
; +} +`, + { + "package.json": JSON.stringify({ + name: "test-api-cjs-deps", + type: "module", + dependencies: { + react: "^19.0.0", + "react-dom": "^19.0.0", + legacy: "1.0.0", + "esm-only": "1.0.0", + }, + }), + "node_modules/legacy/package.json": JSON.stringify({ + name: "legacy", + version: "1.0.0", + exports: { ".": { require: "./lib/index.js", default: "./lib/index.js" } }, + }), + "node_modules/legacy/lib/index.js": `"use strict"; +var fs = require("fs"); +var path = require("path"); +var data = require("./data.json"); +var optional; +try { optional = require("not-installed-optional"); } catch (error) { optional = error.code; } +exports.name = data.name; +exports.ownFile = path.basename(__filename); +exports.readsSibling = fs.readFileSync(path.join(__dirname, "data.json"), "utf8").length > 0; +exports.optional = optional; +`, + "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), + "node_modules/esm-only/package.json": JSON.stringify({ + name: "esm-only", + version: "1.0.0", + type: "module", + exports: "./index.js", + }), + "node_modules/esm-only/index.js": `export const kind = "esm";\n`, + "pages/api/cjs.ts": ` +import legacy, { name } from "legacy"; +import { kind } from "esm-only"; + +export function GET() { + return Response.json({ legacy, name, kind }); +} +`, + }, + ); + + await withServer(projectDir, async (server) => { + const response = await fetch(`http://127.0.0.1:${server.port}/api/cjs`); + const body = await response.text(); + + assertEquals( + response.status, + 200, + `Should return 200\nResponse body: ${body}\n${server.logs.join("").slice(-16000)}`, + ); + assertEquals(JSON.parse(body), { + legacy: { + name: "legacy-data", + ownFile: "index.js", + readsSibling: true, + optional: "MODULE_NOT_FOUND", + }, + name: "legacy-data", + kind: "esm", + }); + }); + }); + it("fails closed when API isolation conflicts with a broad host execution grant", async () => { const projectDir = await createTestProject( "isolation-fail-closed-test", From 0a9f18da471de8c391d33965bbe0787c52863416 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 19:42:17 +0100 Subject: [PATCH 02/29] fix(routing): resolve and load bundled dependencies as Node does Files under the project's node_modules that a compiled-binary route bundles now resolve their own imports with esbuild's Node resolution (directory indexes, extensionless and .json requires) instead of the project's import map, @/ alias and adapter resolver, and are not held to the route-code import rules, which rejected ordinary package code such as computed property reads (mime-types, jsonwebtoken, semver). Elsewhere these dependencies load at runtime unscanned; they still load only from inside the project. Refs veryfront/veryfront-issue-inbox#4177 --- src/routing/api/module-loader/loader.ts | 70 +++++++++++++++++-- .../integration/compiled-binary-e2e.suite.ts | 11 ++- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index f28d72de40..536436b67a 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -1305,6 +1305,28 @@ async function loadValidatedJSModule( ); } +/** Whether a path the bundler resolved is an installed dependency's file. */ +type DependencyFilePredicate = (path: string) => boolean; + +const noDependencyFiles: DependencyFilePredicate = () => false; + +/** + * Files under the project's own node_modules, which a compiled-binary route + * bundles. Both the given and the canonical project root count: esbuild + * reports resolved paths with symlinks followed. + */ +async function projectDependencyFiles(projectDir: string): Promise { + const roots = new Set([pathHelper.resolve(projectDir)]); + try { + roots.add(await realPath(projectDir)); + } catch { + // expected: an adapter-backed project may have no host directory + } + const prefixes = [...roots].map((root) => pathHelper.join(root, "node_modules") + pathHelper.sep); + return (path) => + path !== "" && prefixes.some((prefix) => pathHelper.resolve(path).startsWith(prefix)); +} + function createImportMapPlugin( projectDir: string, sourceSnapshot: ProjectSourceSnapshot, @@ -1313,6 +1335,7 @@ function createImportMapPlugin( denoImports: DenoImportMap, workerImportMap: DenoImportMap | null, config?: VeryfrontConfig, + isDependencyFile: DependencyFilePredicate = noDependencyFiles, ): Plugin { // A project's own veryfront config wins over its Deno config for the same // specifier, matching how the rest of the build reads resolve options. @@ -1332,6 +1355,9 @@ function createImportMapPlugin( setup(build) { build.onResolve({ filter: /.*/ }, (args) => { if (args.path.startsWith("node:")) return { path: args.path, external: true }; + // A dependency resolves its own imports as Node does: the project's + // import map does not apply inside node_modules. + if (isDependencyFile(args.importer)) return undefined; if ( !REMOTE_URL_SPECIFIER.test(args.path) && @@ -1733,6 +1759,7 @@ function createProjectAliasPlugin( routeModulePath: string, allowedHosts: string[], workerImportMap: DenoImportMap | null, + isDependencyFile: DependencyFilePredicate = noDependencyFiles, ): Plugin { const projectRoot = pathHelper.resolve(projectDir); @@ -1740,6 +1767,8 @@ function createProjectAliasPlugin( name: "vf-project-alias", setup(build) { build.onResolve({ filter: /^@\// }, (args) => { + // `@/` names a project file only in project code, never in a dependency. + if (isDependencyFile(args.importer)) return undefined; const aliasedPath = args.path.slice(2); const absolutePath = pathHelper.resolve(projectRoot, modulePathOfSpecifier(aliasedPath)); if (!isWithinDirectory(projectRoot, absolutePath)) { @@ -1780,12 +1809,16 @@ function createAdapterResolvePlugin( routeModulePath: string, allowedHosts: string[], workerImportMap: DenoImportMap | null, + isDependencyFile: DependencyFilePredicate = noDependencyFiles, ): Plugin { return { name: "vf-adapter-resolve", setup(build) { build.onResolve({ filter: /^\.\.?\// }, (args) => { if (args.namespace === "http-url" || args.namespace === "import-map") return undefined; + // Node resolves a dependency's relative imports, including directory + // indexes and .json files; the project boundary still loads them. + if (isDependencyFile(args.importer)) return undefined; const baseDir = args.importer ? pathHelper.dirname(args.importer) : args.resolveDir; if (!baseDir) return undefined; @@ -1869,7 +1902,7 @@ function createProjectBoundaryPlugin( routeModulePath: string, allowedHosts: string[], workerImportMap: DenoImportMap | null, - definesCommonJSPaths = false, + isDependencyFile: DependencyFilePredicate = noDependencyFiles, ): Plugin { return { name: "vf-project-boundary", @@ -1879,8 +1912,14 @@ function createProjectBoundaryPlugin( const source = await sourceSnapshot.read(args.path); const executableModule = !isJSONModulePath(source.logicalPath); const commonJS = mayBeCommonJSDependency(source.logicalPath); + // An installed dependency the compiled binary bundles gets the trust + // it has everywhere else, where it loads at runtime unscanned: its + // code is not held to the route-code rules, which reject ordinary + // package code such as computed property reads. It still loads only + // from inside the project. + const dependency = isDependencyFile(args.path); // JSON is parsed as data, never executed; see the note above. - if (executableModule) { + if (executableModule && !dependency) { const scan = await validateHTTPImports(source.contents, allowedHosts, { commonJS, path: projectModuleLabel(source.logicalPath, projectDir), @@ -1895,12 +1934,23 @@ function createProjectBoundaryPlugin( importMap: workerImportMap, }); } - const pathPrelude = executableModule && commonJS && definesCommonJSPaths - ? commonJSPathPrelude(source.contents, source.logicalPath) - : ""; + if (executableModule && dependency) { + return { + contents: commonJSPathPrelude(source.contents, source.logicalPath) + + (/\bimport\.meta\b/.test(source.contents) + ? await rewriteBundledImportMetaUrl( + source.contents, + pathHelper.toFileUrl(source.logicalPath).href, + null, + ) + : source.contents), + loader: getLoaderForFile(source.logicalPath), + resolveDir: pathHelper.dirname(source.logicalPath), + }; + } return { contents: executableModule - ? pathPrelude + await rewriteBundledImportMetaUrl( + ? await rewriteBundledImportMetaUrl( source.contents, pathHelper.toFileUrl(source.logicalPath).href, workerImportMap, @@ -2115,6 +2165,9 @@ function buildTranspiledModuleSource( `var require = __vf_createRequire(${safeProjectDir});`, ].join("\n"); const requiredBuiltins = new Set(); + const isDependencyFile = compiled + ? await projectDependencyFiles(projectDir) + : noDependencyFiles; const routeSourceUrl = pathHelper.toFileUrl(resolvedPath).href; const workerShim = [ `const __vf_routeSourceUrl = ${JSON.stringify(routeSourceUrl)};`, @@ -2178,6 +2231,7 @@ function buildTranspiledModuleSource( denoImports, workerImportMap, config, + isDependencyFile, ), createProjectAliasPlugin( sourceSnapshot, @@ -2185,6 +2239,7 @@ function buildTranspiledModuleSource( resolvedPath, allowedHosts, workerImportMap, + isDependencyFile, ), createAdapterResolvePlugin( sourceSnapshot, @@ -2192,6 +2247,7 @@ function buildTranspiledModuleSource( resolvedPath, allowedHosts, workerImportMap, + isDependencyFile, ), createHTTPPlugin({ allowedHosts, @@ -2205,7 +2261,7 @@ function buildTranspiledModuleSource( resolvedPath, allowedHosts, workerImportMap, - compiled, + isDependencyFile, ), ], // Only the opt-in decorator transform needs a working directory: adding diff --git a/tests/integration/compiled-binary-e2e.suite.ts b/tests/integration/compiled-binary-e2e.suite.ts index 483011cfa7..db3cfda872 100644 --- a/tests/integration/compiled-binary-e2e.suite.ts +++ b/tests/integration/compiled-binary-e2e.suite.ts @@ -806,18 +806,26 @@ export default function Home() { version: "1.0.0", exports: { ".": { require: "./lib/index.js", default: "./lib/index.js" } }, }), + // Ordinary package code the route-code checks reject: computed reads, + // directory and extensionless requires. "node_modules/legacy/lib/index.js": `"use strict"; var fs = require("fs"); var path = require("path"); -var data = require("./data.json"); +var data = require("./data"); +var util = require("./util"); var optional; try { optional = require("not-installed-optional"); } catch (error) { optional = error.code; } exports.name = data.name; exports.ownFile = path.basename(__filename); exports.readsSibling = fs.readFileSync(path.join(__dirname, "data.json"), "utf8").length > 0; exports.optional = optional; +exports.picked = util.pick(data, "name"); `, "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), + "node_modules/legacy/lib/util/index.js": `exports.pick = function (object, key) { + return object[key]; +}; +`, "node_modules/esm-only/package.json": JSON.stringify({ name: "esm-only", version: "1.0.0", @@ -851,6 +859,7 @@ export function GET() { ownFile: "index.js", readsSibling: true, optional: "MODULE_NOT_FOUND", + picked: "legacy-data", }, name: "legacy-data", kind: "esm", From f58844a58f917a65fc2653fe525fa5c960fdf88d Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 19:46:24 +0100 Subject: [PATCH 03/29] refactor(routing): remove the runtime CommonJS loader for compiled-binary routes The route build now bundles CommonJS dependencies, so the generated runtime loader (generateCompiledBinaryRequireShim), the require() rewrite of CommonJS dependency imports and its destructuring helper are unused. Remove them and their tests. Project-boundary symlink containment stays covered by the source snapshot tests. Move the esbuild bundling tests to tests/integration/semantic-unit-boundary, since they write temp files; the unit file keeps the pure helpers. Refs veryfront/veryfront-issue-inbox#4177 --- .../compiled-cjs-bundling.test.ts | 120 +-------- .../external-import-rewriter.test.ts | 63 +---- .../module-loader/external-import-rewriter.ts | 103 ------- .../api/module-loader/loader-helpers.test.ts | 10 - .../api/module-loader/loader-helpers.ts | 17 -- src/routing/api/module-loader/loader.test.ts | 251 +----------------- src/routing/api/module-loader/loader.ts | 3 - src/transforms/import-rewriter/core.test.ts | 5 +- .../import-rewriter/route-adapter.ts | 63 +---- .../compiled-cjs-bundling.test.ts | 128 +++++++++ 10 files changed, 146 insertions(+), 617 deletions(-) create mode 100644 tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 28a54fa1ee..c8270918ee 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,64 +1,12 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { assert, assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; -import * as esbuild from "veryfront/extensions/bundler"; -import type { Plugin } from "veryfront/extensions/bundler"; import { builtinModuleName, commonJSPathPrelude, - createCompiledBuiltinPlugin, prependBuiltinRequire, } from "./compiled-cjs-bundling.ts"; -async function writeFiles(root: string, files: Record): Promise { - for (const [relative, contents] of Object.entries(files)) { - const path = `${root}/${relative}`; - await Deno.mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true }); - await Deno.writeTextFile(path, contents); - } -} - -/** Defines __dirname/__filename for node_modules files, as the route build does. */ -const pathPreludePlugin: Plugin = { - name: "test-path-prelude", - setup(build) { - build.onLoad({ filter: /node_modules[\\/].*\.c?js$/ }, async (args) => { - const contents = await Deno.readTextFile(args.path); - return { contents: commonJSPathPrelude(contents, args.path) + contents, loader: "js" }; - }); - }, -}; - -/** Bundles a route the way the compiled binary does and imports the result. */ -async function bundleAndImport( - projectDir: string, - route: string, -): Promise> { - const requiredBuiltins = new Set(); - const result = await esbuild.build({ - bundle: true, - write: false, - format: "esm", - platform: "neutral", - target: "es2022", - mainFields: ["main", "module"], - conditions: ["node"], - resolveExtensions: [".ts", ".js", ".mjs", ".json"], - logLevel: "silent", - stdin: { - contents: route, - loader: "ts", - resolveDir: projectDir, - sourcefile: `${projectDir}/route.ts`, - }, - plugins: [createCompiledBuiltinPlugin(requiredBuiltins), pathPreludePlugin], - }); - const code = prependBuiltinRequire(result.outputFiles?.[0]?.text ?? "", requiredBuiltins); - const file = `${projectDir}/handler.mjs`; - await Deno.writeTextFile(file, code); - return await import(`file://${file}`); -} - describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { describe("builtinModuleName()", () => { it("names built-ins with or without the node: prefix, including subpaths", () => { @@ -103,70 +51,4 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(code.includes("import * as __vf_builtin_0 from"), code); }); }); - - describe("bundling a route's CommonJS dependency", () => { - it("runs exports maps, built-in and JSON requires, __dirname and optional requires", async () => { - const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); - try { - await writeFiles(projectDir, { - "node_modules/legacy/package.json": JSON.stringify({ - name: "legacy", - // The node condition must win over default, as it does in Node. - exports: { ".": { node: "./lib/index.js", default: "./lib/missing.js" } }, - }), - "node_modules/legacy/lib/index.js": [ - '"use strict";', - 'var fs = require("fs");', - 'var path = require("node:path");', - 'var data = require("./data");', - "var optional;", - 'try { optional = require("not-installed"); } catch (error) { optional = error.code; }', - "exports.name = data.name;", - "exports.ownFile = path.basename(__filename);", - 'exports.readsSibling = fs.existsSync(path.join(__dirname, "data.json"));', - "exports.optional = optional;", - ].join("\n"), - "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), - }); - const mod = await bundleAndImport( - projectDir, - 'const require = (id: string) => "route:" + id;\n' + - 'export const routeRequire = require("x");\n' + - 'import legacy from "legacy";\nexport { legacy };\n', - ); - assertEquals(mod.legacy, { - name: "legacy-data", - ownFile: "index.js", - readsSibling: true, - optional: "MODULE_NOT_FOUND", - }); - // esbuild renames a route's own `require`, so the prologue cannot clash. - assertEquals(mod.routeRequire, "route:x"); - } finally { - await esbuild.stop(); - await Deno.remove(projectDir, { recursive: true }); - } - }); - - it("rejects a native addon at build time", async () => { - const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); - try { - await writeFiles(projectDir, { - "node_modules/native/package.json": JSON.stringify({ name: "native", main: "index.js" }), - "node_modules/native/index.js": 'module.exports = require("./build/binding.node");', - "node_modules/native/build/binding.node": "not a real addon", - }); - const error = await assertRejects(() => - bundleAndImport(projectDir, 'import native from "native";\nexport { native };\n') - ); - assert( - String((error as Error).message).includes("Native addons are not supported"), - String(error), - ); - } finally { - await esbuild.stop(); - await Deno.remove(projectDir, { recursive: true }); - } - }); - }); }); diff --git a/src/routing/api/module-loader/external-import-rewriter.test.ts b/src/routing/api/module-loader/external-import-rewriter.test.ts index 2da0bd57e5..a8b62aebb6 100644 --- a/src/routing/api/module-loader/external-import-rewriter.test.ts +++ b/src/routing/api/module-loader/external-import-rewriter.test.ts @@ -1,6 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; import { - assert, assertEquals, assertStringIncludes, assertThrows, @@ -9,9 +8,7 @@ import { describe, it } from "#veryfront/testing/bdd.ts"; import type { FileInfo } from "#veryfront/platform/adapters/base.ts"; import type { FileSystem } from "#veryfront/platform/compat/fs.ts"; import { - generateCompiledBinaryRequireShim, getNodeExternalPackagesToResolve, - NODE_BUILTINS, resolveEsmUserDependencies, resolveVeryfrontPackageExport, rewriteCompiledBinaryUserDependencyImports, @@ -116,45 +113,12 @@ describe("external-import-rewriter", () => { describe("rewriteCompiledBinaryUserDependencyImports", () => { const deps = new Map([["lodash", "^4"]]); - it("rewrites a default import to interop require", () => { - const out = rewriteCompiledBinaryUserDependencyImports( - `import _ from "lodash";`, - deps, - ); - assertEquals(out, `const _ = __vf_interopDefault(require("lodash"));`); - }); - - it("rewrites a namespace import to require", () => { - const out = rewriteCompiledBinaryUserDependencyImports( - `import * as _ from "lodash";`, - deps, - ); - assertEquals(out, `const _ = require("lodash");`); - }); - - it("rewrites a named import to a destructuring require", () => { - const out = rewriteCompiledBinaryUserDependencyImports( - `import { merge } from "lodash";`, - deps, - ); - assertStringIncludes(out, `= require("lodash")`); - assertStringIncludes(out, "merge"); - }); - - it("rewrites a dynamic import to a resolved require promise", () => { - const out = rewriteCompiledBinaryUserDependencyImports( - `const m = import("lodash");`, - deps, - ); - assertStringIncludes(out, `Promise.resolve(require("lodash"))`); - }); - - it("rewrites a subpath import including the subpath in require", () => { - const out = rewriteCompiledBinaryUserDependencyImports( - `import merge from "lodash/merge";`, - deps, - ); - assertStringIncludes(out, `require("lodash/merge")`); + // The route build bundles CommonJS dependencies (inbox#4177), so an import + // of one never reaches this rewrite; it must not be turned into require(). + it("leaves imports of CommonJS dependencies untouched", () => { + const code = + `import _ from "lodash";\nimport merge from "lodash/merge";\nconst m = import("lodash");`; + assertEquals(rewriteCompiledBinaryUserDependencyImports(code, deps), code); }); it("leaves imports for packages not in userDeps untouched", () => { @@ -383,21 +347,6 @@ describe("external-import-rewriter", () => { }); }); - describe("generateCompiledBinaryRequireShim", () => { - it("emits a self-contained CJS require shim referencing the project dir", () => { - const shim = generateCompiledBinaryRequireShim("/srv/app"); - // Declares the require shim and embeds the project package.json path. - assertStringIncludes(shim, "__vf_createRequire"); - assertStringIncludes(shim, "/srv/app/package.json"); - // Embeds the node builtin set used for routing builtin requires. - assertStringIncludes(shim, JSON.stringify(NODE_BUILTINS)); - // Defines a require function and an interop helper. - assertStringIncludes(shim, "var require = function(id)"); - assertStringIncludes(shim, "__vf_interopDefault"); - assert(shim.length > 0); - }); - }); - describe("Node Veryfront package imports", () => { // An API route that imports `veryfront/tool` must reach the SAME module // instance the running server uses, or `toolRegistry.get()` looks into a diff --git a/src/routing/api/module-loader/external-import-rewriter.ts b/src/routing/api/module-loader/external-import-rewriter.ts index 00b9bd3931..873ac6f936 100644 --- a/src/routing/api/module-loader/external-import-rewriter.ts +++ b/src/routing/api/module-loader/external-import-rewriter.ts @@ -50,109 +50,6 @@ export async function readProjectDependencies( return await readProjectDependenciesForRoute(projectDir, fs); } -/** - * Generates a CJS module loader shim for compiled Deno binaries. - * - * In compiled binaries, `createRequire()` can resolve module paths and load - * built-in modules (fs, path, etc.), but cannot load CJS files from disk - * (loadMaybeCjs fails with "path not found"). This shim works around that - * limitation by using `Deno.readTextFileSync` to read CJS files and - * `new Function` to evaluate them in a proper CJS wrapper with require, - * exports, module, __filename, and __dirname bindings. - */ -export function generateCompiledBinaryRequireShim(projectDir: string): string { - const builtinSet = JSON.stringify(NODE_BUILTINS); - const safeProjectDir = JSON.stringify(projectDir + "/package.json"); - const safeProjectRoot = JSON.stringify(pathHelper.resolve(projectDir)); - - return ` -import { createRequire as __vf_createRequire } from "node:module"; -import { dirname as __vf_dirname, resolve as __vf_resolve } from "node:path"; -var __vf_builtinRequire = __vf_createRequire(${safeProjectDir}); -var __vf_builtinSet = new Set(${builtinSet}); -var __vf_projectRoot = ${safeProjectRoot}; -// VULN-FS-5: Canonicalize the project root so containment checks using -// Deno.realPathSync(resolved) compare canonical-vs-canonical. Without this, -// when the project itself is opened via a symlink, the realpath'd resolved -// module path has a different prefix than the non-canonical projectRoot and -// legitimate dependencies would be rejected. -try { __vf_projectRoot = Deno.realPathSync(__vf_projectRoot); } catch (_) { /* expected: projectRoot may not exist at shim init in some environments */ } -var __vf_cache = Object.create(null); -// Resolve symlinks so containment compares canonical paths. A module that does -// not exist yet cannot be canonicalized, so fall back to its parent directory, -// which is what carries a symlinked project root; only if that fails too is the -// plain resolved path used. -function __vf_canonicalize(candidate) { - try { - return Deno.realPathSync(candidate); - } catch (_) { /* expected: candidate may not exist */ } - try { - var parent = __vf_dirname(candidate); - return Deno.realPathSync(parent) + candidate.slice(parent.length); - } catch (_) { /* expected: parent may not exist either */ } - return __vf_resolve(candidate); -} -function __vf_assertContained(resolved) { - var norm = __vf_resolve(resolved).replace(/\\\\/g, "/"); - var root = __vf_projectRoot.replace(/\\\\/g, "/"); - if (!norm.startsWith(root + "/") && norm !== root) { - throw new Error("CJS loader blocked path outside project: " + resolved); - } -} -function __vf_loadCjs(id, parentDir) { - if (id.startsWith("node:")) return __vf_builtinRequire(id); - if (__vf_builtinSet.has(id)) return __vf_builtinRequire(id); - var slashIdx = id.indexOf("/"); - if (slashIdx > 0 && __vf_builtinSet.has(id.slice(0, slashIdx))) return __vf_builtinRequire(id); - var resolved; - if (id.startsWith(".") || id.startsWith("/")) { - resolved = __vf_resolve(parentDir, id); - if (!resolved.match(/\\.[a-zA-Z0-9]+$/)) { - var exts = [".js", ".cjs", ".json", "/index.js", "/index.cjs", "/index.json"]; - for (var i = 0; i < exts.length; i++) { - try { Deno.statSync(resolved + exts[i]); resolved += exts[i]; break; } catch (_) { /* expected: probing file extensions */ } - } - } - } else { - resolved = __vf_builtinRequire.resolve(id); - } - // VULN-FS-5: Canonicalize before asserting containment, so the candidate and - // the already-canonical project root are compared like with like, then assert - // once. Checking the non-canonical path first rejected dependencies genuinely - // inside a project whose root is reached through a symlink, which is the - // ordinary case for package managers, CI checkouts and macOS /tmp. - // Canonicalizing first also keeps the guard against symlinked node_modules - // entries pointing outside the root: the escape is only visible once resolved. - resolved = __vf_canonicalize(resolved); - __vf_assertContained(resolved); - if (resolved in __vf_cache) return __vf_cache[resolved]; - var code = Deno.readTextFileSync(resolved); - if (resolved.endsWith(".json")) { - var json = JSON.parse(code); - __vf_cache[resolved] = json; - return json; - } - var mod = { exports: {} }; - __vf_cache[resolved] = mod.exports; - var dir = __vf_dirname(resolved); - var childReq = function(childId) { return __vf_loadCjs(childId, dir); }; - childReq.resolve = function(childId) { - if (childId.startsWith(".") || childId.startsWith("/")) return __vf_resolve(dir, childId); - return __vf_builtinRequire.resolve(childId); - }; - childReq.ensure = function(mods, cb) { cb(); }; - var fn = new Function("exports", "require", "module", "__filename", "__dirname", "global", "globalThis", "Worker", code); - fn(mod.exports, childReq, mod, resolved, dir, globalThis, globalThis, undefined); - __vf_cache[resolved] = mod.exports; - return mod.exports; -} -function __vf_interopDefault(m) { return m && m.__esModule && m.default !== undefined ? m.default : m; } -var require = function(id) { return __vf_loadCjs(id, ${JSON.stringify(projectDir)}); }; -require.resolve = function(id) { return __vf_builtinRequire.resolve(id); }; -require.ensure = function(mods, cb) { cb(); }; -`.trim(); -} - export function getNodeExternalPackagesToResolve(userDeps: Map): string[] { return getNodeExternalPackagesToResolveForRoute(userDeps); } diff --git a/src/routing/api/module-loader/loader-helpers.test.ts b/src/routing/api/module-loader/loader-helpers.test.ts index d58a6aebd7..91badec5dc 100644 --- a/src/routing/api/module-loader/loader-helpers.test.ts +++ b/src/routing/api/module-loader/loader-helpers.test.ts @@ -5,7 +5,6 @@ import { FILE_EXTENSIONS, getLoaderForFile, resolveExportEntry, - toCjsDestructureBindings, validateModulePath, } from "./loader-helpers.ts"; @@ -34,15 +33,6 @@ describe("routing/api/module-loader helpers", () => { assertEquals(resolveExportEntry({ require: "./cjs.js" }), undefined); }); - it("converts CJS destructuring bindings", () => { - assertEquals( - toCjsDestructureBindings("{ parse as parsePdf, version }"), - "{ parse: parsePdf, version }", - ); - assertEquals(toCjsDestructureBindings("{ foo, bar }"), "{ foo, bar }"); - assertEquals(toCjsDestructureBindings("{ }"), "{}"); - }); - it("returns the expected file loaders and extensions", () => { assertEquals( FILE_EXTENSIONS, diff --git a/src/routing/api/module-loader/loader-helpers.ts b/src/routing/api/module-loader/loader-helpers.ts index bfba9f40df..1663624ccb 100644 --- a/src/routing/api/module-loader/loader-helpers.ts +++ b/src/routing/api/module-loader/loader-helpers.ts @@ -46,23 +46,6 @@ export function resolveExportEntry(entry: unknown): string | undefined { return undefined; } -export function toCjsDestructureBindings(bindings: string): string { - const inner = bindings.trim().replace(/^\{\s*/, "").trimEnd().replace(/\}$/, ""); - if (!inner) return "{}"; - - const converted = inner - .split(",") - .map((part) => part.trim()) - .filter(Boolean) - .map((part) => { - const aliasMatch = part.match(/^([A-Za-z_$][\w$]*)\s+as\s+([A-Za-z_$][\w$]*)$/); - if (aliasMatch) return `${aliasMatch[1]}: ${aliasMatch[2]}`; - return part; - }); - - return `{ ${converted.join(", ")} }`; -} - export function getLoaderForFile(filePath: string): "tsx" | "jsx" | "ts" | "js" | "json" { const ext = filePath.split(".").pop() ?? ""; return EXT_TO_LOADER[ext] ?? "js"; diff --git a/src/routing/api/module-loader/loader.test.ts b/src/routing/api/module-loader/loader.test.ts index f1e2ff2cc9..1cc8044847 100644 --- a/src/routing/api/module-loader/loader.test.ts +++ b/src/routing/api/module-loader/loader.test.ts @@ -6,7 +6,6 @@ import { assertMatch, assertNotEquals, assertRejects, - assertThrows, } from "#veryfront/testing/assert.ts"; import { afterAll, describe, it } from "#veryfront/testing/bdd.ts"; import { join, toFileUrl } from "#veryfront/compat/path"; @@ -15,7 +14,6 @@ import { bundledModuleScopeDiscriminator, bundlerForcesTypeScript, canDirectImportSpecifier, - generateCompiledBinaryRequireShim, getNodeExternalPackagesToResolve, getUserDependencies, isBareModuleSpecifier, @@ -31,7 +29,6 @@ import { rewriteDenoNodeBuiltinImports, rewriteDenoNpmDependencyImports, rewriteNodeExternalImports, - toCjsDestructureBindings, typeScriptBuildOptions, } from "./loader.ts"; import { __setCompiledBinaryForTests } from "#veryfront/security/sandbox/isolation-capability.ts"; @@ -1668,21 +1665,6 @@ describe("loadHandlerModule", { sanitizeResources: false, sanitizeOps: false }, assertEquals(typeof route?.GET, "function"); }); - it("converts aliased named imports to valid CJS destructuring", () => { - assertEquals( - toCjsDestructureBindings("{ parse as parsePdf, version }"), - "{ parse: parsePdf, version }", - ); - assertEquals( - toCjsDestructureBindings("{ default as foo, bar as baz }"), - "{ default: foo, bar: baz }", - ); - assertEquals( - toCjsDestructureBindings("{ foo, bar }"), - "{ foo, bar }", - ); - }); - it("builds the node external package resolution list without duplicates", () => { const packages = getNodeExternalPackagesToResolve( new Map([ @@ -1823,32 +1805,6 @@ describe("loadHandlerModule", { sanitizeResources: false, sanitizeOps: false }, assertMatch(rewritten, /import\("\.\/_vf_tool\.mjs"\)/); }); - it("rewrites compiled-binary user dependency imports to require-based shims", () => { - const source = [ - 'import thing from "my-lib";', - 'import { alpha as beta } from "my-lib";', - 'import * as namespace from "my-lib";', - 'import combo, { gamma } from "my-lib";', - 'import widget from "my-lib/subpath";', - 'const loaded = import("my-lib/subpath");', - ].join("\n"); - - const rewritten = rewriteCompiledBinaryUserDependencyImports( - source, - new Map([["my-lib", "^1.0.0"]]), - ); - - assertMatch(rewritten, /const thing = __vf_interopDefault\(require\("my-lib"\)\)/); - assertMatch(rewritten, /const \{ alpha: beta \} = require\("my-lib"\)/); - assertMatch(rewritten, /const namespace = require\("my-lib"\)/); - assertMatch( - rewritten, - /const __vf_tmp_combo = require\("my-lib"\); const combo = __vf_interopDefault\(__vf_tmp_combo\); const \{ gamma \} = __vf_tmp_combo/, - ); - assertMatch(rewritten, /const widget = require\("my-lib\/subpath"\)/); - assertMatch(rewritten, /Promise\.resolve\(require\("my-lib\/subpath"\)\)/); - }); - it("rewrites ESM-only user dependency imports to real file:// module URLs", () => { const source = [ 'import thing from "esm-lib";', @@ -1895,7 +1851,7 @@ describe("loadHandlerModule", { sanitizeResources: false, sanitizeOps: false }, assertEquals(rewritten.includes('require("esm-lib")'), false); }); - it("keeps CJS deps on the require shim while ESM deps use file:// URLs", () => { + it("rewrites ESM deps to file:// URLs and leaves bundled CJS deps alone", () => { const source = [ 'import esm from "esm-lib";', 'import cjs from "cjs-lib";', @@ -1914,7 +1870,7 @@ describe("loadHandlerModule", { sanitizeResources: false, sanitizeOps: false }, ); assertMatch(rewritten, /import esm from "file:\/\/\/proj\/node_modules\/esm-lib\/index\.mjs"/); - assertMatch(rewritten, /const cjs = __vf_interopDefault\(require\("cjs-lib"\)\)/); + assertMatch(rewritten, /import cjs from "cjs-lib"/); }); it("detects ESM dependencies via type:module and .mjs entry points", async () => { @@ -5416,209 +5372,6 @@ describe("loadHandlerModule", { sanitizeResources: false, sanitizeOps: false }, }); }); -/** - * Load the emitted compiled-binary shim as a real module so its containment - * checks are exercised instead of being re-implemented by the test. - */ -async function importCompiledBinaryRequireShim( - projectDir: string, -): Promise<(id: string) => unknown> { - const shimPath = join(projectDir, "vf-require-shim.mjs"); - await fs.writeTextFile( - shimPath, - `${generateCompiledBinaryRequireShim(projectDir)}\nexport { require as vfRequire };\n`, - ); - const module = await import(toFileUrl(shimPath).href) as { - vfRequire: (id: string) => unknown; - }; - return module.vfRequire; -} - -// VULN-FS-5: compiled-binary CJS loader must enforce project-root containment -// on BOTH branches of __vf_loadCjs (relative/absolute ids AND bare-package -// ids), and must re-canonicalise via Deno.realPathSync so that a symlinked -// node_modules entry cannot escape the project root. -describe("generateCompiledBinaryRequireShim - static checks (VULN-FS-5)", () => { - it("emits a __vf_assertContained call after bare-package resolution", () => { - const shim = generateCompiledBinaryRequireShim("/fake/project"); - // The original (vulnerable) layout called __vf_assertContained only inside - // the relative/absolute branch. The fix moves the assertion to run after - // both branches (i.e. AFTER the `} else { resolved = ...resolve(id); }`). - const elseIdx = shim.indexOf("__vf_builtinRequire.resolve(id)"); - const assertIdx = shim.indexOf("__vf_assertContained(resolved)", elseIdx); - assertEquals( - assertIdx > elseIdx, - true, - "containment check must follow bare-package resolution", - ); - }); - - it("canonicalises the resolved path before checking containment", () => { - const shim = generateCompiledBinaryRequireShim("/fake/project"); - assertEquals(shim.includes("Deno.realPathSync"), true); - // The containment check must run on the canonical path. Asserting on the - // pre-canonical one instead compares a non-canonical candidate against an - // already-canonical root, which rejects legitimate dependencies whenever - // the project root is itself a symlink. - const canonIdx = shim.indexOf("resolved = __vf_canonicalize(resolved)"); - const assertIdx = shim.indexOf("__vf_assertContained(resolved)", canonIdx); - assertEquals( - canonIdx !== -1 && assertIdx > canonIdx, - true, - "the resolved path must be canonicalised before it is containment-checked", - ); - }); - - it("canonicalises __vf_projectRoot at shim init so symlinked project roots are not falsely rejected", () => { - // Regression: prior to the fix, __vf_projectRoot was only path.resolve()'d, - // so when realPathSync(resolved) returned a canonical path whose prefix - // differed from a symlinked projectRoot, every legitimate dep was blocked. - const shim = generateCompiledBinaryRequireShim("/fake/project"); - const rootInitIdx = shim.indexOf("var __vf_projectRoot ="); - const canonIdx = shim.indexOf( - "Deno.realPathSync(__vf_projectRoot)", - rootInitIdx, - ); - const fnIdx = shim.indexOf("function __vf_assertContained"); - assertEquals( - canonIdx > rootInitIdx && canonIdx < fnIdx, - true, - "__vf_projectRoot must be realPathSync'd between its declaration and the assertContained definition", - ); - }); - - denoIt("the emitted containment check rejects paths outside the project root", async () => { - // Execute the shim the generator actually emits: a local re-implementation - // would keep passing even if the emitted __vf_assertContained were gutted. - const projectDir = Deno.realPathSync(await makeTempDir()); - const packageDir = join(projectDir, "node_modules", "ok"); - await fs.mkdir(packageDir, { recursive: true }); - await fs.writeTextFile(join(packageDir, "index.js"), "module.exports = { ok: true };"); - - const vfRequire = await importCompiledBinaryRequireShim(projectDir); - - assertEquals( - (vfRequire(join(packageDir, "index.js")) as { ok: boolean }).ok, - true, - "in-project CJS must load", - ); - assertThrows( - () => vfRequire("/etc/passwd"), - Error, - "blocked path outside project", - "absolute host paths must be refused", - ); - assertThrows( - () => vfRequire(`${projectDir}ile/secret.js`), - Error, - "blocked path outside project", - "prefix-sibling dirs must be refused", - ); - assertThrows( - () => vfRequire(join(projectDir, "node_modules", "..", "..", "escape.js")), - Error, - "blocked path outside project", - "'..' segments must be normalised before the containment check", - ); - - try { - await fs.remove(projectDir, { recursive: true }); - } catch (_) { /* best effort */ } - }); -}); - -describe("generateCompiledBinaryRequireShim - symlink resistance (VULN-FS-5)", { - sanitizeResources: false, - sanitizeOps: false, -}, () => { - denoIt("re-canonicalisation via realPathSync catches a node_modules symlink escape", async () => { - // Create a project root, a decoy "evil" package whose entry file is a - // symlink pointing at a file outside the project root. If the shim only - // checked the pre-symlink path, the containment test would pass but the - // readTextFileSync would still leak the external file. The emitted shim is - // executed here so the realPathSync + second __vf_assertContained is the - // thing under test, not a copy of it. - const projectDir = Deno.realPathSync(await makeTempDir()); - const outsideDir = Deno.realPathSync(await makeTempDir()); - const outsideFile = join(outsideDir, "secret.txt"); - await fs.writeTextFile(outsideFile, "top-secret-contents"); - - const nodeModules = join(projectDir, "node_modules", "evil"); - await fs.mkdir(nodeModules, { recursive: true }); - const symlinkEntry = join(nodeModules, "index.js"); - try { - await Deno.symlink(outsideFile, symlinkEntry); - } catch (e) { - // On platforms without symlink permission, skip this test rather than - // misreport a failure. The static check above still covers the fix. - const msg = e instanceof Error ? e.message : String(e); - if (msg.includes("permission") || msg.includes("not supported")) return; - throw e; - } - - const vfRequire = await importCompiledBinaryRequireShim(projectDir); - - const error = assertThrows( - () => vfRequire(symlinkEntry), - Error, - "blocked path outside project", - "a symlinked dependency escaping the project root must be rejected", - ); - assertEquals( - (error as Error).message.includes("top-secret-contents"), - false, - "the outside file contents must never reach the caller", - ); - - // Clean up. - try { - await fs.remove(projectDir, { recursive: true }); - } catch (_) { /* best effort */ } - try { - await fs.remove(outsideDir, { recursive: true }); - } catch (_) { /* best effort */ } - }); - - denoIt("accepts dependencies through a symlinked project root", async () => { - // Regression for #4091. The previous version of this test re-implemented - // __vf_assertContained locally and only ever called it with the already - // canonicalised path, so it passed without ever exercising the check that - // actually rejected these dependencies. The emitted shim is executed here - // so the containment check itself is the thing under test. - const realProject = Deno.realPathSync(await makeTempDir()); - const symlinkedProject = `${realProject}-link`; - try { - await Deno.symlink(realProject, symlinkedProject); - } catch (e) { - const msg = e instanceof Error ? e.message : String(e); - if (msg.includes("permission") || msg.includes("not supported")) return; - throw e; - } - - const depDir = join(realProject, "node_modules", "ok"); - await fs.mkdir(depDir, { recursive: true }); - await fs.writeTextFile(join(depDir, "index.js"), "module.exports = { ok: true };"); - - // The project root is handed to the shim as the symlinked path, which is - // what happens with package managers, CI checkouts, and macOS /tmp. - const vfRequire = await importCompiledBinaryRequireShim(symlinkedProject); - - assertEquals( - (vfRequire(join(symlinkedProject, "node_modules", "ok", "index.js")) as { ok: boolean }) - .ok, - true, - "a dependency genuinely inside the project must load when the root is a symlink", - ); - - try { - await fs.remove(symlinkedProject); - } catch (_) { /* best effort */ } - try { - await fs.remove(realProject, { recursive: true }); - } catch (_) { /* best effort */ } - }); -}); - describe("isSpecifierResolutionError", () => { // Direct import leaves specifier resolution to the runtime, which knows // nothing about the project's `@/` alias — those routes used to 500 with a diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 536436b67a..cf21e61cec 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -71,7 +71,6 @@ import { tryGetRegistryScopeId } from "#veryfront/cache/cache-key-builder.ts"; import { getProjectEnvSnapshot } from "#veryfront/server/project-env/storage.ts"; import { currentRequestContext } from "#veryfront/platform/request-context-access.ts"; export { - generateCompiledBinaryRequireShim, getNodeExternalPackagesToResolve, loadVeryfrontExportsMap, resolveEsmUserDependencies, @@ -107,8 +106,6 @@ const MAX_TYPESCRIPT_CONFIG_BYTES = 1024 * 1024; const ROUTE_RUNTIME_SHIM_SPECIFIER = "veryfront:route-runtime-shim"; const ROUTE_RUNTIME_SHIM_NAMESPACE = "veryfront-route-runtime-shim"; -export { toCjsDestructureBindings } from "./loader-helpers.ts"; - export function loadHandlerModule(options: LoadHostModuleOptions): Promise { if (!isExplicitHostProjectCodeExecutionAllowed(options)) { return Promise.reject( diff --git a/src/transforms/import-rewriter/core.test.ts b/src/transforms/import-rewriter/core.test.ts index 208142d36d..c841d4368b 100644 --- a/src/transforms/import-rewriter/core.test.ts +++ b/src/transforms/import-rewriter/core.test.ts @@ -143,12 +143,13 @@ describe("import rewrite compatibility golden tests", () => { rewriteCompiledBinaryVeryfrontImports(`import { x } from "veryfront/agent";`), `import { x } from "./_vf_agent.mjs";`, ); - assertStringIncludes( + // CommonJS dependencies are bundled, so their imports are left alone. + assertEquals( rewriteCompiledBinaryUserDependencyImports( `const m = import("lodash/merge");`, new Map([["lodash", "^4"]]), ), - `Promise.resolve(require("lodash/merge"))`, + `const m = import("lodash/merge");`, ); assertEquals( rewriteDenoNodeBuiltinImports(`import { readFile } from "fs";`), diff --git a/src/transforms/import-rewriter/route-adapter.ts b/src/transforms/import-rewriter/route-adapter.ts index 178bde8147..4a7bd14595 100644 --- a/src/transforms/import-rewriter/route-adapter.ts +++ b/src/transforms/import-rewriter/route-adapter.ts @@ -4,10 +4,7 @@ import { serverLogger } from "#veryfront/utils"; import { parseImports, replaceSpecifiers } from "#veryfront/transforms/esm/lexer.ts"; import { resolveContainedPackagePath } from "./package-resolution.ts"; import { isWithinDirectory } from "#veryfront/security/path-validation.ts"; -import { - resolveExportEntry as resolveRouteExportEntry, - toCjsDestructureBindings, -} from "#veryfront/routing/api/module-loader/loader-helpers.ts"; +import { resolveExportEntry as resolveRouteExportEntry } from "#veryfront/routing/api/module-loader/loader-helpers.ts"; import { rethrowProjectBoundaryViolation } from "#veryfront/routing/api/module-loader/project-source-snapshot.ts"; const logger = serverLogger.component("api"); @@ -126,8 +123,7 @@ function resolveEsmEntry(pkgJson: Record): string | undefined { /** * Identify the subset of user dependencies that are ESM-only and resolve each * to file:// URLs so the compiled-binary loader can import them as real ES - * modules instead of transpiling them to CommonJS. CJS dependencies are omitted - * and continue to load through the `createRequire`-based shim. + * modules. CommonJS dependencies are omitted: the route build bundles them. */ export async function resolveEsmUserDependenciesForRoute( projectDir: string, @@ -150,8 +146,7 @@ export async function resolveEsmUserDependenciesForRoute( // attacker-influenceable (a malicious/compromised package could set // "main": "../../../etc/passwd"). Reject entries that resolve outside the // package directory so a crafted package.json cannot turn into a file:// - // import that escapes node_modules. This mirrors the containment guard the - // CJS loader shim enforces via __vf_assertContained. + // import that escapes node_modules. const entryPath = pathHelper.resolve(pathHelper.join(packageDir, entry)); if (!isWithinDirectory(packageDir, entryPath)) { logger.warn(`Skipping ESM dependency ${name}: entry escapes package directory (${entry})`); @@ -204,9 +199,9 @@ export function rewriteCompiledUserDependencyImportsForRoute( for (const name of userDeps.keys()) { const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - // ESM-only dependencies are rewritten to real ES module file:// URLs so that - // import.meta, top-level await, etc. work. They must NOT be transpiled to - // CommonJS and evaluated via the `new Function` shim (see __vf_loadCjs). + // ESM dependencies are rewritten to real ES module file:// URLs so that + // import.meta, top-level await, etc. work. The route build bundles + // CommonJS dependencies, so no import of one is left to rewrite. const esm = esmDeps.get(name); if (esm) { // Resolve a subpath import to a contained file:// URL. The subpath comes @@ -244,53 +239,7 @@ export function rewriteCompiledUserDependencyImportsForRoute( new RegExp(`import\\s*\\(\\s*["']${escaped}["']\\s*\\)`, "g"), () => `import("${esm.entryUrl}")`, ); - continue; } - - transformed = transformed.replace( - new RegExp(`import\\s+(\\w+)\\s+from\\s+["']${escaped}["']`, "g"), - (_, localName) => `const ${localName} = __vf_interopDefault(require("${name}"))`, - ); - transformed = transformed.replace( - new RegExp(`import\\s+(\\{[^}]+\\})\\s+from\\s+["']${escaped}["']`, "g"), - (_, bindings) => `const ${toCjsDestructureBindings(bindings)} = require("${name}")`, - ); - transformed = transformed.replace( - new RegExp(`import\\s+\\*\\s+as\\s+(\\w+)\\s+from\\s+["']${escaped}["']`, "g"), - (_, localName) => `const ${localName} = require("${name}")`, - ); - transformed = transformed.replace( - new RegExp( - `import\\s+(\\w+)\\s*,\\s*(\\{[^}]+\\})\\s+from\\s+["']${escaped}["']`, - "g", - ), - (_, defaultName, bindings) => { - const tmp = `__vf_tmp_${defaultName}`; - return `const ${tmp} = require("${name}"); const ${defaultName} = __vf_interopDefault(${tmp}); const ${ - toCjsDestructureBindings(bindings) - } = ${tmp}`; - }, - ); - transformed = transformed.replace( - new RegExp( - `import\\s+(\\w+|\\*\\s+as\\s+\\w+|\\{[^}]+\\})\\s+from\\s+["']${escaped}(/[^"']+)["']`, - "g", - ), - (_, binding, subpath) => { - const trimmedBinding = String(binding).trim(); - if (trimmedBinding.startsWith("{")) { - return `const ${toCjsDestructureBindings(trimmedBinding)} = require("${name}${subpath}")`; - } - const name_ = trimmedBinding.startsWith("*") - ? trimmedBinding.replace(/\*\s+as\s+/, "") - : trimmedBinding; - return `const ${name_} = require("${name}${subpath}")`; - }, - ); - transformed = transformed.replace( - new RegExp(`import\\s*\\(\\s*["']${escaped}(/[^"']*)?["']\\s*\\)`, "g"), - (_, subpath) => `Promise.resolve(require("${name}${subpath || ""}"))`, - ); } return transformed; diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts new file mode 100644 index 0000000000..75e93d3add --- /dev/null +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -0,0 +1,128 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import * as esbuild from "veryfront/extensions/bundler"; +import type { Plugin } from "veryfront/extensions/bundler"; +import { + commonJSPathPrelude, + createCompiledBuiltinPlugin, + prependBuiltinRequire, +} from "#veryfront/routing/api/module-loader/compiled-cjs-bundling.ts"; + +async function writeFiles(root: string, files: Record): Promise { + for (const [relative, contents] of Object.entries(files)) { + const path = `${root}/${relative}`; + await Deno.mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true }); + await Deno.writeTextFile(path, contents); + } +} + +/** Defines __dirname/__filename for node_modules files, as the route build does. */ +const pathPreludePlugin: Plugin = { + name: "test-path-prelude", + setup(build) { + build.onLoad({ filter: /node_modules[\\/].*\.c?js$/ }, async (args) => { + const contents = await Deno.readTextFile(args.path); + return { contents: commonJSPathPrelude(contents, args.path) + contents, loader: "js" }; + }); + }, +}; + +/** Bundles a route the way the compiled binary does and imports the result. */ +async function bundleAndImport( + projectDir: string, + route: string, +): Promise> { + const requiredBuiltins = new Set(); + const result = await esbuild.build({ + bundle: true, + write: false, + format: "esm", + platform: "neutral", + target: "es2022", + mainFields: ["main", "module"], + conditions: ["node"], + resolveExtensions: [".ts", ".js", ".mjs", ".json"], + logLevel: "silent", + stdin: { + contents: route, + loader: "ts", + resolveDir: projectDir, + sourcefile: `${projectDir}/route.ts`, + }, + plugins: [createCompiledBuiltinPlugin(requiredBuiltins), pathPreludePlugin], + }); + const code = prependBuiltinRequire(result.outputFiles?.[0]?.text ?? "", requiredBuiltins); + const file = `${projectDir}/handler.mjs`; + await Deno.writeTextFile(file, code); + return await import(`file://${file}`); +} + +// The compiled binary bundles a route's CommonJS dependencies (inbox#4177). +describe("compiled-binary route bundling of CommonJS dependencies", () => { + describe("bundling a route's CommonJS dependency", () => { + it("runs exports maps, built-in and JSON requires, __dirname and optional requires", async () => { + const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); + try { + await writeFiles(projectDir, { + "node_modules/legacy/package.json": JSON.stringify({ + name: "legacy", + // The node condition must win over default, as it does in Node. + exports: { ".": { node: "./lib/index.js", default: "./lib/missing.js" } }, + }), + "node_modules/legacy/lib/index.js": [ + '"use strict";', + 'var fs = require("fs");', + 'var path = require("node:path");', + 'var data = require("./data");', + "var optional;", + 'try { optional = require("not-installed"); } catch (error) { optional = error.code; }', + "exports.name = data.name;", + "exports.ownFile = path.basename(__filename);", + 'exports.readsSibling = fs.existsSync(path.join(__dirname, "data.json"));', + "exports.optional = optional;", + ].join("\n"), + "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), + }); + const mod = await bundleAndImport( + projectDir, + 'const require = (id: string) => "route:" + id;\n' + + 'export const routeRequire = require("x");\n' + + 'import legacy from "legacy";\nexport { legacy };\n', + ); + assertEquals(mod.legacy, { + name: "legacy-data", + ownFile: "index.js", + readsSibling: true, + optional: "MODULE_NOT_FOUND", + }); + // esbuild renames a route's own `require`, so the prologue cannot clash. + assertEquals(mod.routeRequire, "route:x"); + } finally { + await esbuild.stop(); + await Deno.remove(projectDir, { recursive: true }); + } + }); + + it("rejects a native addon at build time", async () => { + const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); + try { + await writeFiles(projectDir, { + "node_modules/native/package.json": JSON.stringify({ name: "native", main: "index.js" }), + "node_modules/native/index.js": 'module.exports = require("./build/binding.node");', + "node_modules/native/build/binding.node": "not a real addon", + }); + const error = await assertRejects(() => + bundleAndImport(projectDir, 'import native from "native";\nexport { native };\n') + ); + assert( + String((error as Error).message).includes("Native addons are not supported"), + String(error), + ); + } finally { + await esbuild.stop(); + await Deno.remove(projectDir, { recursive: true }); + } + }); + }); +}); From a7753e278dbf32a4dd3542ab1beec846f47244c6 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 19:47:01 +0100 Subject: [PATCH 04/29] refactor(routing): address Sonar findings in compiled CJS bundling prologue Refs veryfront/veryfront-issue-inbox#4177 --- .../module-loader/compiled-cjs-bundling.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 129c127bf1..534ac9261c 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -87,18 +87,21 @@ export function commonJSPathPrelude(contents: string, filePath: string): string * so the prologue cannot collide with route code. */ export function prependBuiltinRequire(code: string, requiredBuiltins: Iterable): string { - const ids = [...requiredBuiltins].sort(); + // A fixed order keeps the output byte-identical for the same set of ids. + const ids = [...requiredBuiltins].sort((a, b) => a.localeCompare(b, "en")); let prefix = "__vf_builtin"; while (code.includes(prefix)) prefix += "_"; - const lines = ids.map((id, index) => - `import * as ${prefix}${index} from ${JSON.stringify(`node:${builtinModuleName(id)}`)};` - ); - const entries = ids.map((id, index) => - `${JSON.stringify(id)}: ${prefix}${index}.default ?? ${prefix}${index}` - ); + const lines = ids.map((id, index) => { + const specifier = JSON.stringify("node:" + builtinModuleName(id)); + return `import * as ${prefix}${index} from ${specifier};`; + }); + const fields = ["__proto__: null"]; + for (const [index, id] of ids.entries()) { + fields.push(`${JSON.stringify(id)}: ${prefix}${index}.default ?? ${prefix}${index}`); + } lines.push( - `const ${prefix}s = { __proto__: null${entries.map((entry) => `, ${entry}`).join("")} };`, + `const ${prefix}s = { ${fields.join(", ")} };`, "const require = (id) => {", // The table has no prototype, so `in` reads only its own keys and does // not depend on built-ins route code could replace. From 22b529dd7d395f55e2c46d34b686f3a8415d8f2b Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 20:00:34 +0100 Subject: [PATCH 05/29] fix(api): keep CommonJS semantics for bundled route dependencies Wrap a CommonJS dependency in a function that passes __dirname and __filename, so a hashbang and a "use strict" prologue stay in effect and a file that declares one of them still reads the other. ESM dependency files (by extension, package type or syntax) are left alone. Resolve literal require.resolve() calls at build time, and serve runtime require()/require.resolve() for built-ins and for existing files inside the project; anything else throws MODULE_NOT_FOUND. --- scripts/lint/check-sanitizer-baseline.ts | 2 +- .../compiled-cjs-bundling.test.ts | 94 +++++- .../module-loader/compiled-cjs-bundling.ts | 213 +++++++++++-- .../external-import-rewriter.test.ts | 6 +- src/routing/api/module-loader/loader.ts | 81 ++++- .../compiled-cjs-bundling.test.ts | 282 ++++++++++++------ 6 files changed, 521 insertions(+), 157 deletions(-) diff --git a/scripts/lint/check-sanitizer-baseline.ts b/scripts/lint/check-sanitizer-baseline.ts index 6545507214..ac39c14d10 100644 --- a/scripts/lint/check-sanitizer-baseline.ts +++ b/scripts/lint/check-sanitizer-baseline.ts @@ -28,7 +28,7 @@ import { // audits removed opt-outs. Platform/compat process and opaque-deps suites now // restore the globals and child processes they touch instead of suppressing // leak reports. -export const SANITIZER_OPT_OUT_BASELINE = 340; +export const SANITIZER_OPT_OUT_BASELINE = 338; const OPT_OUT_PATTERN = /sanitize(?:Resources|Ops|Exit)\s*:\s*false/g; diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index c8270918ee..467cd2b378 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -3,8 +3,10 @@ import { assert, assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { builtinModuleName, - commonJSPathPrelude, - prependBuiltinRequire, + isCommonJSFile, + prependRuntimeRequire, + rewriteStaticRequireResolve, + wrapCommonJSPaths, } from "./compiled-cjs-bundling.ts"; describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { @@ -22,32 +24,96 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); }); - describe("commonJSPathPrelude()", () => { - it("declares the file's own directory and path when the file reads them", () => { + describe("isCommonJSFile()", () => { + const file = "/app/node_modules/a/index.js"; + + it("follows Node's rule for extensions and the package type", async () => { + assertEquals( + await isCommonJSFile("module.exports = 1;", "/app/node_modules/a/i.cjs", "module"), + true, + ); assertEquals( - commonJSPathPrelude("module.exports = __dirname;", "/app/node_modules/a/index.js"), - 'var __dirname = "/app/node_modules/a", __filename = "/app/node_modules/a/index.js";\n', + await isCommonJSFile("export default 1;", "/app/node_modules/a/i.mjs", undefined), + false, + ); + assertEquals(await isCommonJSFile("module.exports = 1;", file, undefined), true); + assertEquals(await isCommonJSFile("module.exports = 1;", file, "commonjs"), true); + assertEquals(await isCommonJSFile("globalThis.x = typeof __dirname;", file, "module"), false); + }); + + it("treats a .js file with ESM syntax as ESM, as esbuild bundles it", async () => { + assertEquals(await isCommonJSFile('import x from "x";\nx();', file, undefined), false); + assertEquals(await isCommonJSFile("export const a = 1;", file, undefined), false); + assertEquals(await isCommonJSFile("const u = import.meta.url;", file, undefined), false); + assertEquals(await isCommonJSFile('const m = import("x");', file, undefined), true); + }); + }); + + describe("wrapCommonJSPaths()", () => { + const file = "/app/node_modules/a/index.js"; + + it("leaves a file that does not read the wrapper bindings unchanged", () => { + assertEquals(wrapCommonJSPaths("module.exports = 1;", file), "module.exports = 1;"); + }); + + it("passes both bindings, so a file's own declaration shadows only its own name", () => { + const wrapped = wrapCommonJSPaths( + "var __dirname = 'custom'; module.exports = [__dirname, __filename];", + file, ); + const module = { exports: [] as string[] }; + new Function("module", wrapped)(module); + assertEquals(module.exports, ["custom", file]); }); - it("adds nothing to a file that does not read them", () => { - assertEquals(commonJSPathPrelude("module.exports = 1;", "/app/node_modules/a/i.js"), ""); + it("keeps a hashbang first and a directive prologue in effect", () => { + const wrapped = wrapCommonJSPaths( + '#!/usr/bin/env node\n"use strict";\nmodule.exports = [__dirname, (function () { return this; })()];', + file, + ); + assert( + wrapped.startsWith("#!/usr/bin/env node\n(function (__dirname, __filename) {"), + wrapped, + ); + const module = { exports: [] as unknown[] }; + new Function("module", wrapped.slice(wrapped.indexOf("\n") + 1))(module); + assertEquals(module.exports, ["/app/node_modules/a", undefined]); }); + }); - it("leaves a file that declares its own binding alone", () => { - const esm = 'import { dirname } from "path";\nconst __dirname = dirname("x");'; - assertEquals(commonJSPathPrelude(esm, "/app/node_modules/a/i.js"), ""); + describe("rewriteStaticRequireResolve()", () => { + it("replaces resolvable literals and leaves the rest for runtime", async () => { + const source = [ + 'const a = require.resolve("./asset.json");', + "const b = require.resolve('optional-missing');", + 'const c = require.resolve("fs");', + 'const d = notrequire.resolve("./asset.json");', + ].join("\n"); + const out = await rewriteStaticRequireResolve( + source, + (specifier) => + Promise.resolve(specifier === "./asset.json" ? "/app/node_modules/a/asset.json" : null), + ); + assertEquals( + out, + [ + 'const a = "/app/node_modules/a/asset.json";', + "const b = require.resolve('optional-missing');", + 'const c = "fs";', + 'const d = notrequire.resolve("./asset.json");', + ].join("\n"), + ); }); }); - describe("prependBuiltinRequire()", () => { + describe("prependRuntimeRequire()", () => { it("keeps a hashbang on the first line", () => { - const code = prependBuiltinRequire("#!/usr/bin/env node\nexport {};\n", ["path"]); + const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", ["path"], "/app"); assert(code.startsWith("#!/usr/bin/env node\nimport * as "), code); }); it("picks helper names the bundle does not already use", () => { - const code = prependBuiltinRequire("const __vf_builtin0 = 1;\n", ["path"]); + const code = prependRuntimeRequire("const __vf_builtin0 = 1;\n", ["path"], "/app"); assert(code.includes("import * as __vf_builtin_0 from"), code); }); }); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 534ac9261c..13e9e63d2c 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -5,21 +5,27 @@ * `require` there reads only the files embedded at compile time, and importing * a `.cjs` file evaluates it as ESM. So the route build bundles a project's * CommonJS dependencies into the route module itself, and esbuild supplies Node - * resolution and interop. Three pieces keep the bundled code runnable: + * resolution and interop. These pieces keep the bundled code runnable: * * - A `require()` of a built-in stays external. esbuild emits it as a call to - * the module's free `require`, which {@link prependBuiltinRequire} binds to + * the module's free `require`, which {@link prependRuntimeRequire} binds to * built-ins imported statically, so nothing loads `node:module` at runtime. * - `__dirname` and `__filename` are not defined in an ES module, so - * {@link commonJSPathPrelude} defines them for each bundled file that uses - * them, as Node's CommonJS wrapper would. + * {@link wrapCommonJSPaths} passes them to each CommonJS file that reads + * them, as Node's CommonJS wrapper does. + * - `require.resolve("literal")` is resolved at build time to the file's path + * on disk ({@link rewriteStaticRequireResolve}). What is left for runtime, a + * computed `require()` or `require.resolve()`, gets built-ins, existing + * project paths and JSON files; anything else throws `MODULE_NOT_FOUND`. * - A native addon cannot be bundled or loaded; the build rejects it with a * clear message rather than an esbuild loader error. * * @module routing/api/module-loader/compiled-cjs-bundling */ import { builtinModules } from "node:module"; -import type { Plugin } from "veryfront/extensions/bundler"; +import type { Plugin, PluginBuild } from "veryfront/extensions/bundler"; +import { parseImports } from "#veryfront/transforms/esm/lexer.ts"; +import { toFileUrl } from "#veryfront/compat/path"; // Names Node accepts only with the `node:` prefix. A bare `test` is an npm package. const PREFIX_ONLY_BUILTINS = new Set(["test", "test/reporters", "sqlite", "sea"]); @@ -37,7 +43,7 @@ export function builtinModuleName(specifier: string): string | undefined { /** * Keeps built-ins external and records each one a bundled module `require`s, - * so {@link prependBuiltinRequire} can bind it. Must run before any plugin + * so {@link prependRuntimeRequire} can bind it. Must run before any plugin * that resolves built-ins itself. */ export function createCompiledBuiltinPlugin(requiredBuiltins: Set): Plugin { @@ -62,53 +68,198 @@ export function createCompiledBuiltinPlugin(requiredBuiltins: Set): Plug }; } -const DECLARES_PATH_BINDING = - /\b(?:var|let|const|function|class)\s+(?:__dirname|__filename)\b|\bimport\s[^;]*\b(?:__dirname|__filename)\b/; +// An `export` statement; the module lexer reports only imports. +const EXPORT_STATEMENT = + /(?:^|[;}\n])\s*export\s*(?:default\b|[{*]|(?:async\s+)?function\b|class\b|const\b|let\b|var\b)/; /** - * Declarations of `__dirname` and `__filename` for a bundled dependency file - * that reads them without declaring them, or "" when it needs none. esbuild - * wraps each CommonJS file in its own function, so the values stay per file. + * Whether Node would load a dependency file as CommonJS: a `.cjs` file, or a + * `.js` file outside a `"type": "module"` package scope that has no ESM syntax. + * esbuild bundles ESM syntax in a `.js` file as ESM whatever the package says. */ -export function commonJSPathPrelude(contents: string, filePath: string): string { - const usesDirname = /\b__dirname\b/.test(contents); - const usesFilename = /\b__filename\b/.test(contents); - if (!usesDirname && !usesFilename) return ""; - if (DECLARES_PATH_BINDING.test(contents)) return ""; +export async function isCommonJSFile( + contents: string, + filePath: string, + packageType: string | undefined, +): Promise { + if (/\.cjs$/i.test(filePath)) return true; + if (!/\.js$/i.test(filePath) || packageType === "module") return false; + if (EXPORT_STATEMENT.test(contents)) return false; + try { + // A static import (d === -1) or import.meta (d === -2) is ESM syntax. + return !(await parseImports(contents)).some((specifier) => specifier.d < 0); + } catch { + // expected: the lexer rejects some valid scripts; Node's rule decides. + return true; + } +} + +/** + * Passes `__dirname` and `__filename` to a CommonJS file that reads them, as + * Node's module wrapper does: as parameters of a function around the file, so a + * file's own `var __dirname` still shadows them and its directive prologue + * ("use strict") stays first in that function. A hashbang stays first in the + * file. Top-level `this` is passed through, which esbuild maps to `exports`. + */ +export function wrapCommonJSPaths(contents: string, filePath: string): string { + if (!/\b__(?:dirname|filename)\b/.test(contents)) return contents; + const hashbangEnd = contents.startsWith("#!") ? contents.indexOf("\n") + 1 || contents.length : 0; const dir = filePath.slice(0, Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\"))); - return `var __dirname = ${JSON.stringify(dir)}, __filename = ${JSON.stringify(filePath)};\n`; + return contents.slice(0, hashbangEnd) + + "(function (__dirname, __filename) {" + contents.slice(hashbangEnd) + + `\n}).call(this, ${JSON.stringify(dir)}, ${JSON.stringify(filePath)});\n`; +} + +const STATIC_REQUIRE_RESOLVE = /(^|[^\w$.])require\.resolve\(\s*(["'])([^"'\\\n]+)\2\s*\)/g; + +/** + * Replaces each `require.resolve("literal")` that resolves at build time with + * the string Node would return: the resolved file path, or the id itself for a + * built-in. One that does not resolve is left for runtime, where it throws + * `MODULE_NOT_FOUND` as Node does, so an optional dependency probed this way + * still falls back. + */ +export async function rewriteStaticRequireResolve( + contents: string, + resolve: (specifier: string) => Promise, +): Promise { + if (!contents.includes("require.resolve")) return contents; + const resolved = new Map(); + for (const match of contents.matchAll(STATIC_REQUIRE_RESOLVE)) { + const specifier = match[3]!; + if (!resolved.has(specifier)) { + resolved.set( + specifier, + builtinModuleName(specifier) === undefined ? await resolve(specifier) : specifier, + ); + } + } + return contents.replace( + STATIC_REQUIRE_RESOLVE, + (whole, before: string, _q, specifier: string) => { + const target = resolved.get(specifier); + return typeof target === "string" ? before + JSON.stringify(target) : whole; + }, + ); +} + +/** Input for {@link prepareDependencyFile}. */ +export interface DependencyFileInput { + readonly build: PluginBuild; + readonly contents: string; + /** The file's path as the project sees it. */ + readonly logicalPath: string; + /** The path the bundler resolved, used as the importer for resolution. */ + readonly resolvedPath: string; + /** The `type` field of the package.json nearest the file. */ + readonly packageType: string | undefined; + readonly isDependencyFile: (path: string) => boolean; + /** Points a module's `import.meta` locations at its own URL. */ + readonly rewriteImportMeta: (contents: string, moduleUrl: string) => Promise; +} + +/** + * A bundled dependency file as Node would run it. In a CommonJS file, + * `require.resolve("literal")` becomes the path the bundler resolves inside + * the project's dependencies, and `__dirname`/`__filename` are passed in as + * Node's wrapper passes them. An ESM file keeps its own semantics; only its + * `import.meta` locations are pointed at the file's own URL. + */ +export async function prepareDependencyFile(input: DependencyFileInput): Promise { + const { build, contents, logicalPath, resolvedPath, isDependencyFile } = input; + if (!await isCommonJSFile(contents, logicalPath, input.packageType)) { + return /\bimport\.meta\b/.test(contents) + ? await input.rewriteImportMeta(contents, toFileUrl(logicalPath).href) + : contents; + } + const resolveDir = resolvedPath.slice( + 0, + Math.max(resolvedPath.lastIndexOf("/"), resolvedPath.lastIndexOf("\\")), + ); + const resolved = await rewriteStaticRequireResolve(contents, async (specifier) => { + if (!build.resolve) return null; + const result = await build.resolve(specifier, { + kind: "require-resolve", + importer: resolvedPath, + resolveDir, + }); + // Only a file inside the project's dependencies is a path to hand back. + if (result.errors?.length || !result.path || result.external) return null; + return isDependencyFile(result.path) ? result.path : null; + }); + return wrapCommonJSPaths(resolved, logicalPath); } /** - * Binds the module-level `require` that esbuild's output calls for an external - * `require()` to the built-ins recorded during the build. Any other id throws - * the error Node gives for a missing module, so optional dependencies probed - * in a `try` still fall back. esbuild renames route bindings named `require`, - * so the prologue cannot collide with route code. + * Binds the module-level `require` that esbuild's output calls for anything it + * did not bundle: the built-ins recorded during the build, and, for a computed + * id, an existing file inside the project, which `require.resolve` returns and + * `require` loads when it is JSON. Anything else throws `MODULE_NOT_FOUND`, so + * optional dependencies probed in a `try` still fall back. Errors name a path + * relative to the project, never the host path. esbuild renames route bindings + * named `require`, so the prologue cannot collide with route code. */ -export function prependBuiltinRequire(code: string, requiredBuiltins: Iterable): string { +export function prependRuntimeRequire( + code: string, + requiredBuiltins: Iterable, + projectRoot: string, +): string { // A fixed order keeps the output byte-identical for the same set of ids. const ids = [...requiredBuiltins].sort((a, b) => a.localeCompare(b, "en")); let prefix = "__vf_builtin"; while (code.includes(prefix)) prefix += "_"; - const lines = ids.map((id, index) => { + const lines = [ + `import * as ${prefix}_fs from "node:fs";`, + `import * as ${prefix}_path from "node:path";`, + ]; + for (const [index, id] of ids.entries()) { const specifier = JSON.stringify("node:" + builtinModuleName(id)); - return `import * as ${prefix}${index} from ${specifier};`; - }); + lines.push(`import * as ${prefix}${index} from ${specifier};`); + } const fields = ["__proto__: null"]; for (const [index, id] of ids.entries()) { fields.push(`${JSON.stringify(id)}: ${prefix}${index}.default ?? ${prefix}${index}`); } + const root = JSON.stringify(projectRoot); + // Every id Node's require.resolve returns unchanged: a bare built-in name, or + // any built-in, prefix-only ones included, with the node: prefix. + const builtinIds = [ + ...BUILTINS, + ...[...BUILTINS, ...PREFIX_ONLY_BUILTINS].map((name) => "node:" + name), + ].map((id) => JSON.stringify(id) + ": 1"); lines.push( - `const ${prefix}s = { ${fields.join(", ")} };`, - "const require = (id) => {", // The table has no prototype, so `in` reads only its own keys and does // not depend on built-ins route code could replace. - ` if (typeof id === "string" && id in ${prefix}s) return ${prefix}s[id];`, - ' const error = new Error("Cannot find module " + JSON.stringify(String(id)));', + `const ${prefix}s = { ${fields.join(", ")} };`, + `const ${prefix}_ids = { __proto__: null, ${builtinIds.join(", ")} };`, + `const ${prefix}_missing = (label) => {`, + ' const error = new Error("Cannot find module " + JSON.stringify(label));', ' error.code = "MODULE_NOT_FOUND";', - " throw error;", + " return error;", + "};", + // A path is usable only when it exists and its real path is in the project. + `const ${prefix}_file = (id) => {`, + ` if (typeof id !== "string" || !${prefix}_path.isAbsolute(id)) return null;`, + " let real;", + ` try { real = ${prefix}_fs.realpathSync(id); } catch { return null; }`, + ` const relative = ${prefix}_path.relative(${root}, real);`, + ` const up = relative === ".." || relative.startsWith(".." + ${prefix}_path.sep);`, + ` if (relative === "" || up || ${prefix}_path.isAbsolute(relative)) return null;`, + " return { real, relative };", + "};", + `const ${prefix}_label = (id) => typeof id === "string" && ${prefix}_path.isAbsolute(id) ? "a path outside this project or missing" : String(id);`, + "const require = (id) => {", + ` if (typeof id === "string" && id in ${prefix}s) return ${prefix}s[id];`, + ` const file = ${prefix}_file(id);`, + ` if (file !== null && file.real.endsWith(".json")) return JSON.parse(${prefix}_fs.readFileSync(file.real, "utf8"));`, + ` throw ${prefix}_missing(file !== null ? file.relative : ${prefix}_label(id));`, + "};", + "require.resolve = (id) => {", + ` if (typeof id === "string" && (id in ${prefix}s || id in ${prefix}_ids)) return id;`, + ` const file = ${prefix}_file(id);`, + " if (file !== null) return id;", + ` throw ${prefix}_missing(${prefix}_label(id));`, "};", ); diff --git a/src/routing/api/module-loader/external-import-rewriter.test.ts b/src/routing/api/module-loader/external-import-rewriter.test.ts index a8b62aebb6..c2a4c60b54 100644 --- a/src/routing/api/module-loader/external-import-rewriter.test.ts +++ b/src/routing/api/module-loader/external-import-rewriter.test.ts @@ -1,9 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { - assertEquals, - assertStringIncludes, - assertThrows, -} from "#veryfront/testing/assert.ts"; +import { assertEquals, assertStringIncludes, assertThrows } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import type { FileInfo } from "#veryfront/platform/adapters/base.ts"; import type { FileSystem } from "#veryfront/platform/compat/fs.ts"; diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index cf21e61cec..a199fbcdbc 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -49,9 +49,9 @@ import { rewriteExternalImports, } from "./external-import-rewriter.ts"; import { - commonJSPathPrelude, createCompiledBuiltinPlugin, - prependBuiltinRequire, + prepareDependencyFile, + prependRuntimeRequire, } from "./compiled-cjs-bundling.ts"; import { MAX_WORKER_MODULE_SOURCE_BYTES, @@ -1312,6 +1312,16 @@ const noDependencyFiles: DependencyFilePredicate = () => false; * bundles. Both the given and the canonical project root count: esbuild * reports resolved paths with symlinks followed. */ +/** The project root with symlinks followed, or as given when it has none on the host. */ +async function canonicalProjectRoot(projectDir: string): Promise { + try { + return await realPath(projectDir); + } catch { + // expected: an adapter-backed project may have no host directory + return pathHelper.resolve(projectDir); + } +} + async function projectDependencyFiles(projectDir: string): Promise { const roots = new Set([pathHelper.resolve(projectDir)]); try { @@ -1324,6 +1334,50 @@ async function projectDependencyFiles(projectDir: string): Promise pathHelper.resolve(path).startsWith(prefix)); } +/** + * The `type` field of the package.json nearest a directory, as Node uses it to + * decide whether a `.js` file is ESM. The walk stays inside the project. + */ +class PackageTypeLookup { + readonly #byDirectory = new Map>(); + readonly #root: string; + + constructor(private readonly sourceSnapshot: ProjectSourceSnapshot, projectDir: string) { + this.#root = pathHelper.resolve(projectDir); + } + + of(directory: string): Promise { + let found = this.#byDirectory.get(directory); + if (!found) { + found = this.#read(directory); + this.#byDirectory.set(directory, found); + } + return found; + } + + async #read(directory: string): Promise { + let manifest: string | null = null; + try { + manifest = await this.sourceSnapshot.readTextFile(pathHelper.join(directory, "package.json")); + } catch (error) { + if (error instanceof ProjectBoundaryViolationError) throw error; + // expected: no package.json here; look in the parent directory + } + if (manifest !== null) { + try { + const type = (JSON.parse(manifest) as { type?: unknown }).type; + return typeof type === "string" ? type : undefined; + } catch { + // Node rejects a malformed package.json; esbuild reports it as well. + return undefined; + } + } + const parent = pathHelper.dirname(directory); + if (parent === directory || !isWithinDirectory(this.#root, parent)) return undefined; + return await this.of(parent); + } +} + function createImportMapPlugin( projectDir: string, sourceSnapshot: ProjectSourceSnapshot, @@ -1901,6 +1955,7 @@ function createProjectBoundaryPlugin( workerImportMap: DenoImportMap | null, isDependencyFile: DependencyFilePredicate = noDependencyFiles, ): Plugin { + const packageTypes = new PackageTypeLookup(sourceSnapshot, projectDir); return { name: "vf-project-boundary", setup(build) { @@ -1933,14 +1988,16 @@ function createProjectBoundaryPlugin( } if (executableModule && dependency) { return { - contents: commonJSPathPrelude(source.contents, source.logicalPath) + - (/\bimport\.meta\b/.test(source.contents) - ? await rewriteBundledImportMetaUrl( - source.contents, - pathHelper.toFileUrl(source.logicalPath).href, - null, - ) - : source.contents), + contents: await prepareDependencyFile({ + build, + contents: source.contents, + logicalPath: source.logicalPath, + resolvedPath: args.path, + packageType: await packageTypes.of(pathHelper.dirname(source.logicalPath)), + isDependencyFile, + rewriteImportMeta: (contents, moduleUrl) => + rewriteBundledImportMetaUrl(contents, moduleUrl, null), + }), loader: getLoaderForFile(source.logicalPath), resolveDir: pathHelper.dirname(source.logicalPath), }; @@ -2277,7 +2334,9 @@ function buildTranspiledModuleSource( logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); const output = result.outputFiles?.[0]?.text ?? "export {}"; - const js = compiled ? prependBuiltinRequire(output, requiredBuiltins) : output; + const js = compiled + ? prependRuntimeRequire(output, requiredBuiltins, await canonicalProjectRoot(projectDir)) + : output; logger.debug(`transpiled size ${js.length} bytes`); // Only dependencies left external are rewritten: bundled package code diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 75e93d3add..6eb3bdc5d3 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,12 +1,13 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import { withTempDir } from "#veryfront/testing/deno-compat.ts"; import * as esbuild from "veryfront/extensions/bundler"; import type { Plugin } from "veryfront/extensions/bundler"; import { - commonJSPathPrelude, createCompiledBuiltinPlugin, - prependBuiltinRequire, + prepareDependencyFile, + prependRuntimeRequire, } from "#veryfront/routing/api/module-loader/compiled-cjs-bundling.ts"; async function writeFiles(root: string, files: Record): Promise { @@ -17,16 +18,42 @@ async function writeFiles(root: string, files: Record): Promise< } } -/** Defines __dirname/__filename for node_modules files, as the route build does. */ -const pathPreludePlugin: Plugin = { - name: "test-path-prelude", - setup(build) { - build.onLoad({ filter: /node_modules[\\/].*\.c?js$/ }, async (args) => { - const contents = await Deno.readTextFile(args.path); - return { contents: commonJSPathPrelude(contents, args.path) + contents, loader: "js" }; - }); - }, -}; +/** The `type` of the package.json nearest a file, as the route build reads it. */ +async function packageTypeOf(filePath: string, root: string): Promise { + for (let dir = filePath.slice(0, filePath.lastIndexOf("/")); dir.startsWith(root);) { + try { + return JSON.parse(await Deno.readTextFile(`${dir}/package.json`)).type; + } catch { + dir = dir.slice(0, dir.lastIndexOf("/")); + } + } + return undefined; +} + +/** Prepares node_modules files the way the compiled route build does. */ +function dependencyFilePlugin(projectDir: string): Plugin { + const isDependencyFile = (path: string) => path.startsWith(`${projectDir}/node_modules/`); + return { + name: "test-dependency-files", + setup(build) { + build.onLoad({ filter: /node_modules[\\/].*\.[cm]?js$/ }, async (args) => { + const contents = await Deno.readTextFile(args.path); + return { + contents: await prepareDependencyFile({ + build, + contents, + logicalPath: args.path, + resolvedPath: args.path, + packageType: await packageTypeOf(args.path, projectDir), + isDependencyFile, + rewriteImportMeta: (source) => Promise.resolve(source), + }), + loader: "js", + }; + }); + }, + }; +} /** Bundles a route the way the compiled binary does and imports the result. */ async function bundleAndImport( @@ -34,95 +61,160 @@ async function bundleAndImport( route: string, ): Promise> { const requiredBuiltins = new Set(); - const result = await esbuild.build({ - bundle: true, - write: false, - format: "esm", - platform: "neutral", - target: "es2022", - mainFields: ["main", "module"], - conditions: ["node"], - resolveExtensions: [".ts", ".js", ".mjs", ".json"], - logLevel: "silent", - stdin: { - contents: route, - loader: "ts", - resolveDir: projectDir, - sourcefile: `${projectDir}/route.ts`, - }, - plugins: [createCompiledBuiltinPlugin(requiredBuiltins), pathPreludePlugin], - }); - const code = prependBuiltinRequire(result.outputFiles?.[0]?.text ?? "", requiredBuiltins); - const file = `${projectDir}/handler.mjs`; - await Deno.writeTextFile(file, code); - return await import(`file://${file}`); + try { + const result = await esbuild.build({ + bundle: true, + write: false, + format: "esm", + platform: "neutral", + target: "es2022", + mainFields: ["main", "module"], + conditions: ["node"], + resolveExtensions: [".ts", ".js", ".mjs", ".json"], + logLevel: "silent", + stdin: { + contents: route, + loader: "ts", + resolveDir: projectDir, + sourcefile: `${projectDir}/route.ts`, + }, + plugins: [createCompiledBuiltinPlugin(requiredBuiltins), dependencyFilePlugin(projectDir)], + }); + const code = prependRuntimeRequire( + result.outputFiles?.[0]?.text ?? "", + requiredBuiltins, + projectDir, + ); + const file = `${projectDir}/handler.mjs`; + await Deno.writeTextFile(file, code); + return await import(`file://${file}`); + } finally { + await esbuild.stop(); + } +} + +/** Runs a test in a fresh project directory, with symlinks in its path followed. */ +function inProject(fn: (projectDir: string) => Promise): Promise { + return withTempDir((dir) => fn(Deno.realPathSync(dir)), { prefix: "vf-cjs-bundle-" }); } // The compiled binary bundles a route's CommonJS dependencies (inbox#4177). describe("compiled-binary route bundling of CommonJS dependencies", () => { - describe("bundling a route's CommonJS dependency", () => { - it("runs exports maps, built-in and JSON requires, __dirname and optional requires", async () => { - const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); - try { - await writeFiles(projectDir, { - "node_modules/legacy/package.json": JSON.stringify({ - name: "legacy", - // The node condition must win over default, as it does in Node. - exports: { ".": { node: "./lib/index.js", default: "./lib/missing.js" } }, - }), - "node_modules/legacy/lib/index.js": [ - '"use strict";', - 'var fs = require("fs");', - 'var path = require("node:path");', - 'var data = require("./data");', - "var optional;", - 'try { optional = require("not-installed"); } catch (error) { optional = error.code; }', - "exports.name = data.name;", - "exports.ownFile = path.basename(__filename);", - 'exports.readsSibling = fs.existsSync(path.join(__dirname, "data.json"));', - "exports.optional = optional;", - ].join("\n"), - "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), - }); - const mod = await bundleAndImport( - projectDir, - 'const require = (id: string) => "route:" + id;\n' + - 'export const routeRequire = require("x");\n' + - 'import legacy from "legacy";\nexport { legacy };\n', - ); - assertEquals(mod.legacy, { - name: "legacy-data", - ownFile: "index.js", - readsSibling: true, - optional: "MODULE_NOT_FOUND", - }); - // esbuild renames a route's own `require`, so the prologue cannot clash. - assertEquals(mod.routeRequire, "route:x"); - } finally { - await esbuild.stop(); - await Deno.remove(projectDir, { recursive: true }); - } + it("runs exports maps, built-in and JSON requires, __dirname and optional requires", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/legacy/package.json": JSON.stringify({ + name: "legacy", + // The node condition must win over default, as it does in Node. + exports: { ".": { node: "./lib/index.js", default: "./lib/missing.js" } }, + }), + "node_modules/legacy/lib/index.js": [ + '"use strict";', + 'var fs = require("fs");', + 'var path = require("node:path");', + 'var data = require("./data");', + "var optional;", + 'try { optional = require("not-installed"); } catch (error) { optional = error.code; }', + "exports.name = data.name;", + "exports.ownFile = path.basename(__filename);", + 'exports.readsSibling = fs.existsSync(path.join(__dirname, "data.json"));', + "exports.optional = optional;", + ].join("\n"), + "node_modules/legacy/lib/data.json": JSON.stringify({ name: "legacy-data" }), + }); + const mod = await bundleAndImport( + projectDir, + 'const require = (id: string) => "route:" + id;\n' + + 'export const routeRequire = require("x");\n' + + 'import legacy from "legacy";\nexport { legacy };\n', + ); + assertEquals(mod.legacy, { + name: "legacy-data", + ownFile: "index.js", + readsSibling: true, + optional: "MODULE_NOT_FOUND", + }); + // esbuild renames a route's own `require`, so the prologue cannot clash. + assertEquals(mod.routeRequire, "route:x"); + }); + }); + + it("serves require.resolve and computed requires left for runtime", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/assets/package.json": JSON.stringify({ name: "assets", main: "index.js" }), + "node_modules/assets/data/en.json": JSON.stringify({ hello: "world" }), + "node_modules/assets/index.js": [ + 'var path = require("path");', + 'exports.asset = require.resolve("./data/en.json");', + 'exports.own = require.resolve("assets/package.json");', + 'exports.builtin = require.resolve("fs");', + "var missing;", + 'try { require.resolve("optional-native"); } catch (error) { missing = error.code; }', + "exports.missing = missing;", + 'var computed = path.join(__dirname, "data", "en.json");', + "exports.computedPath = require.resolve(computed);", + "exports.computed = require(computed).hello;", + "var outside;", + 'var hosts = ["", "etc", "hosts"].join("/");', + "try { require(hosts); } catch (error) { outside = [error.code, error.message]; }", + "exports.outside = outside;", + ].join("\n"), + }); + const mod = await bundleAndImport( + projectDir, + 'import assets from "assets";\nexport { assets };\n', + ); + const asset = `${projectDir}/node_modules/assets/data/en.json`; + assertEquals(mod.assets, { + asset, + own: `${projectDir}/node_modules/assets/package.json`, + builtin: "fs", + missing: "MODULE_NOT_FOUND", + computedPath: asset, + computed: "world", + outside: [ + "MODULE_NOT_FOUND", + 'Cannot find module "a path outside this project or missing"', + ], + }); }); + }); + + it("gives ESM dependency files no CommonJS bindings", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/scoped/package.json": JSON.stringify({ + name: "scoped", + type: "module", + main: "index.js", + }), + "node_modules/scoped/index.js": "export const mode = typeof __dirname;\n", + "node_modules/plain/package.json": JSON.stringify({ name: "plain", module: "index.js" }), + "node_modules/plain/index.js": "export const mode = typeof __filename;\n", + }); + const mod = await bundleAndImport( + projectDir, + 'import { mode as a } from "scoped";\nimport { mode as b } from "plain";\nexport const modes = [a, b];\n', + ); + assertEquals(mod.modes, ["undefined", "undefined"]); + }); + }); - it("rejects a native addon at build time", async () => { - const projectDir = await Deno.makeTempDir({ prefix: "vf-cjs-bundle-" }); - try { - await writeFiles(projectDir, { - "node_modules/native/package.json": JSON.stringify({ name: "native", main: "index.js" }), - "node_modules/native/index.js": 'module.exports = require("./build/binding.node");', - "node_modules/native/build/binding.node": "not a real addon", - }); - const error = await assertRejects(() => - bundleAndImport(projectDir, 'import native from "native";\nexport { native };\n') - ); - assert( - String((error as Error).message).includes("Native addons are not supported"), - String(error), - ); - } finally { - await esbuild.stop(); - await Deno.remove(projectDir, { recursive: true }); - } + it("rejects a native addon at build time", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/native/package.json": JSON.stringify({ name: "native", main: "index.js" }), + "node_modules/native/index.js": 'module.exports = require("./build/binding.node");', + "node_modules/native/build/binding.node": "not a real addon", + }); + const error = await assertRejects(() => + bundleAndImport(projectDir, 'import native from "native";\nexport { native };\n') + ); + assert( + String((error as Error).message).includes("Native addons are not supported"), + String(error), + ); }); }); }); From a41c0b042b9fe551c84ab230b51681f55ea6a0f0 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 20:12:03 +0100 Subject: [PATCH 06/29] test(api): run the CommonJS bundling test on every runtime The Bun shard has no Deno global; use node:fs/promises instead. --- .../module-loader/compiled-cjs-bundling.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 6eb3bdc5d3..00bcd1acb0 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,6 +1,8 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; import { withTempDir } from "#veryfront/testing/deno-compat.ts"; import * as esbuild from "veryfront/extensions/bundler"; import type { Plugin } from "veryfront/extensions/bundler"; @@ -13,8 +15,8 @@ import { async function writeFiles(root: string, files: Record): Promise { for (const [relative, contents] of Object.entries(files)) { const path = `${root}/${relative}`; - await Deno.mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true }); - await Deno.writeTextFile(path, contents); + await mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true }); + await writeFile(path, contents); } } @@ -22,7 +24,7 @@ async function writeFiles(root: string, files: Record): Promise< async function packageTypeOf(filePath: string, root: string): Promise { for (let dir = filePath.slice(0, filePath.lastIndexOf("/")); dir.startsWith(root);) { try { - return JSON.parse(await Deno.readTextFile(`${dir}/package.json`)).type; + return JSON.parse(await readFile(`${dir}/package.json`, "utf8")).type; } catch { dir = dir.slice(0, dir.lastIndexOf("/")); } @@ -37,7 +39,7 @@ function dependencyFilePlugin(projectDir: string): Plugin { name: "test-dependency-files", setup(build) { build.onLoad({ filter: /node_modules[\\/].*\.[cm]?js$/ }, async (args) => { - const contents = await Deno.readTextFile(args.path); + const contents = await readFile(args.path, "utf8"); return { contents: await prepareDependencyFile({ build, @@ -86,8 +88,8 @@ async function bundleAndImport( projectDir, ); const file = `${projectDir}/handler.mjs`; - await Deno.writeTextFile(file, code); - return await import(`file://${file}`); + await writeFile(file, code); + return await import(pathToFileURL(file).href); } finally { await esbuild.stop(); } @@ -95,7 +97,7 @@ async function bundleAndImport( /** Runs a test in a fresh project directory, with symlinks in its path followed. */ function inProject(fn: (projectDir: string) => Promise): Promise { - return withTempDir((dir) => fn(Deno.realPathSync(dir)), { prefix: "vf-cjs-bundle-" }); + return withTempDir(async (dir) => fn(await realpath(dir)), { prefix: "vf-cjs-bundle-" }); } // The compiled binary bundles a route's CommonJS dependencies (inbox#4177). From 52d7ee7db34dc2e807c0a2592829e87f61f33ff9 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 20:26:08 +0100 Subject: [PATCH 07/29] test(api): check ESM dependencies in the bundle, not at runtime Bun defines __dirname and __filename in ESM modules too. --- .../api/module-loader/compiled-cjs-bundling.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 00bcd1acb0..b49d27abd8 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -195,11 +195,14 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { "node_modules/plain/package.json": JSON.stringify({ name: "plain", module: "index.js" }), "node_modules/plain/index.js": "export const mode = typeof __filename;\n", }); - const mod = await bundleAndImport( + await bundleAndImport( projectDir, 'import { mode as a } from "scoped";\nimport { mode as b } from "plain";\nexport const modes = [a, b];\n', ); - assertEquals(mod.modes, ["undefined", "undefined"]); + // Checked in the bundle, not at runtime: Bun defines both names in ESM too. + const code = await readFile(`${projectDir}/handler.mjs`, "utf8"); + assert(!code.includes("function (__dirname, __filename)"), code); + assert(code.includes("typeof __dirname") && code.includes("typeof __filename"), code); }); }); From 088bf7e8cc9235b2bacf77b491450c2b699f6650 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 20:43:40 +0100 Subject: [PATCH 08/29] fix(api): parse bundled CommonJS dependencies and resolve them as Node does Parse each CommonJS dependency file with the bundler instead of matching its text: ESM syntax comes from the parser, and unbound __dirname, __filename and require.resolve("literal") are replaced through define. No wrapper function is added, so arguments, the hashbang and the directive prologue are untouched. Inside dependencies, resolve extensionless relative imports in Node's order (.js, .json, .node), read only the main field, and stop the package-type lookup at node_modules. Workspace packages linked from node_modules count as dependencies. Prefix-only built-ins such as "test" no longer resolve without the node: prefix. --- .../compiled-cjs-bundling.test.ts | 130 +++--- .../module-loader/compiled-cjs-bundling.ts | 429 ++++++++++++++---- src/routing/api/module-loader/loader.ts | 88 ++-- .../compiled-cjs-bundling.test.ts | 161 ++++++- 4 files changed, 568 insertions(+), 240 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 467cd2b378..cf3ce01541 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -3,10 +3,10 @@ import { assert, assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { builtinModuleName, - isCommonJSFile, + nodePreferredFile, + nodeRelativeFile, + PackageTypeLookup, prependRuntimeRequire, - rewriteStaticRequireResolve, - wrapCommonJSPaths, } from "./compiled-cjs-bundling.ts"; describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { @@ -24,85 +24,73 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); }); - describe("isCommonJSFile()", () => { - const file = "/app/node_modules/a/index.js"; + describe("PackageTypeLookup", () => { + const manifests: Record = { + "/app/package.json": JSON.stringify({ type: "module" }), + "/app/node_modules/typed/package.json": JSON.stringify({ type: "commonjs" }), + }; + const lookup = () => + new PackageTypeLookup((path) => Promise.resolve(manifests[path] ?? null), "/app"); - it("follows Node's rule for extensions and the package type", async () => { - assertEquals( - await isCommonJSFile("module.exports = 1;", "/app/node_modules/a/i.cjs", "module"), - true, - ); - assertEquals( - await isCommonJSFile("export default 1;", "/app/node_modules/a/i.mjs", undefined), - false, - ); - assertEquals(await isCommonJSFile("module.exports = 1;", file, undefined), true); - assertEquals(await isCommonJSFile("module.exports = 1;", file, "commonjs"), true); - assertEquals(await isCommonJSFile("globalThis.x = typeof __dirname;", file, "module"), false); + it("reads the nearest package.json inside the project", async () => { + assertEquals(await lookup().of("/app/src/lib"), "module"); + assertEquals(await lookup().of("/app/node_modules/typed/lib"), "commonjs"); }); - it("treats a .js file with ESM syntax as ESM, as esbuild bundles it", async () => { - assertEquals(await isCommonJSFile('import x from "x";\nx();', file, undefined), false); - assertEquals(await isCommonJSFile("export const a = 1;", file, undefined), false); - assertEquals(await isCommonJSFile("const u = import.meta.url;", file, undefined), false); - assertEquals(await isCommonJSFile('const m = import("x");', file, undefined), true); + it("stops at node_modules, as Node does, so a package without a manifest is CommonJS", async () => { + assertEquals(await lookup().of("/app/node_modules/bare/lib"), undefined); + assertEquals(await lookup().of("/app/node_modules/@scope/bare"), undefined); }); }); - describe("wrapCommonJSPaths()", () => { - const file = "/app/node_modules/a/index.js"; + describe("nodePreferredFile()", () => { + const files = new Set(["/d/impl.js", "/d/lib/index.js", "/d/data.json"]); + const isFile = (path: string) => Promise.resolve(files.has(path)); - it("leaves a file that does not read the wrapper bindings unchanged", () => { - assertEquals(wrapCommonJSPaths("module.exports = 1;", file), "module.exports = 1;"); - }); - - it("passes both bindings, so a file's own declaration shadows only its own name", () => { - const wrapped = wrapCommonJSPaths( - "var __dirname = 'custom'; module.exports = [__dirname, __filename];", - file, + it("prefers Node's extensions over a TypeScript sibling the bundler probed", async () => { + assertEquals(await nodePreferredFile("/d/impl.ts", "./impl", "/d", isFile), "/d/impl.js"); + assertEquals( + await nodePreferredFile("/d/lib/index.tsx", "./lib", "/d", isFile), + "/d/lib/index.js", ); - const module = { exports: [] as string[] }; - new Function("module", wrapped)(module); - assertEquals(module.exports, ["custom", file]); + assertEquals(await nodePreferredFile("/d/data.mjs", "./data", "/d", isFile), "/d/data.json"); }); - it("keeps a hashbang first and a directive prologue in effect", () => { - const wrapped = wrapCommonJSPaths( - '#!/usr/bin/env node\n"use strict";\nmodule.exports = [__dirname, (function () { return this; })()];', - file, - ); - assert( - wrapped.startsWith("#!/usr/bin/env node\n(function (__dirname, __filename) {"), - wrapped, + it("keeps a path the import named exactly or a package chose", async () => { + assertEquals(await nodePreferredFile("/d/impl.ts", "./impl.ts", "/d", isFile), "/d/impl.ts"); + assertEquals( + await nodePreferredFile("/d/node_modules/p/index.cjs", "p", "/d", isFile), + "/d/node_modules/p/index.cjs", ); - const module = { exports: [] as unknown[] }; - new Function("module", wrapped.slice(wrapped.indexOf("\n") + 1))(module); - assertEquals(module.exports, ["/app/node_modules/a", undefined]); + assertEquals(await nodePreferredFile("/d/only.ts", "./only", "/d", isFile), "/d/only.ts"); }); }); - describe("rewriteStaticRequireResolve()", () => { - it("replaces resolvable literals and leaves the rest for runtime", async () => { - const source = [ - 'const a = require.resolve("./asset.json");', - "const b = require.resolve('optional-missing');", - 'const c = require.resolve("fs");', - 'const d = notrequire.resolve("./asset.json");', - ].join("\n"); - const out = await rewriteStaticRequireResolve( - source, - (specifier) => - Promise.resolve(specifier === "./asset.json" ? "/app/node_modules/a/asset.json" : null), - ); - assertEquals( - out, - [ - 'const a = "/app/node_modules/a/asset.json";', - "const b = require.resolve('optional-missing');", - 'const c = "fs";', - 'const d = notrequire.resolve("./asset.json");', - ].join("\n"), - ); + describe("nodeRelativeFile()", () => { + const files = new Set([ + "/d/impl.js", + "/d/impl.ts", + "/d/exact", + "/d/lib/index.js", + "/d/lib/index.ts", + "/d/pkg/package.json", + "/d/addon.node", + ]); + const isFile = (path: string) => Promise.resolve(files.has(path)); + + it("picks Node's candidates for an extensionless relative import", async () => { + assertEquals(await nodeRelativeFile("./impl", "/d", isFile), "/d/impl.js"); + assertEquals(await nodeRelativeFile("./lib", "/d", isFile), "/d/lib/index.js"); + assertEquals(await nodeRelativeFile("/d/impl", "/x", isFile), "/d/impl.js"); + assertEquals(await nodeRelativeFile("./addon", "/d", isFile), "/d/addon.node"); + }); + + it("leaves exact files, package directories, bare and missing imports to the bundler", async () => { + assertEquals(await nodeRelativeFile("./impl.ts", "/d", isFile), null); + assertEquals(await nodeRelativeFile("./exact", "/d", isFile), null); + assertEquals(await nodeRelativeFile("./pkg", "/d", isFile), null); + assertEquals(await nodeRelativeFile("impl", "/d", isFile), null); + assertEquals(await nodeRelativeFile("./missing", "/d", isFile), null); }); }); @@ -116,5 +104,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { const code = prependRuntimeRequire("const __vf_builtin0 = 1;\n", ["path"], "/app"); assert(code.includes("import * as __vf_builtin_0 from"), code); }); + + it("resolves a prefix-only built-in only with its prefix", () => { + const code = prependRuntimeRequire("export {};\n", [], "/app"); + assert(code.includes('"node:test": 1'), code); + assert(!code.includes('"test": 1'), code); + }); }); }); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 13e9e63d2c..b45322224c 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -10,26 +10,34 @@ * - A `require()` of a built-in stays external. esbuild emits it as a call to * the module's free `require`, which {@link prependRuntimeRequire} binds to * built-ins imported statically, so nothing loads `node:module` at runtime. - * - `__dirname` and `__filename` are not defined in an ES module, so - * {@link wrapCommonJSPaths} passes them to each CommonJS file that reads - * them, as Node's CommonJS wrapper does. - * - `require.resolve("literal")` is resolved at build time to the file's path - * on disk ({@link rewriteStaticRequireResolve}). What is left for runtime, a - * computed `require()` or `require.resolve()`, gets built-ins, existing - * project paths and JSON files; anything else throws `MODULE_NOT_FOUND`. + * - {@link prepareCommonJSFile} parses each dependency file once. A CommonJS + * file gets `__dirname` and `__filename` as constants wherever it reads them + * unbound, and each `require.resolve("literal")` becomes the path Node would + * return. What is left for runtime, a computed `require()` or + * `require.resolve()`, gets built-ins, existing project paths and JSON files; + * anything else throws `MODULE_NOT_FOUND`. + * - Inside dependencies, an extensionless `require()` prefers the file Node + * would load ({@link createDependencyResolvePlugin}), not a TypeScript + * sibling the project's own resolution would pick. * - A native addon cannot be bundled or loaded; the build rejects it with a * clear message rather than an esbuild loader error. * * @module routing/api/module-loader/compiled-cjs-bundling */ import { builtinModules } from "node:module"; +import { build as bundle } from "veryfront/extensions/bundler"; import type { Plugin, PluginBuild } from "veryfront/extensions/bundler"; -import { parseImports } from "#veryfront/transforms/esm/lexer.ts"; -import { toFileUrl } from "#veryfront/compat/path"; +import * as pathHelper from "#veryfront/compat/path"; +import { isWithinDirectory } from "#veryfront/security/path-validation.ts"; // Names Node accepts only with the `node:` prefix. A bare `test` is an npm package. const PREFIX_ONLY_BUILTINS = new Set(["test", "test/reporters", "sqlite", "sea"]); -const BUILTINS = new Set(builtinModules.map((name) => name.replace(/^node:/, ""))); +// Names Node accepts without the prefix. +const BUILTINS = new Set( + builtinModules + .map((name) => name.replace(/^node:/, "")) + .filter((name) => !PREFIX_ONLY_BUILTINS.has(name)), +); /** The built-in module a specifier names, without its `node:` prefix. */ export function builtinModuleName(specifier: string): string | undefined { @@ -37,10 +45,19 @@ export function builtinModuleName(specifier: string): string | undefined { const name = specifier.slice(5); return BUILTINS.has(name) || PREFIX_ONLY_BUILTINS.has(name) ? name : undefined; } - if (PREFIX_ONLY_BUILTINS.has(specifier)) return undefined; return BUILTINS.has(specifier) ? specifier : undefined; } +const NATIVE_ADDON = /\.node$/i; + +function nativeAddonError(path: string) { + return { + errors: [{ + text: `Native addons are not supported in API routes: ${JSON.stringify(path)}`, + }], + }; +} + /** * Keeps built-ins external and records each one a bundled module `require`s, * so {@link prependRuntimeRequire} can bind it. Must run before any plugin @@ -59,74 +76,288 @@ export function createCompiledBuiltinPlugin(requiredBuiltins: Set): Plug } return { path: `node:${name}`, external: true }; }); - build.onResolve({ filter: /\.node$/ }, (args) => ({ - errors: [{ - text: `Native addons are not supported in API routes: ${JSON.stringify(args.path)}`, - }], - })); + build.onResolve({ filter: NATIVE_ADDON }, (args) => nativeAddonError(args.path)); }, }; } -// An `export` statement; the module lexer reports only imports. -const EXPORT_STATEMENT = - /(?:^|[;}\n])\s*export\s*(?:default\b|[{*]|(?:async\s+)?function\b|class\b|const\b|let\b|var\b)/; +/** + * Resolution options of a compiled-binary route build. Bundled packages read + * the `node` condition and `main`, as Node does; Node never reads `module`. + * The extension order serves project code, and + * {@link createDependencyResolvePlugin} restores Node's order inside packages. + */ +export const COMPILED_RESOLVE_OPTIONS = { + mainFields: ["main"], + conditions: ["node"], + resolveExtensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".json"], +}; + +/** Whether a path is a regular file on the host. */ +export type IsFile = (path: string) => Promise; + +// Extensions Node's require() tries, in order, for an extensionless path. +const NODE_REQUIRE_EXTENSIONS = [".js", ".json", ".node"]; /** - * Whether Node would load a dependency file as CommonJS: a `.cjs` file, or a - * `.js` file outside a `"type": "module"` package scope that has no ESM syntax. - * esbuild bundles ESM syntax in a `.js` file as ESM whatever the package says. + * The file Node would load for a relative or absolute import that the bundler + * resolved to `resolvedPath`. The bundler's extension order serves project + * code and puts TypeScript first. When it added an extension Node's require() + * does not try, to the path or to its directory's `index`, Node's own + * candidates for the same stem win. A path the import named exactly, or one a + * package's `exports` or `main` chose, is kept. */ -export async function isCommonJSFile( - contents: string, - filePath: string, - packageType: string | undefined, -): Promise { - if (/\.cjs$/i.test(filePath)) return true; - if (!/\.js$/i.test(filePath) || packageType === "module") return false; - if (EXPORT_STATEMENT.test(contents)) return false; +export async function nodePreferredFile( + resolvedPath: string, + specifier: string, + resolveDir: string, + isFile: IsFile, +): Promise { + const relative = specifier.startsWith("./") || specifier.startsWith("../") || + specifier === "." || specifier === ".." || pathHelper.isAbsolute(specifier); + const extension = pathHelper.extname(resolvedPath); + if (!relative || NODE_REQUIRE_EXTENSIONS.includes(extension)) return resolvedPath; + const stem = resolvedPath.slice(0, resolvedPath.length - extension.length); + const named = pathHelper.resolve(resolveDir, specifier); + if (stem !== named && stem !== pathHelper.join(named, "index")) return resolvedPath; + for (const candidate of NODE_REQUIRE_EXTENSIONS) { + if (await isFile(stem + candidate)) return stem + candidate; + } + return resolvedPath; +} + +const RELATIVE_SPECIFIER = /^(?:\.\.?(?:\/|$)|\/)/; + +/** + * The file Node's require() loads for a relative or absolute extensionless + * import, when that may differ from the bundler's pick: the path plus `.js`, + * `.json` or `.node`, or its directory's `index` with one of them. Null when + * the bundler picks the same file: the path names an existing file exactly, + * it is a package directory whose `main` decides, or Node finds nothing. + */ +export async function nodeRelativeFile( + specifier: string, + resolveDir: string, + isFile: IsFile, +): Promise { + if (!RELATIVE_SPECIFIER.test(specifier)) return null; + const base = pathHelper.resolve(resolveDir, specifier); + if (await isFile(base)) return null; + for (const extension of NODE_REQUIRE_EXTENSIONS) { + if (await isFile(base + extension)) return base + extension; + } + if (await isFile(pathHelper.join(base, "package.json"))) return null; + for (const extension of NODE_REQUIRE_EXTENSIONS) { + const index = pathHelper.join(base, "index" + extension); + if (await isFile(index)) return index; + } + return null; +} + +/** + * Resolves the relative imports of dependency files the way Node does: the + * bundler's extension order serves project code and would load a TypeScript + * sibling first. Only relative imports differ, so only they are probed, with a + * few file checks and no second resolution. Must run after + * {@link createCompiledBuiltinPlugin} and before the project's own resolvers. + */ +export function createDependencyResolvePlugin( + isDependencyFile: (path: string) => boolean, + isFile: IsFile, +): Plugin { + return { + name: "vf-dependency-resolve", + setup(build) { + build.onResolve({ filter: RELATIVE_SPECIFIER }, async (args) => { + if (!args.importer || !isDependencyFile(args.importer)) return undefined; + const file = await nodeRelativeFile(args.path, args.resolveDir, isFile); + if (file === null) return undefined; + if (NATIVE_ADDON.test(file)) return nativeAddonError(args.path); + return { path: file }; + }); + }, + }; +} + +/** Reads a package.json as text, or null when the directory has none. */ +export type ReadManifest = (path: string) => Promise; + +/** + * The `type` field of the package.json that scopes a directory, as Node uses it + * to decide whether a `.js` file is ESM. Like Node, the search stops at a + * `node_modules` directory, so a dependency without a package.json is + * CommonJS whatever the project declares. It never leaves the project. + */ +export class PackageTypeLookup { + readonly #byDirectory = new Map>(); + readonly #root: string; + + constructor(private readonly readManifest: ReadManifest, projectDir: string) { + this.#root = pathHelper.resolve(projectDir); + } + + of(directory: string): Promise { + let found = this.#byDirectory.get(directory); + if (!found) { + found = this.#read(directory); + this.#byDirectory.set(directory, found); + } + return found; + } + + async #read(directory: string): Promise { + if (pathHelper.basename(directory) === "node_modules") return undefined; + const manifest = await this.readManifest(pathHelper.join(directory, "package.json")); + if (manifest !== null) { + try { + const type = (JSON.parse(manifest) as { type?: unknown }).type; + return typeof type === "string" ? type : undefined; + } catch { + // Node rejects a malformed package.json; esbuild reports it as well. + return undefined; + } + } + const parent = pathHelper.dirname(directory); + if (parent === directory || !isWithinDirectory(this.#root, parent)) return undefined; + return await this.of(parent); + } +} + +/** Host operations {@link createDependencyFilePredicate} needs. */ +export interface DependencyRootFileSystem { + realPath(path: string): Promise; + readDir( + path: string, + ): AsyncIterable<{ name: string; isDirectory: boolean; isSymlink?: boolean }>; +} + +async function directoryNames(fs: DependencyRootFileSystem, path: string) { + const names: { name: string; isDirectory: boolean; isSymlink?: boolean }[] = []; try { - // A static import (d === -1) or import.meta (d === -2) is ESM syntax. - return !(await parseImports(contents)).some((specifier) => specifier.d < 0); + for await (const entry of fs.readDir(path)) names.push(entry); } catch { - // expected: the lexer rejects some valid scripts; Node's rule decides. - return true; + // expected: no such directory, as for an adapter-backed project } + return names; } /** - * Passes `__dirname` and `__filename` to a CommonJS file that reads them, as - * Node's module wrapper does: as parameters of a function around the file, so a - * file's own `var __dirname` still shadows them and its directive prologue - * ("use strict") stays first in that function. A hashbang stays first in the - * file. Top-level `this` is passed through, which esbuild maps to `exports`. + * Whether a path the bundler resolved is an installed dependency's file: a + * file under the project's `node_modules`, under either the given or the + * canonical root (esbuild reports paths with symlinks followed), or under a + * workspace package that `node_modules` links to inside the project. */ -export function wrapCommonJSPaths(contents: string, filePath: string): string { - if (!/\b__(?:dirname|filename)\b/.test(contents)) return contents; - const hashbangEnd = contents.startsWith("#!") ? contents.indexOf("\n") + 1 || contents.length : 0; - const dir = filePath.slice(0, Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\"))); - return contents.slice(0, hashbangEnd) + - "(function (__dirname, __filename) {" + contents.slice(hashbangEnd) + - `\n}).call(this, ${JSON.stringify(dir)}, ${JSON.stringify(filePath)});\n`; +export async function createDependencyFilePredicate( + projectDir: string, + fs: DependencyRootFileSystem, +): Promise<(path: string) => boolean> { + const given = pathHelper.resolve(projectDir); + let canonical: string | null = null; + try { + canonical = await fs.realPath(given); + } catch { + // expected: an adapter-backed project may have no host directory + } + const prefixes = new Set( + [given, canonical ?? given].map((root) => + pathHelper.join(root, "node_modules") + pathHelper.sep + ), + ); + + if (canonical !== null) { + const modules = pathHelper.join(canonical, "node_modules"); + const links: string[] = []; + for (const entry of await directoryNames(fs, modules)) { + const path = pathHelper.join(modules, entry.name); + // Only a link can point outside node_modules; a host that does not say + // whether an entry is one gets every entry checked. + if (!entry.name.startsWith("@")) { + if (entry.isSymlink !== false) links.push(path); + continue; + } + for (const scoped of await directoryNames(fs, path)) { + if (scoped.isSymlink !== false) links.push(pathHelper.join(path, scoped.name)); + } + } + for (const link of links) { + let target: string; + try { + target = await fs.realPath(link); + } catch { + continue; // expected: a dangling link + } + // A package linked to a directory of the project itself, as npm, Yarn + // and pnpm workspaces do. The project root and node_modules never count. + if ( + target !== link && target !== canonical && isWithinDirectory(canonical, target) && + !isWithinDirectory(modules, target) + ) { + prefixes.add(target + pathHelper.sep); + } + } + } + + const roots = [...prefixes]; + return (path) => { + if (path === "") return false; + const resolved = pathHelper.resolve(path); + return roots.some((prefix) => resolved.startsWith(prefix)); + }; } -const STATIC_REQUIRE_RESOLVE = /(^|[^\w$.])require\.resolve\(\s*(["'])([^"'\\\n]+)\2\s*\)/g; +/** Resolves a `require.resolve` literal to a path, or null to leave it for runtime. */ +export type ResolveRequireLiteral = (specifier: string) => Promise; /** - * Replaces each `require.resolve("literal")` that resolves at build time with - * the string Node would return: the resolved file path, or the id itself for a - * built-in. One that does not resolve is left for runtime, where it throws - * `MODULE_NOT_FOUND` as Node does, so an optional dependency probed this way - * still falls back. + * A dependency file as Node would run it when Node loads it as CommonJS, or + * null when Node loads it as ESM: a `.mjs` file, a `.js` file in a + * `"type": "module"` package, or a `.js` file with ESM syntax, which esbuild + * bundles as ESM whatever its package says. + * + * The file is parsed, not matched as text, so strings and comments are never + * touched. Unbound reads of `__dirname` and `__filename` become constants, so + * a file's own binding of either name still wins, and nothing wraps the file: + * its hashbang, directive prologue and `arguments` are unchanged. Each unbound + * `require.resolve("literal")` becomes the string Node returns: the id of a + * built-in, or the path `resolve` finds. One that does not resolve is left for + * runtime, where it throws `MODULE_NOT_FOUND` as Node does, so an optional + * dependency probed this way still falls back. */ -export async function rewriteStaticRequireResolve( +export async function prepareCommonJSFile( contents: string, - resolve: (specifier: string) => Promise, -): Promise { - if (!contents.includes("require.resolve")) return contents; + filePath: string, + packageType: string | undefined, + resolve: ResolveRequireLiteral, +): Promise { + const cjsExtension = /\.cjs$/i.test(filePath); + const jsExtension = /\.js$/i.test(filePath); + if (!cjsExtension && !(jsExtension && packageType !== "module")) return null; + + // A fresh name per file: the source cannot contain it, so every occurrence + // in the output is one the parser put there. It never reaches the bundle. + const marker = `__vf_require_resolve_${crypto.randomUUID().replaceAll("-", "")}`; + const result = await bundle({ + stdin: { contents, loader: "js", sourcefile: filePath }, + bundle: false, + write: false, + metafile: true, + platform: "neutral", + logLevel: "silent", + define: { + "require.resolve": marker, + __dirname: JSON.stringify(pathHelper.dirname(filePath)), + __filename: JSON.stringify(filePath), + }, + }); + const format = Object.values(result.metafile?.inputs ?? {})[0]?.format; + if (jsExtension && format === "esm") return null; + const code = result.outputFiles?.[0]?.text; + if (code === undefined) throw new Error(`No output for dependency file ${filePath}`); + + const literalCall = new RegExp(`${marker}\\((["'])([^"'\\\\\\n]*)\\1\\)`, "g"); const resolved = new Map(); - for (const match of contents.matchAll(STATIC_REQUIRE_RESOLVE)) { - const specifier = match[3]!; + for (const match of code.matchAll(literalCall)) { + const specifier = match[2]!; if (!resolved.has(specifier)) { resolved.set( specifier, @@ -134,13 +365,14 @@ export async function rewriteStaticRequireResolve( ); } } - return contents.replace( - STATIC_REQUIRE_RESOLVE, - (whole, before: string, _q, specifier: string) => { + return code + .replace(literalCall, (_whole, _quote, specifier: string) => { const target = resolved.get(specifier); - return typeof target === "string" ? before + JSON.stringify(target) : whole; - }, - ); + return typeof target === "string" + ? JSON.stringify(target) + : `require.resolve(${JSON.stringify(specifier)})`; + }) + .replaceAll(marker, "require.resolve"); } /** Input for {@link prepareDependencyFile}. */ @@ -151,43 +383,51 @@ export interface DependencyFileInput { readonly logicalPath: string; /** The path the bundler resolved, used as the importer for resolution. */ readonly resolvedPath: string; - /** The `type` field of the package.json nearest the file. */ + /** The `type` field of the package.json that scopes the file. */ readonly packageType: string | undefined; readonly isDependencyFile: (path: string) => boolean; + readonly isFile: IsFile; /** Points a module's `import.meta` locations at its own URL. */ readonly rewriteImportMeta: (contents: string, moduleUrl: string) => Promise; } /** - * A bundled dependency file as Node would run it. In a CommonJS file, - * `require.resolve("literal")` becomes the path the bundler resolves inside - * the project's dependencies, and `__dirname`/`__filename` are passed in as - * Node's wrapper passes them. An ESM file keeps its own semantics; only its - * `import.meta` locations are pointed at the file's own URL. + * A bundled dependency file as Node would run it: see + * {@link prepareCommonJSFile}. In a CommonJS file a `require.resolve` literal + * resolves only to a file inside the project's dependencies. An ESM file keeps + * its own semantics; only its `import.meta` locations point at its own URL. */ export async function prepareDependencyFile(input: DependencyFileInput): Promise { const { build, contents, logicalPath, resolvedPath, isDependencyFile } = input; - if (!await isCommonJSFile(contents, logicalPath, input.packageType)) { - return /\bimport\.meta\b/.test(contents) - ? await input.rewriteImportMeta(contents, toFileUrl(logicalPath).href) - : contents; - } - const resolveDir = resolvedPath.slice( - 0, - Math.max(resolvedPath.lastIndexOf("/"), resolvedPath.lastIndexOf("\\")), + // Most package files read none of these names; parsing them would change + // nothing. The check is textual, so it can only send extra files to the parser. + const importMeta = /\bimport\.meta\b/.test(contents); + const readsCommonJSNames = contents.includes("__dirname") || + contents.includes("__filename") || + (contents.includes("require") && contents.includes("resolve")); + if (!readsCommonJSNames && !importMeta) return contents; + const resolveDir = pathHelper.dirname(resolvedPath); + const commonJS = await prepareCommonJSFile( + contents, + logicalPath, + input.packageType, + async (specifier) => { + if (!build.resolve) return null; + const result = await build.resolve(specifier, { + kind: "require-resolve", + importer: resolvedPath, + resolveDir, + }); + // Only a file inside the project's dependencies is a path to hand back. + if (result.errors?.length || !result.path || result.external) return null; + if (!isDependencyFile(result.path)) return null; + return await nodePreferredFile(result.path, specifier, resolveDir, input.isFile); + }, ); - const resolved = await rewriteStaticRequireResolve(contents, async (specifier) => { - if (!build.resolve) return null; - const result = await build.resolve(specifier, { - kind: "require-resolve", - importer: resolvedPath, - resolveDir, - }); - // Only a file inside the project's dependencies is a path to hand back. - if (result.errors?.length || !result.path || result.external) return null; - return isDependencyFile(result.path) ? result.path : null; - }); - return wrapCommonJSPaths(resolved, logicalPath); + if (commonJS !== null) return commonJS; + return importMeta + ? await input.rewriteImportMeta(contents, pathHelper.toFileUrl(logicalPath).href) + : contents; } /** @@ -196,8 +436,9 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise * id, an existing file inside the project, which `require.resolve` returns and * `require` loads when it is JSON. Anything else throws `MODULE_NOT_FOUND`, so * optional dependencies probed in a `try` still fall back. Errors name a path - * relative to the project, never the host path. esbuild renames route bindings - * named `require`, so the prologue cannot collide with route code. + * relative to the project, never the host path. esbuild's ESM output never + * declares a top-level `require` (it renames a route's own binding of that + * name), so the prologue cannot collide with route code. */ export function prependRuntimeRequire( code: string, @@ -222,8 +463,8 @@ export function prependRuntimeRequire( fields.push(`${JSON.stringify(id)}: ${prefix}${index}.default ?? ${prefix}${index}`); } const root = JSON.stringify(projectRoot); - // Every id Node's require.resolve returns unchanged: a bare built-in name, or - // any built-in, prefix-only ones included, with the node: prefix. + // Every id Node's require.resolve returns unchanged: a built-in Node accepts + // without the prefix, or any built-in with the node: prefix. const builtinIds = [ ...BUILTINS, ...[...BUILTINS, ...PREFIX_ONLY_BUILTINS].map((name) => "node:" + name), diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index a199fbcdbc..1cced184f6 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -49,7 +49,12 @@ import { rewriteExternalImports, } from "./external-import-rewriter.ts"; import { + COMPILED_RESOLVE_OPTIONS, createCompiledBuiltinPlugin, + createDependencyFilePredicate, + createDependencyResolvePlugin, + type IsFile, + PackageTypeLookup, prepareDependencyFile, prependRuntimeRequire, } from "./compiled-cjs-bundling.ts"; @@ -1307,11 +1312,6 @@ type DependencyFilePredicate = (path: string) => boolean; const noDependencyFiles: DependencyFilePredicate = () => false; -/** - * Files under the project's own node_modules, which a compiled-binary route - * bundles. Both the given and the canonical project root count: esbuild - * reports resolved paths with symlinks followed. - */ /** The project root with symlinks followed, or as given when it has none on the host. */ async function canonicalProjectRoot(projectDir: string): Promise { try { @@ -1322,60 +1322,25 @@ async function canonicalProjectRoot(projectDir: string): Promise { } } -async function projectDependencyFiles(projectDir: string): Promise { - const roots = new Set([pathHelper.resolve(projectDir)]); +/** Whether a path is a regular file on the host. */ +const hostIsFile: IsFile = async (path) => { try { - roots.add(await realPath(projectDir)); + return (await createFileSystem().stat(path)).isFile; } catch { - // expected: an adapter-backed project may have no host directory - } - const prefixes = [...roots].map((root) => pathHelper.join(root, "node_modules") + pathHelper.sep); - return (path) => - path !== "" && prefixes.some((prefix) => pathHelper.resolve(path).startsWith(prefix)); -} - -/** - * The `type` field of the package.json nearest a directory, as Node uses it to - * decide whether a `.js` file is ESM. The walk stays inside the project. - */ -class PackageTypeLookup { - readonly #byDirectory = new Map>(); - readonly #root: string; - - constructor(private readonly sourceSnapshot: ProjectSourceSnapshot, projectDir: string) { - this.#root = pathHelper.resolve(projectDir); + return false; // expected: no such file } +}; - of(directory: string): Promise { - let found = this.#byDirectory.get(directory); - if (!found) { - found = this.#read(directory); - this.#byDirectory.set(directory, found); - } - return found; - } - - async #read(directory: string): Promise { - let manifest: string | null = null; +/** Reads a package.json through the project snapshot, or null when there is none. */ +function snapshotManifestReader(sourceSnapshot: ProjectSourceSnapshot) { + return async (path: string): Promise => { try { - manifest = await this.sourceSnapshot.readTextFile(pathHelper.join(directory, "package.json")); + return await sourceSnapshot.readTextFile(path); } catch (error) { if (error instanceof ProjectBoundaryViolationError) throw error; - // expected: no package.json here; look in the parent directory - } - if (manifest !== null) { - try { - const type = (JSON.parse(manifest) as { type?: unknown }).type; - return typeof type === "string" ? type : undefined; - } catch { - // Node rejects a malformed package.json; esbuild reports it as well. - return undefined; - } + return null; // expected: no package.json here } - const parent = pathHelper.dirname(directory); - if (parent === directory || !isWithinDirectory(this.#root, parent)) return undefined; - return await this.of(parent); - } + }; } function createImportMapPlugin( @@ -1955,7 +1920,7 @@ function createProjectBoundaryPlugin( workerImportMap: DenoImportMap | null, isDependencyFile: DependencyFilePredicate = noDependencyFiles, ): Plugin { - const packageTypes = new PackageTypeLookup(sourceSnapshot, projectDir); + const packageTypes = new PackageTypeLookup(snapshotManifestReader(sourceSnapshot), projectDir); return { name: "vf-project-boundary", setup(build) { @@ -1995,6 +1960,7 @@ function createProjectBoundaryPlugin( resolvedPath: args.path, packageType: await packageTypes.of(pathHelper.dirname(source.logicalPath)), isDependencyFile, + isFile: hostIsFile, rewriteImportMeta: (contents, moduleUrl) => rewriteBundledImportMetaUrl(contents, moduleUrl, null), }), @@ -2220,7 +2186,10 @@ function buildTranspiledModuleSource( ].join("\n"); const requiredBuiltins = new Set(); const isDependencyFile = compiled - ? await projectDependencyFiles(projectDir) + ? await createDependencyFilePredicate(projectDir, { + realPath, + readDir: (path) => createFileSystem().readDir(path), + }) : noDependencyFiles; const routeSourceUrl = pathHelper.toFileUrl(resolvedPath).href; const workerShim = [ @@ -2249,11 +2218,9 @@ function buildTranspiledModuleSource( target: "es2022", jsx: "automatic", jsxImportSource: "react", - resolveExtensions: compiled - ? [".ts", ".tsx", ".js", ".jsx", ".mjs", ".json"] - : [".ts", ".tsx", ".js", ".jsx", ".mjs"], + resolveExtensions: [".ts", ".tsx", ".js", ".jsx", ".mjs"], // Bundled CommonJS packages resolve as Node resolves them for require(). - ...(compiled ? { mainFields: ["main", "module"], conditions: ["node"] } : {}), + ...(compiled ? COMPILED_RESOLVE_OPTIONS : {}), // Injection binds only free identifiers and lets esbuild rename either // side of a collision. A raw banner would redeclare a route import such // as `import { Worker } from "worker-package"` and emit invalid ESM. @@ -2275,7 +2242,12 @@ function buildTranspiledModuleSource( sourcefile: resolvedPath, }, plugins: [ - ...(compiled ? [createCompiledBuiltinPlugin(requiredBuiltins)] : []), + ...(compiled + ? [ + createCompiledBuiltinPlugin(requiredBuiltins), + createDependencyResolvePlugin(isDependencyFile, hostIsFile), + ] + : []), createRouteRuntimeShimPlugin(routeRuntimeShim), createImportMapPlugin( projectDir, diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index b49d27abd8..1d6b638f31 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,13 +1,17 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; -import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import { mkdir, readdir, readFile, realpath, stat, symlink, writeFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; import { withTempDir } from "#veryfront/testing/deno-compat.ts"; import * as esbuild from "veryfront/extensions/bundler"; import type { Plugin } from "veryfront/extensions/bundler"; import { + COMPILED_RESOLVE_OPTIONS, createCompiledBuiltinPlugin, + createDependencyFilePredicate, + createDependencyResolvePlugin, + PackageTypeLookup, prepareDependencyFile, prependRuntimeRequire, } from "#veryfront/routing/api/module-loader/compiled-cjs-bundling.ts"; @@ -20,25 +24,44 @@ async function writeFiles(root: string, files: Record): Promise< } } -/** The `type` of the package.json nearest a file, as the route build reads it. */ -async function packageTypeOf(filePath: string, root: string): Promise { - for (let dir = filePath.slice(0, filePath.lastIndexOf("/")); dir.startsWith(root);) { +const isFile = async (path: string) => { + try { + return (await stat(path)).isFile(); + } catch { + return false; // expected: no such file + } +}; + +const hostFs = { + realPath: (path: string) => realpath(path), + async *readDir(path: string) { + for (const entry of await readdir(path, { withFileTypes: true })) { + yield { + name: entry.name, + isDirectory: entry.isDirectory(), + isSymlink: entry.isSymbolicLink(), + }; + } + }, +}; + +/** Prepares dependency files the way the compiled route build does. */ +function dependencyFilePlugin( + projectDir: string, + isDependencyFile: (path: string) => boolean, +): Plugin { + const packageTypes = new PackageTypeLookup(async (path) => { try { - return JSON.parse(await readFile(`${dir}/package.json`, "utf8")).type; + return await readFile(path, "utf8"); } catch { - dir = dir.slice(0, dir.lastIndexOf("/")); + return null; // expected: no package.json here } - } - return undefined; -} - -/** Prepares node_modules files the way the compiled route build does. */ -function dependencyFilePlugin(projectDir: string): Plugin { - const isDependencyFile = (path: string) => path.startsWith(`${projectDir}/node_modules/`); + }, projectDir); return { name: "test-dependency-files", setup(build) { - build.onLoad({ filter: /node_modules[\\/].*\.[cm]?js$/ }, async (args) => { + build.onLoad({ filter: /\.[cm]?js$/ }, async (args) => { + if (!isDependencyFile(args.path)) return undefined; const contents = await readFile(args.path, "utf8"); return { contents: await prepareDependencyFile({ @@ -46,8 +69,9 @@ function dependencyFilePlugin(projectDir: string): Plugin { contents, logicalPath: args.path, resolvedPath: args.path, - packageType: await packageTypeOf(args.path, projectDir), + packageType: await packageTypes.of(args.path.slice(0, args.path.lastIndexOf("/"))), isDependencyFile, + isFile, rewriteImportMeta: (source) => Promise.resolve(source), }), loader: "js", @@ -63,6 +87,7 @@ async function bundleAndImport( route: string, ): Promise> { const requiredBuiltins = new Set(); + const isDependencyFile = await createDependencyFilePredicate(projectDir, hostFs); try { const result = await esbuild.build({ bundle: true, @@ -70,9 +95,7 @@ async function bundleAndImport( format: "esm", platform: "neutral", target: "es2022", - mainFields: ["main", "module"], - conditions: ["node"], - resolveExtensions: [".ts", ".js", ".mjs", ".json"], + ...COMPILED_RESOLVE_OPTIONS, logLevel: "silent", stdin: { contents: route, @@ -80,7 +103,11 @@ async function bundleAndImport( resolveDir: projectDir, sourcefile: `${projectDir}/route.ts`, }, - plugins: [createCompiledBuiltinPlugin(requiredBuiltins), dependencyFilePlugin(projectDir)], + plugins: [ + createCompiledBuiltinPlugin(requiredBuiltins), + createDependencyResolvePlugin(isDependencyFile, isFile), + dependencyFilePlugin(projectDir, isDependencyFile), + ], }); const code = prependRuntimeRequire( result.outputFiles?.[0]?.text ?? "", @@ -161,6 +188,9 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { 'var hosts = ["", "etc", "hosts"].join("/");', "try { require(hosts); } catch (error) { outside = [error.code, error.message]; }", "exports.outside = outside;", + "var prefixOnly;", + 'try { require.resolve("test"); } catch (error) { prefixOnly = error.code; }', + "exports.prefixOnly = prefixOnly;", ].join("\n"), }); const mod = await bundleAndImport( @@ -179,6 +209,7 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { "MODULE_NOT_FOUND", 'Cannot find module "a path outside this project or missing"', ], + prefixOnly: "MODULE_NOT_FOUND", }); }); }); @@ -201,11 +232,101 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { ); // Checked in the bundle, not at runtime: Bun defines both names in ESM too. const code = await readFile(`${projectDir}/handler.mjs`, "utf8"); - assert(!code.includes("function (__dirname, __filename)"), code); assert(code.includes("typeof __dirname") && code.includes("typeof __filename"), code); }); }); + it("parses a CommonJS file instead of matching its text", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/parsed/package.json": JSON.stringify({ name: "parsed", main: "index.js" }), + "node_modules/parsed/data.json": "{}", + "node_modules/parsed/index.js": [ + "var text = 'require.resolve(\"./data.json\")';", + "/*", + "export default 1", + "*/", + "var __filename = 'own';", + "exports.text = text;", + "exports.dir = __dirname;", + "exports.file = __filename;", + "exports.firstArgument = arguments[0] === exports;", + ].join("\n"), + }); + const mod = await bundleAndImport( + projectDir, + 'import parsed from "parsed";\nexport { parsed };\n', + ); + assertEquals(mod.parsed, { + text: 'require.resolve("./data.json")', + dir: `${projectDir}/node_modules/parsed`, + file: "own", + // No wrapper of ours: arguments are the bundler's CommonJS wrapper's. + firstArgument: true, + }); + }); + }); + + it("bundles a workspace package that node_modules links to", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "packages/ws/package.json": JSON.stringify({ name: "ws", main: "index.js" }), + "packages/ws/index.js": + 'exports.dir = __dirname;\nexports.self = require.resolve("./index.js");', + }); + await mkdir(`${projectDir}/node_modules`, { recursive: true }); + await symlink("../packages/ws", `${projectDir}/node_modules/ws`); + const mod = await bundleAndImport(projectDir, 'import ws from "ws";\nexport { ws };\n'); + assertEquals(mod.ws, { + dir: `${projectDir}/packages/ws`, + self: `${projectDir}/packages/ws/index.js`, + }); + }); + }); + + it("resolves inside dependencies in Node's order", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/order/package.json": JSON.stringify({ name: "order", main: "index.js" }), + "node_modules/order/index.js": + 'exports.impl = require("./impl");\nexports.data = require("./data");', + "node_modules/order/data.mjs": 'export default "mjs";', + "node_modules/order/data.json": '"json"', + // A workspace package resolves outside node_modules, where the + // bundler's own order would pick the TypeScript sibling. + "packages/wsorder/package.json": JSON.stringify({ name: "wsorder", main: "index.js" }), + "packages/wsorder/index.js": 'module.exports = require("./impl");', + "packages/wsorder/impl.js": 'module.exports = "js";', + "packages/wsorder/impl.ts": 'export default "ts";', + "node_modules/order/impl.js": 'module.exports = "js";', + "node_modules/order/impl.ts": 'export default "ts";', + "node_modules/modonly/package.json": JSON.stringify({ name: "modonly", module: "esm.js" }), + "node_modules/modonly/index.js": 'module.exports = "index";', + "node_modules/modonly/esm.js": 'export default "module";', + }); + await symlink("../packages/wsorder", `${projectDir}/node_modules/wsorder`); + const mod = await bundleAndImport( + projectDir, + 'import order from "order";\nimport modonly from "modonly";\nimport wsorder from "wsorder";\n' + + "export const picked = [order.impl, order.data, modonly, wsorder];\n", + ); + assertEquals(mod.picked, ["js", "json", "index", "js"]); + }); + }); + + it("keeps a route's own require binding when nothing is bundled", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "helper.ts": 'export const require = (id: string) => "helper:" + id;\n', + }); + const mod = await bundleAndImport( + projectDir, + 'import { require } from "./helper.ts";\nexport const value = require("q");\n', + ); + assertEquals(mod.value, "helper:q"); + }); + }); + it("rejects a native addon at build time", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { From ab1a42d5766911dcd3c3be880f87f5f2547e0b3b Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 20:44:32 +0100 Subject: [PATCH 09/29] fix(api): cache JSON loaded by a computed require in compiled routes Node's module cache returns the same object for every require() of a file; the runtime require now keeps parsed JSON by real path. --- src/routing/api/module-loader/compiled-cjs-bundling.ts | 7 ++++++- .../api/module-loader/compiled-cjs-bundling.test.ts | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index b45322224c..d63c3378c3 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -474,6 +474,8 @@ export function prependRuntimeRequire( // not depend on built-ins route code could replace. `const ${prefix}s = { ${fields.join(", ")} };`, `const ${prefix}_ids = { __proto__: null, ${builtinIds.join(", ")} };`, + // JSON a computed require() loaded, by real path, as Node's module cache keeps it. + `const ${prefix}_json = { __proto__: null };`, `const ${prefix}_missing = (label) => {`, ' const error = new Error("Cannot find module " + JSON.stringify(label));', ' error.code = "MODULE_NOT_FOUND";', @@ -493,7 +495,10 @@ export function prependRuntimeRequire( "const require = (id) => {", ` if (typeof id === "string" && id in ${prefix}s) return ${prefix}s[id];`, ` const file = ${prefix}_file(id);`, - ` if (file !== null && file.real.endsWith(".json")) return JSON.parse(${prefix}_fs.readFileSync(file.real, "utf8"));`, + ` if (file !== null && file.real.endsWith(".json")) {`, + ` if (!(file.real in ${prefix}_json)) ${prefix}_json[file.real] = JSON.parse(${prefix}_fs.readFileSync(file.real, "utf8"));`, + ` return ${prefix}_json[file.real];`, + " }", ` throw ${prefix}_missing(file !== null ? file.relative : ${prefix}_label(id));`, "};", "require.resolve = (id) => {", diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 1d6b638f31..ea0c14aede 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -184,6 +184,7 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { 'var computed = path.join(__dirname, "data", "en.json");', "exports.computedPath = require.resolve(computed);", "exports.computed = require(computed).hello;", + "exports.cached = require(computed) === require(computed);", "var outside;", 'var hosts = ["", "etc", "hosts"].join("/");', "try { require(hosts); } catch (error) { outside = [error.code, error.message]; }", @@ -205,6 +206,7 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { missing: "MODULE_NOT_FOUND", computedPath: asset, computed: "world", + cached: true, outside: [ "MODULE_NOT_FOUND", 'Cannot find module "a path outside this project or missing"', From f2b2de6863f2c50e3f95d939f58bdb335ba74b0e Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 21:02:50 +0100 Subject: [PATCH 10/29] test(api): unit-cover compiled CommonJS bundling and fix the Bun shard Split the dependency predicate into small helpers and cover it, both plugins, the runtime require and the unparsed dependency paths with unit tests. Rename the workspace fixture package: Bun lists ws as a built-in. --- .../compiled-cjs-bundling.test.ts | 190 ++++++++++++++++++ .../module-loader/compiled-cjs-bundling.ts | 111 +++++----- .../compiled-cjs-bundling.test.ts | 17 +- 3 files changed, 260 insertions(+), 58 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index cf3ce01541..0d01465ef1 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,14 +1,35 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import type { PluginBuild } from "veryfront/extensions/bundler"; import { builtinModuleName, + createCompiledBuiltinPlugin, + createDependencyFilePredicate, + createDependencyResolvePlugin, nodePreferredFile, nodeRelativeFile, PackageTypeLookup, + prepareDependencyFile, prependRuntimeRequire, } from "./compiled-cjs-bundling.ts"; +type ResolveHandler = (args: Record) => unknown; + +/** The onResolve handlers a plugin registers, in order. */ +async function resolveHandlers( + plugin: { setup(build: PluginBuild): void | Promise }, +): Promise { + const handlers: ResolveHandler[] = []; + const build = { + onResolve: (_options: unknown, handler: ResolveHandler) => handlers.push(handler), + onLoad: () => {}, + onDispose: () => {}, + }; + await plugin.setup(build as unknown as PluginBuild); + return handlers; +} + describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { describe("builtinModuleName()", () => { it("names built-ins with or without the node: prefix, including subpaths", () => { @@ -94,7 +115,176 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); }); + describe("createDependencyFilePredicate()", () => { + const links: Record = { + "/app": "/real/app", + "/real/app/node_modules/linked": "/real/app/packages/linked", + "/real/app/node_modules/@s/w": "/real/app/packages/w", + "/real/app/node_modules/out": "/elsewhere/out", + "/real/app/node_modules/self": "/real/app", + "/real/app/node_modules/inner": "/real/app/node_modules/plain", + }; + const entries: Record = { + "/real/app/node_modules": [ + { name: "linked", isDirectory: false, isSymlink: true }, + { name: "@s", isDirectory: true, isSymlink: false }, + { name: "plain", isDirectory: true, isSymlink: false }, + { name: "out", isDirectory: false }, + { name: "self", isDirectory: false, isSymlink: true }, + { name: "inner", isDirectory: false, isSymlink: true }, + { name: "dead", isDirectory: false, isSymlink: true }, + ], + "/real/app/node_modules/@s": [{ name: "w", isDirectory: false, isSymlink: true }], + }; + const fs = { + realPath(path: string) { + if (path.endsWith("/dead")) return Promise.reject(new Error("dangling")); + return Promise.resolve(links[path] ?? path); + }, + async *readDir(path: string) { + const listed = entries[path]; + if (!listed) throw new Error("no such directory"); + yield* listed; + }, + }; + + it("counts node_modules under both roots and in-project workspace links", async () => { + const isDependency = await createDependencyFilePredicate("/app", fs); + assert(isDependency("/app/node_modules/x/index.js")); + assert(isDependency("/real/app/node_modules/x/index.js")); + assert(isDependency("/real/app/packages/linked/index.js")); + assert(isDependency("/real/app/packages/w/index.js")); + }); + + it("never counts project code, the project root or targets outside the project", async () => { + const isDependency = await createDependencyFilePredicate("/app", fs); + assert(!isDependency("/real/app/pages/api/route.ts")); + assert(!isDependency("/real/app/package.json")); + assert(!isDependency("/elsewhere/out/index.js")); + assert(!isDependency("")); + }); + + it("uses the given root when the project has no host directory", async () => { + const isDependency = await createDependencyFilePredicate("/virtual", { + realPath: () => Promise.reject(new Error("no host directory")), + readDir: fs.readDir, + }); + assert(isDependency("/virtual/node_modules/x/index.js")); + assert(!isDependency("/virtual/pages/index.ts")); + }); + }); + + describe("createCompiledBuiltinPlugin()", () => { + it("keeps built-ins external and records the ones required", async () => { + const required = new Set(); + const [builtins, native] = await resolveHandlers(createCompiledBuiltinPlugin(required)); + assertEquals(builtins!({ path: "fs", kind: "require-call" }), { + path: "fs", + external: true, + }); + assertEquals(builtins!({ path: "path", kind: "import-statement" }), { + path: "node:path", + external: true, + }); + assertEquals(builtins!({ path: "left-pad", kind: "require-call" }), undefined); + assertEquals([...required], ["fs"]); + assertEquals(native!({ path: "./build/addon.node" }), { + errors: [{ text: 'Native addons are not supported in API routes: "./build/addon.node"' }], + }); + }); + }); + + describe("createDependencyResolvePlugin()", () => { + const files = new Set(["/d/node_modules/p/impl.js", "/d/node_modules/p/addon.node"]); + const isFile = (path: string) => Promise.resolve(files.has(path)); + const isDependency = (path: string) => path.startsWith("/d/node_modules/"); + const args = (path: string, importer = "/d/node_modules/p/index.js") => ({ + path, + importer, + resolveDir: importer.slice(0, importer.lastIndexOf("/")), + kind: "require-call", + }); + + it("resolves a dependency's relative import in Node's order", async () => { + const [handler] = await resolveHandlers(createDependencyResolvePlugin(isDependency, isFile)); + assertEquals(await handler!(args("./impl")), { path: "/d/node_modules/p/impl.js" }); + assertEquals(await handler!(args("./missing")), undefined); + assertEquals(await handler!(args("./impl", "/d/pages/api/route.ts")), undefined); + }); + + it("rejects a native addon found by Node's extension order", async () => { + const [handler] = await resolveHandlers(createDependencyResolvePlugin(isDependency, isFile)); + assertEquals(await handler!(args("./addon")), { + errors: [{ text: 'Native addons are not supported in API routes: "./addon"' }], + }); + }); + }); + + describe("prepareDependencyFile() without parsing", () => { + const input = (contents: string, logicalPath: string, packageType?: string) => ({ + build: {} as PluginBuild, + contents, + logicalPath, + resolvedPath: logicalPath, + packageType, + isDependencyFile: () => true, + isFile: () => Promise.resolve(false), + rewriteImportMeta: (source: string, url: string) => Promise.resolve(`${url}|${source}`), + }); + + it("returns a file that reads no CommonJS names and no import.meta unchanged", async () => { + const contents = 'module.exports = require("./other");'; + assertEquals(await prepareDependencyFile(input(contents, "/d/a.js")), contents); + }); + + it("points import.meta at the file's own URL in an ESM file", async () => { + assertEquals( + await prepareDependencyFile(input("export const u = import.meta.url;", "/d/a.mjs")), + "file:///d/a.mjs|export const u = import.meta.url;", + ); + assertEquals( + await prepareDependencyFile( + input("const d = __dirname; import.meta.url;", "/d/b.js", "module"), + ), + "file:///d/b.js|const d = __dirname; import.meta.url;", + ); + }); + + it("leaves files that are neither JavaScript nor CommonJS alone", async () => { + const contents = "export const d = __dirname;"; + assertEquals(await prepareDependencyFile(input(contents, "/d/a.ts")), contents); + }); + }); + describe("prependRuntimeRequire()", () => { + it("binds required built-ins and fails closed for anything else", async () => { + const code = prependRuntimeRequire( + "export const load = (id) => require(id);\nexport const resolve = (id) => require.resolve(id);\n", + ["node:path"], + "/app", + ); + const mod = await import(`data:text/javascript,${encodeURIComponent(code)}`); + assertEquals(typeof mod.load("node:path").join, "function"); + assertEquals(mod.resolve("fs"), "fs"); + assertEquals(mod.resolve("node:test"), "node:test"); + for (const id of ["left-pad", "./relative.json", 42]) { + let code: unknown; + try { + mod.load(id); + } catch (error) { + code = (error as { code?: unknown }).code; + } + assertEquals(code, "MODULE_NOT_FOUND", String(id)); + } + let unresolved: unknown; + try { + mod.resolve("test"); + } catch (error) { + unresolved = (error as { code?: unknown }).code; + } + assertEquals(unresolved, "MODULE_NOT_FOUND"); + }); + it("keeps a hashbang on the first line", () => { const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", ["path"], "/app"); assert(code.startsWith("#!/usr/bin/env node\nimport * as "), code); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index d63c3378c3..acd9445541 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -231,14 +231,59 @@ export interface DependencyRootFileSystem { ): AsyncIterable<{ name: string; isDirectory: boolean; isSymlink?: boolean }>; } -async function directoryNames(fs: DependencyRootFileSystem, path: string) { - const names: { name: string; isDirectory: boolean; isSymlink?: boolean }[] = []; +async function directoryEntries(fs: DependencyRootFileSystem, path: string) { + const entries: { name: string; isDirectory: boolean; isSymlink?: boolean }[] = []; try { - for await (const entry of fs.readDir(path)) names.push(entry); + for await (const entry of fs.readDir(path)) entries.push(entry); } catch { // expected: no such directory, as for an adapter-backed project } - return names; + return entries; +} + +/** + * Paths in `node_modules` that may be links: package entries, including the + * packages inside `@scope` directories. Only a link can point outside + * `node_modules`; a host that does not say whether an entry is one gets every + * entry checked. + */ +async function possibleLinks(fs: DependencyRootFileSystem, modules: string): Promise { + const links: string[] = []; + for (const entry of await directoryEntries(fs, modules)) { + const path = pathHelper.join(modules, entry.name); + const scoped = entry.name.startsWith("@") + ? (await directoryEntries(fs, path)).map((inner) => ({ + ...inner, + path: pathHelper.join(path, inner.name), + })) + : [{ ...entry, path }]; + for (const candidate of scoped) { + if (candidate.isSymlink !== false) links.push(candidate.path); + } + } + return links; +} + +/** + * Directories inside the project that `node_modules` links to, as npm, Yarn + * and pnpm workspaces do. The project root and node_modules never count. + */ +async function linkedWorkspaceDirectories( + fs: DependencyRootFileSystem, + root: string, +): Promise { + const modules = pathHelper.join(root, "node_modules"); + const targets: string[] = []; + for (const link of await possibleLinks(fs, modules)) { + const target = await fs.realPath(link).catch(() => null); // null: a dangling link + if ( + target !== null && target !== link && target !== root && + isWithinDirectory(root, target) && !isWithinDirectory(modules, target) + ) { + targets.push(target); + } + } + return targets; } /** @@ -252,56 +297,20 @@ export async function createDependencyFilePredicate( fs: DependencyRootFileSystem, ): Promise<(path: string) => boolean> { const given = pathHelper.resolve(projectDir); - let canonical: string | null = null; - try { - canonical = await fs.realPath(given); - } catch { - // expected: an adapter-backed project may have no host directory - } - const prefixes = new Set( - [given, canonical ?? given].map((root) => - pathHelper.join(root, "node_modules") + pathHelper.sep - ), - ); - - if (canonical !== null) { - const modules = pathHelper.join(canonical, "node_modules"); - const links: string[] = []; - for (const entry of await directoryNames(fs, modules)) { - const path = pathHelper.join(modules, entry.name); - // Only a link can point outside node_modules; a host that does not say - // whether an entry is one gets every entry checked. - if (!entry.name.startsWith("@")) { - if (entry.isSymlink !== false) links.push(path); - continue; - } - for (const scoped of await directoryNames(fs, path)) { - if (scoped.isSymlink !== false) links.push(pathHelper.join(path, scoped.name)); - } - } - for (const link of links) { - let target: string; - try { - target = await fs.realPath(link); - } catch { - continue; // expected: a dangling link - } - // A package linked to a directory of the project itself, as npm, Yarn - // and pnpm workspaces do. The project root and node_modules never count. - if ( - target !== link && target !== canonical && isWithinDirectory(canonical, target) && - !isWithinDirectory(modules, target) - ) { - prefixes.add(target + pathHelper.sep); - } - } - } - - const roots = [...prefixes]; + // null: an adapter-backed project may have no host directory + const canonical = await fs.realPath(given).catch(() => null); + const directories = [ + pathHelper.join(given, "node_modules"), + ...(canonical === null ? [] : [ + pathHelper.join(canonical, "node_modules"), + ...await linkedWorkspaceDirectories(fs, canonical), + ]), + ]; + const prefixes = [...new Set(directories)].map((directory) => directory + pathHelper.sep); return (path) => { if (path === "") return false; const resolved = pathHelper.resolve(path); - return roots.some((prefix) => resolved.startsWith(prefix)); + return prefixes.some((prefix) => resolved.startsWith(prefix)); }; } diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index ea0c14aede..7e0a57547e 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -272,16 +272,19 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { it("bundles a workspace package that node_modules links to", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { - "packages/ws/package.json": JSON.stringify({ name: "ws", main: "index.js" }), - "packages/ws/index.js": + "packages/linked/package.json": JSON.stringify({ name: "linked", main: "index.js" }), + "packages/linked/index.js": 'exports.dir = __dirname;\nexports.self = require.resolve("./index.js");', }); await mkdir(`${projectDir}/node_modules`, { recursive: true }); - await symlink("../packages/ws", `${projectDir}/node_modules/ws`); - const mod = await bundleAndImport(projectDir, 'import ws from "ws";\nexport { ws };\n'); - assertEquals(mod.ws, { - dir: `${projectDir}/packages/ws`, - self: `${projectDir}/packages/ws/index.js`, + await symlink("../packages/linked", `${projectDir}/node_modules/linked`); + const mod = await bundleAndImport( + projectDir, + 'import linked from "linked";\nexport { linked };\n', + ); + assertEquals(mod.linked, { + dir: `${projectDir}/packages/linked`, + self: `${projectDir}/packages/linked/index.js`, }); }); }); From a7ad9437c774cce563ee3cc46dbfcfa45de573df Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 22:00:29 +0100 Subject: [PATCH 11/29] fix(api): serve computed built-ins and bundle dual-package require edges in compiled routes - Runtime require uses process.getBuiltinModule, so a computed built-in id works; the static built-in table is gone. - Extensionless CommonJS entries get __dirname, __filename and literal require.resolve like .js files. - Dependency ESM files keep their own import.meta: constants for url, filename and dirname, literal resolve at build time, a bound resolver otherwise. The build no longer fails on a computed import.meta.resolve. - Native addons fail on load, so a package named *.node bundles. - Packages that load from disk as ESM stay external only for imports; a require() of one bundles its CommonJS entry. --- .../compiled-cjs-bundling.test.ts | 245 ++++++++--- .../module-loader/compiled-cjs-bundling.ts | 416 +++++++++++++----- src/routing/api/module-loader/loader.ts | 16 +- .../compiled-cjs-bundling.test.ts | 144 +++++- 4 files changed, 615 insertions(+), 206 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 0d01465ef1..5467116b1b 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -7,27 +7,44 @@ import { createCompiledBuiltinPlugin, createDependencyFilePredicate, createDependencyResolvePlugin, + createImportExternalsPlugin, + createImportMetaResolve, + createRuntimeRequire, nodePreferredFile, nodeRelativeFile, PackageTypeLookup, prepareDependencyFile, prependRuntimeRequire, + resolveLiteralCalls, + type RuntimeRequireHost, } from "./compiled-cjs-bundling.ts"; type ResolveHandler = (args: Record) => unknown; +type Plugin = { setup(build: PluginBuild): void | Promise }; -/** The onResolve handlers a plugin registers, in order. */ -async function resolveHandlers( - plugin: { setup(build: PluginBuild): void | Promise }, -): Promise { - const handlers: ResolveHandler[] = []; +/** The onResolve or onLoad handlers a plugin registers, in order. */ +async function handlers(plugin: Plugin, hook: "onResolve" | "onLoad"): Promise { + const registered: ResolveHandler[] = []; + const register = (_options: unknown, handler: ResolveHandler) => registered.push(handler); const build = { - onResolve: (_options: unknown, handler: ResolveHandler) => handlers.push(handler), - onLoad: () => {}, + onResolve: hook === "onResolve" ? register : () => {}, + onLoad: hook === "onLoad" ? register : () => {}, onDispose: () => {}, }; await plugin.setup(build as unknown as PluginBuild); - return handlers; + return registered; +} + +const resolveHandlers = (plugin: Plugin) => handlers(plugin, "onResolve"); + +/** Throws the code of the error `fn` throws, or undefined. */ +function errorCode(fn: () => unknown): unknown { + try { + fn(); + } catch (error) { + return (error as { code?: unknown }).code; + } + return undefined; } describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { @@ -175,9 +192,8 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); describe("createCompiledBuiltinPlugin()", () => { - it("keeps built-ins external and records the ones required", async () => { - const required = new Set(); - const [builtins, native] = await resolveHandlers(createCompiledBuiltinPlugin(required)); + it("keeps built-ins external, with a required id unchanged", async () => { + const [builtins] = await resolveHandlers(createCompiledBuiltinPlugin()); assertEquals(builtins!({ path: "fs", kind: "require-call" }), { path: "fs", external: true, @@ -187,9 +203,13 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { external: true, }); assertEquals(builtins!({ path: "left-pad", kind: "require-call" }), undefined); - assertEquals([...required], ["fs"]); - assertEquals(native!({ path: "./build/addon.node" }), { - errors: [{ text: 'Native addons are not supported in API routes: "./build/addon.node"' }], + }); + + it("rejects a native addon once it is loaded, naming only the file", async () => { + assertEquals(await resolveHandlers(createCompiledBuiltinPlugin()).then((h) => h.length), 1); + const [native] = await handlers(createCompiledBuiltinPlugin(), "onLoad"); + assertEquals(native!({ path: "/srv/app/node_modules/x/build/addon.node" }), { + errors: [{ text: 'Native addons are not supported in API routes: "addon.node"' }], }); }); }); @@ -212,11 +232,28 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(await handler!(args("./impl", "/d/pages/api/route.ts")), undefined); }); - it("rejects a native addon found by Node's extension order", async () => { + it("leaves a native addon found by Node's extension order to the loader", async () => { const [handler] = await resolveHandlers(createDependencyResolvePlugin(isDependency, isFile)); - assertEquals(await handler!(args("./addon")), { - errors: [{ text: 'Native addons are not supported in API routes: "./addon"' }], + assertEquals(await handler!(args("./addon")), { path: "/d/node_modules/p/addon.node" }); + }); + }); + + describe("createImportExternalsPlugin()", () => { + it("keeps imports of the packages external and leaves require() to the bundler", async () => { + const [handler] = await resolveHandlers(createImportExternalsPlugin(["dual", "@s/p.js"])); + for (const kind of ["import-statement", "dynamic-import"]) { + assertEquals(handler!({ path: "dual/sub", kind }), { path: "dual/sub", external: true }); + } + assertEquals(handler!({ path: "@s/p.js", kind: "import-statement" }), { + path: "@s/p.js", + external: true, }); + assertEquals(handler!({ path: "dual", kind: "require-call" }), undefined); + assertEquals(handler!({ path: "dual", kind: "require-resolve" }), undefined); + }); + + it("registers nothing without packages", async () => { + assertEquals(await resolveHandlers(createImportExternalsPlugin([])), []); }); }); @@ -229,7 +266,6 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { packageType, isDependencyFile: () => true, isFile: () => Promise.resolve(false), - rewriteImportMeta: (source: string, url: string) => Promise.resolve(`${url}|${source}`), }); it("returns a file that reads no CommonJS names and no import.meta unchanged", async () => { @@ -237,68 +273,147 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(await prepareDependencyFile(input(contents, "/d/a.js")), contents); }); - it("points import.meta at the file's own URL in an ESM file", async () => { - assertEquals( - await prepareDependencyFile(input("export const u = import.meta.url;", "/d/a.mjs")), - "file:///d/a.mjs|export const u = import.meta.url;", + it("leaves files that are neither JavaScript nor CommonJS alone", async () => { + const contents = "export const d = __dirname;"; + assertEquals(await prepareDependencyFile(input(contents, "/d/a.ts")), contents); + }); + }); + + describe("resolveLiteralCalls()", () => { + it("replaces resolved literal calls once per specifier and keeps the rest", async () => { + const asked: string[] = []; + const code = await resolveLiteralCalls( + `M("./a"); M('./a'); M("missing"); M(name); const r = M; M("x\\"y");`, + "M", + (specifier) => { + asked.push(specifier); + return Promise.resolve(specifier === "./a" ? "/d/a.js" : null); + }, ); assertEquals( - await prepareDependencyFile( - input("const d = __dirname; import.meta.url;", "/d/b.js", "module"), - ), - "file:///d/b.js|const d = __dirname; import.meta.url;", + code, + `"/d/a.js"; "/d/a.js"; M("missing"); M(name); const r = M; M("x\\"y");`, ); + assertEquals(asked, ["./a", "missing"]); }); + }); - it("leaves files that are neither JavaScript nor CommonJS alone", async () => { - const contents = "export const d = __dirname;"; - assertEquals(await prepareDependencyFile(input(contents, "/d/a.ts")), contents); + describe("createImportMetaResolve()", () => { + const resolve = createImportMetaResolve( + "file:///app/node_modules/p/index.js", + ["test"], + (id) => (id === "fs" || id === "test" ? {} : undefined), + ); + + it("resolves URLs and relative paths against the module", () => { + assertEquals(resolve("./data.json"), "file:///app/node_modules/p/data.json"); + assertEquals(resolve("../q"), "file:///app/node_modules/q"); + assertEquals(resolve("/etc/x"), "file:///etc/x"); + assertEquals(resolve("node:path"), "node:path"); + assertEquals(resolve("https://example.com/a"), "https://example.com/a"); + }); + + it("gives built-ins their node: id and rejects package names", () => { + assertEquals(resolve("fs"), "node:fs"); + assertEquals(errorCode(() => resolve("test")), "ERR_MODULE_NOT_FOUND"); + assertEquals(errorCode(() => resolve("left-pad")), "ERR_MODULE_NOT_FOUND"); }); }); - describe("prependRuntimeRequire()", () => { - it("binds required built-ins and fails closed for anything else", async () => { - const code = prependRuntimeRequire( - "export const load = (id) => require(id);\nexport const resolve = (id) => require.resolve(id);\n", - ["node:path"], - "/app", - ); - const mod = await import(`data:text/javascript,${encodeURIComponent(code)}`); - assertEquals(typeof mod.load("node:path").join, "function"); - assertEquals(mod.resolve("fs"), "fs"); - assertEquals(mod.resolve("node:test"), "node:test"); - for (const id of ["left-pad", "./relative.json", 42]) { - let code: unknown; - try { - mod.load(id); - } catch (error) { - code = (error as { code?: unknown }).code; - } - assertEquals(code, "MODULE_NOT_FOUND", String(id)); - } - let unresolved: unknown; - try { - mod.resolve("test"); - } catch (error) { - unresolved = (error as { code?: unknown }).code; + describe("createRuntimeRequire()", () => { + const builtins: Record = { fs: { kind: "fs" }, "node:fs": { kind: "fs" } }; + const files: Record = { + "/app/data.json": '{"a":1}', + "/app/link.json": "", // a link to /outside.json + "/app/code.js": "", + }; + let reads = 0; + const host: RuntimeRequireHost = { + // Like Deno's, it knows bare prefix-only names that Node does not. + getBuiltinModule: (id) => id === "test" ? { kind: "test" } : builtins[id], + realpathSync: (path) => { + if (path === "/app/link.json") return "/outside.json"; + if (!(path in files)) throw new Error("ENOENT"); + return path; + }, + readFileSync: (path) => { + reads++; + return files[path]!; + }, + isAbsolute: (path) => path.startsWith("/"), + relative: (from, to) => + to.startsWith(from + "/") ? to.slice(from.length + 1) : to === from ? "" : "../" + to, + sep: "/", + }; + const require = createRuntimeRequire(host, "/app", ["test"]); + + it("serves any built-in, including a computed id", () => { + assertEquals(require("fs"), { kind: "fs" }); + assertEquals(require("node:" + "fs"), { kind: "fs" }); + assertEquals(require.resolve("fs"), "fs"); + }); + + it("treats a bare prefix-only name as a package", () => { + assertEquals(errorCode(() => require("test")), "MODULE_NOT_FOUND"); + assertEquals(errorCode(() => require.resolve("test")), "MODULE_NOT_FOUND"); + }); + + it("loads project JSON once and resolves existing project paths", () => { + const first = require("/app/data.json"); + assertEquals(first, { a: 1 }); + assert(require("/app/data.json") === first); + assertEquals(reads, 1); + assertEquals(require.resolve("/app/code.js"), "/app/code.js"); + }); + + it("fails closed for packages, other files and paths outside the project", () => { + for (const id of ["left-pad", "./data.json", 42, "/app/code.js", "/app/link.json", "/app"]) { + assertEquals(errorCode(() => require(id)), "MODULE_NOT_FOUND", String(id)); } - assertEquals(unresolved, "MODULE_NOT_FOUND"); + assertEquals(errorCode(() => require.resolve("/app/missing.js")), "MODULE_NOT_FOUND"); }); - it("keeps a hashbang on the first line", () => { - const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", ["path"], "/app"); - assert(code.startsWith("#!/usr/bin/env node\nimport * as "), code); + it("names a project path relative to the project and hides any other", () => { + assertEquals( + (() => { + try { + require("/app/code.js"); + } catch (error) { + return (error as Error).message; + } + })(), + 'Cannot find module "code.js"', + ); + assertEquals( + (() => { + try { + require("/app/link.json"); + } catch (error) { + return (error as Error).message; + } + })(), + 'Cannot find module "a path outside this project or missing"', + ); }); + }); - it("picks helper names the bundle does not already use", () => { - const code = prependRuntimeRequire("const __vf_builtin0 = 1;\n", ["path"], "/app"); - assert(code.includes("import * as __vf_builtin_0 from"), code); + describe("prependRuntimeRequire()", () => { + it("binds require to the runtime require with the host built-ins", () => { + const code = prependRuntimeRequire("export {};\n", "/app"); + assert(code.includes(createRuntimeRequire.toString()), code); + assert(code.includes('import __vf_runtime_process from "node:process";'), code); + assert(code.includes('}, "/app", ["test","test/reporters","sqlite","sea"]);'), code); + assert(code.endsWith("\nexport {};\n"), code); + }); + + it("keeps a hashbang on the first line", () => { + const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", "/app"); + assert(code.startsWith("#!/usr/bin/env node\nimport __vf_runtime_process from "), code); }); - it("resolves a prefix-only built-in only with its prefix", () => { - const code = prependRuntimeRequire("export {};\n", [], "/app"); - assert(code.includes('"node:test": 1'), code); - assert(!code.includes('"test": 1'), code); + it("picks helper names the bundle does not already use", () => { + const code = prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app"); + assert(code.includes("import * as __vf_runtime___fs from"), code); }); }); }); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index acd9445541..be02ee6118 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -50,33 +50,34 @@ export function builtinModuleName(specifier: string): string | undefined { const NATIVE_ADDON = /\.node$/i; +// Names only the file, never the host path. function nativeAddonError(path: string) { return { errors: [{ - text: `Native addons are not supported in API routes: ${JSON.stringify(path)}`, + text: `Native addons are not supported in API routes: ${ + JSON.stringify(pathHelper.basename(path)) + }`, }], }; } /** - * Keeps built-ins external and records each one a bundled module `require`s, - * so {@link prependRuntimeRequire} can bind it. Must run before any plugin - * that resolves built-ins itself. + * Keeps built-ins external. A `require()` of one keeps its id, for the runtime + * `require` of {@link prependRuntimeRequire}. A native addon fails the build + * once it resolves to a file: a package whose name ends in `.node` is fine. + * Must run before any plugin that resolves built-ins or loads files itself. */ -export function createCompiledBuiltinPlugin(requiredBuiltins: Set): Plugin { +export function createCompiledBuiltinPlugin(): Plugin { return { name: "vf-compiled-builtins", setup(build) { build.onResolve({ filter: /.*/ }, (args) => { const name = builtinModuleName(args.path); if (name === undefined) return undefined; - if (args.kind === "require-call") { - requiredBuiltins.add(args.path); - return { path: args.path, external: true }; - } + if (args.kind === "require-call") return { path: args.path, external: true }; return { path: `node:${name}`, external: true }; }); - build.onResolve({ filter: NATIVE_ADDON }, (args) => nativeAddonError(args.path)); + build.onLoad({ filter: NATIVE_ADDON }, (args) => nativeAddonError(args.path)); }, }; } @@ -171,14 +172,37 @@ export function createDependencyResolvePlugin( build.onResolve({ filter: RELATIVE_SPECIFIER }, async (args) => { if (!args.importer || !isDependencyFile(args.importer)) return undefined; const file = await nodeRelativeFile(args.path, args.resolveDir, isFile); - if (file === null) return undefined; - if (NATIVE_ADDON.test(file)) return nativeAddonError(args.path); - return { path: file }; + return file === null ? undefined : { path: file }; }); }, }; } +/** + * Keeps the given packages external for `import` and `import()` only. A + * package that loads from disk as ESM may also export CommonJS to `require`; + * a `require()` of it from a bundled dependency bundles that CommonJS entry, + * since the runtime `require` cannot load packages. Must run where esbuild's + * own resolution would apply: after the project's import map and aliases. + */ +export function createImportExternalsPlugin(names: Iterable): Plugin { + const escaped = [...names].map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); + const filter = new RegExp(`^(?:${escaped.join("|")})(?:/.*)?$`); + return { + name: "vf-import-externals", + setup(build) { + if (escaped.length === 0) return; + build.onResolve( + { filter }, + (args) => + args.kind === "import-statement" || args.kind === "dynamic-import" + ? { path: args.path, external: true } + : undefined, + ); + }, + }; +} + /** Reads a package.json as text, or null when the directory has none. */ export type ReadManifest = (path: string) => Promise; @@ -320,8 +344,8 @@ export type ResolveRequireLiteral = (specifier: string) => Promise { const cjsExtension = /\.cjs$/i.test(filePath); const jsExtension = /\.js$/i.test(filePath); - if (!cjsExtension && !(jsExtension && packageType !== "module")) return null; + // Node's require() loads a file without an extension as CommonJS, whatever + // its package declares; a package `main` may name one. + const extensionless = pathHelper.extname(filePath) === ""; + if (!cjsExtension && !extensionless && !(jsExtension && packageType !== "module")) return null; // A fresh name per file: the source cannot contain it, so every occurrence // in the output is one the parser put there. It never reaches the bundle. @@ -359,29 +386,140 @@ export async function prepareCommonJSFile( }, }); const format = Object.values(result.metafile?.inputs ?? {})[0]?.format; - if (jsExtension && format === "esm") return null; + if (!cjsExtension && format === "esm") return null; const code = result.outputFiles?.[0]?.text; if (code === undefined) throw new Error(`No output for dependency file ${filePath}`); + const rewritten = await resolveLiteralCalls( + code, + marker, + async (specifier) => + builtinModuleName(specifier) === undefined ? await resolve(specifier) : specifier, + ); + // A call left unresolved throws at runtime, as in Node. + return rewritten.replaceAll(marker, "require.resolve"); +} + +/** + * Replaces each call of `marker` with one string literal argument by the + * string `resolve` returns for it. A call it returns null for, and any other + * use of `marker`, is left unchanged. The literal must be as the bundler + * prints it: quoted, with no escapes. + */ +export async function resolveLiteralCalls( + code: string, + marker: string, + resolve: (specifier: string) => Promise, +): Promise { const literalCall = new RegExp(`${marker}\\((["'])([^"'\\\\\\n]*)\\1\\)`, "g"); const resolved = new Map(); for (const match of code.matchAll(literalCall)) { const specifier = match[2]!; - if (!resolved.has(specifier)) { - resolved.set( - specifier, - builtinModuleName(specifier) === undefined ? await resolve(specifier) : specifier, - ); - } + if (!resolved.has(specifier)) resolved.set(specifier, await resolve(specifier)); } - return code - .replace(literalCall, (_whole, _quote, specifier: string) => { - const target = resolved.get(specifier); - return typeof target === "string" - ? JSON.stringify(target) - : `require.resolve(${JSON.stringify(specifier)})`; - }) - .replaceAll(marker, "require.resolve"); + return code.replace(literalCall, (whole, _quote, specifier: string) => { + const target = resolved.get(specifier); + return typeof target === "string" ? JSON.stringify(target) : whole; + }); +} + +/** Resolves an `import.meta.resolve` literal to a URL, or null to leave it for runtime. */ +export type ResolveImportMetaLiteral = (specifier: string) => Promise; + +const LOADERS: Record = { + ".ts": "ts", + ".mts": "ts", + ".cts": "ts", + ".tsx": "tsx", + ".jsx": "jsx", +}; + +/** + * An ES module dependency file with its `import.meta` locations pointing at + * the file itself, not at the route bundle it ends up in. Unbound reads of + * `import.meta.url`, `import.meta.filename` and `import.meta.dirname` become + * constants. Each `import.meta.resolve("literal")` becomes the URL `resolve` + * finds. Any other use of `import.meta.resolve`, computed or detached, calls a + * resolver bound to the file: it resolves URLs and relative paths against the + * file and built-ins to their `node:` ids, and throws `ERR_MODULE_NOT_FOUND` + * for a package name, which the bundle cannot locate at runtime. Any other + * read of `import.meta` gets an object with these four properties. + */ +export async function prepareModuleFile( + contents: string, + filePath: string, + resolve: ResolveImportMetaLiteral, +): Promise { + const url = pathHelper.toFileUrl(filePath).href; + const marker = `__vf_import_meta_resolve_${crypto.randomUUID().replaceAll("-", "")}`; + const result = await bundle({ + stdin: { + contents, + loader: LOADERS[pathHelper.extname(filePath).toLowerCase()] ?? "js", + sourcefile: filePath, + }, + bundle: false, + write: false, + platform: "neutral", + logLevel: "silent", + define: { + // The most specific name wins: `import.meta` alone is any other use. + "import.meta": `${marker}_meta`, + "import.meta.resolve": marker, + "import.meta.url": JSON.stringify(url), + "import.meta.filename": JSON.stringify(filePath), + "import.meta.dirname": JSON.stringify(pathHelper.dirname(filePath)), + }, + }); + const code = result.outputFiles?.[0]?.text; + if (code === undefined) throw new Error(`No output for dependency file ${filePath}`); + if (!code.includes(marker)) return code; + + const rewritten = await resolveLiteralCalls(code, marker, resolve); + if (!rewritten.includes(marker)) return rewritten; + const prologue = [ + // The resolver is a declaration, so it is hoisted: a cyclic import may + // call into the file before its body runs. + `function ${marker}(specifier) {`, + ` return (${createImportMetaResolve.toString()})(${JSON.stringify(url)}, ${ + JSON.stringify([...PREFIX_ONLY_BUILTINS]) + }, (id) => globalThis.process?.getBuiltinModule?.(id))(specifier);`, + "}", + `const ${marker}_meta = { url: ${JSON.stringify(url)}, filename: ${ + JSON.stringify(filePath) + }, dirname: ${JSON.stringify(pathHelper.dirname(filePath))}, resolve: ${marker} };`, + ].join("\n"); + // A hashbang is only valid as the first line. + const hashbangEnd = rewritten.startsWith("#!") ? rewritten.indexOf("\n") + 1 : 0; + return rewritten.slice(0, hashbangEnd) + prologue + "\n" + rewritten.slice(hashbangEnd); +} + +/** + * The runtime `import.meta.resolve` of a bundled dependency module at + * `moduleUrl`: URLs and relative paths resolve against the module, built-ins + * to their `node:` ids, and a package name throws `ERR_MODULE_NOT_FOUND`, + * since the bundle cannot locate packages at runtime. + * + * {@link prepareModuleFile} embeds this function's source in the module, so + * it must not refer to anything outside its own body. + */ +export function createImportMetaResolve( + moduleUrl: string, + prefixOnlyNames: readonly string[], + getBuiltinModule: (id: string) => unknown, +): (specifier: unknown) => string { + return (specifier) => { + const id = String(specifier); + if (/^[a-z][a-z0-9+.-]*:/i.test(id)) return new URL(id).href; + if (/^(?:\.\.?(?:\/|$)|\/)/.test(id)) return new URL(id, moduleUrl).href; + // A bare prefix-only name such as `test` is a package, as in Node. + if (!prefixOnlyNames.includes(id) && getBuiltinModule(id) !== undefined) return "node:" + id; + const error = new Error( + "Cannot find package " + JSON.stringify(id) + " imported from a bundled dependency", + ) as Error & { code?: string }; + error.code = "ERR_MODULE_NOT_FOUND"; + throw error; + }; } /** Input for {@link prepareDependencyFile}. */ @@ -396,15 +534,13 @@ export interface DependencyFileInput { readonly packageType: string | undefined; readonly isDependencyFile: (path: string) => boolean; readonly isFile: IsFile; - /** Points a module's `import.meta` locations at its own URL. */ - readonly rewriteImportMeta: (contents: string, moduleUrl: string) => Promise; } /** * A bundled dependency file as Node would run it: see - * {@link prepareCommonJSFile}. In a CommonJS file a `require.resolve` literal - * resolves only to a file inside the project's dependencies. An ESM file keeps - * its own semantics; only its `import.meta` locations point at its own URL. + * {@link prepareCommonJSFile} and {@link prepareModuleFile}. A literal + * location resolves at build time only to a file inside the project's + * dependencies. */ export async function prepareDependencyFile(input: DependencyFileInput): Promise { const { build, contents, logicalPath, resolvedPath, isDependencyFile } = input; @@ -416,108 +552,148 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise (contents.includes("require") && contents.includes("resolve")); if (!readsCommonJSNames && !importMeta) return contents; const resolveDir = pathHelper.dirname(resolvedPath); + const dependencyPath = async ( + specifier: string, + kind: "require-resolve" | "import-statement", + ) => { + if (!build.resolve) return null; + const result = await build.resolve(specifier, { kind, importer: resolvedPath, resolveDir }); + if (result.errors?.length || !result.path || result.external) return null; + if (!isDependencyFile(result.path)) return null; + return await nodePreferredFile(result.path, specifier, resolveDir, input.isFile); + }; const commonJS = await prepareCommonJSFile( contents, logicalPath, input.packageType, - async (specifier) => { - if (!build.resolve) return null; - const result = await build.resolve(specifier, { - kind: "require-resolve", - importer: resolvedPath, - resolveDir, - }); - // Only a file inside the project's dependencies is a path to hand back. - if (result.errors?.length || !result.path || result.external) return null; - if (!isDependencyFile(result.path)) return null; - return await nodePreferredFile(result.path, specifier, resolveDir, input.isFile); - }, + (specifier) => dependencyPath(specifier, "require-resolve"), ); if (commonJS !== null) return commonJS; - return importMeta - ? await input.rewriteImportMeta(contents, pathHelper.toFileUrl(logicalPath).href) - : contents; + if (!importMeta) return contents; + return await prepareModuleFile(contents, logicalPath, async (specifier) => { + const builtin = builtinModuleName(specifier); + if (builtin !== undefined) return "node:" + builtin; + if (/^[a-z][a-z0-9+.-]*:/i.test(specifier) || RELATIVE_SPECIFIER.test(specifier)) { + return new URL(specifier, pathHelper.toFileUrl(logicalPath)).href; + } + const path = await dependencyPath(specifier, "import-statement"); + return path === null ? null : pathHelper.toFileUrl(path).href; + }); +} + +/** Host operations the runtime `require` uses; see {@link createRuntimeRequire}. */ +export interface RuntimeRequireHost { + getBuiltinModule(id: string): unknown; + realpathSync(path: string): string; + readFileSync(path: string, encoding: "utf8"): string; + isAbsolute(path: string): boolean; + relative(from: string, to: string): string; + readonly sep: string; +} + +/** The runtime `require` of a bundled route; see {@link createRuntimeRequire}. */ +export interface RuntimeRequire { + (id: unknown): unknown; + resolve(id: unknown): string; +} + +/** + * The `require` that esbuild's output calls for anything it did not bundle: a + * built-in, under any id Node accepts, including a computed one; and, for a + * computed id, an existing file inside the project, which `require.resolve` + * returns and `require` loads when it is JSON, cached as Node's module cache + * keeps it. Anything else throws `MODULE_NOT_FOUND`, so optional dependencies + * probed in a `try` still fall back. Errors name a path relative to the + * project, never the host path. + * + * {@link prependRuntimeRequire} embeds this function's source in the route + * module, so it must not refer to anything outside its own body. + */ +export function createRuntimeRequire( + host: RuntimeRequireHost, + projectRoot: string, + prefixOnlyNames: readonly string[], +): RuntimeRequire { + // Tables without a prototype: `in` reads only their own keys and does not + // depend on built-ins that route code could replace later. + const prefixOnly: Record = Object.create(null); + for (const name of prefixOnlyNames) prefixOnly[name] = true; + const json: Record = Object.create(null); + const builtin = (id: unknown) => { + // A bare prefix-only name such as `test` is an npm package, as in Node. + if (typeof id !== "string" || id in prefixOnly) return undefined; + return host.getBuiltinModule(id); + }; + const missing = (label: string) => { + const error = new Error("Cannot find module " + JSON.stringify(label)) as Error & { + code?: string; + }; + error.code = "MODULE_NOT_FOUND"; + return error; + }; + // A path is usable only when it exists and its real path is in the project. + const file = (id: unknown) => { + if (typeof id !== "string" || !host.isAbsolute(id)) return null; + let real: string; + try { + real = host.realpathSync(id); + } catch { + return null; + } + const relative = host.relative(projectRoot, real); + const up = relative === ".." || relative.startsWith(".." + host.sep); + if (relative === "" || up || host.isAbsolute(relative)) return null; + return { real, relative }; + }; + const label = (id: unknown) => + typeof id === "string" && host.isAbsolute(id) + ? "a path outside this project or missing" + : String(id); + const require = ((id: unknown) => { + const module = builtin(id); + if (module !== undefined) return module; + const found = file(id); + if (found !== null && found.real.endsWith(".json")) { + if (!(found.real in json)) { + json[found.real] = JSON.parse(host.readFileSync(found.real, "utf8")); + } + return json[found.real]; + } + throw missing(found !== null ? found.relative : label(id)); + }) as RuntimeRequire; + require.resolve = (id: unknown) => { + if (builtin(id) !== undefined) return id as string; + if (file(id) !== null) return id as string; + throw missing(label(id)); + }; + return require; } /** * Binds the module-level `require` that esbuild's output calls for anything it - * did not bundle: the built-ins recorded during the build, and, for a computed - * id, an existing file inside the project, which `require.resolve` returns and - * `require` loads when it is JSON. Anything else throws `MODULE_NOT_FOUND`, so - * optional dependencies probed in a `try` still fall back. Errors name a path - * relative to the project, never the host path. esbuild's ESM output never + * did not bundle, to {@link createRuntimeRequire}. esbuild's ESM output never * declares a top-level `require` (it renames a route's own binding of that * name), so the prologue cannot collide with route code. */ -export function prependRuntimeRequire( - code: string, - requiredBuiltins: Iterable, - projectRoot: string, -): string { - // A fixed order keeps the output byte-identical for the same set of ids. - const ids = [...requiredBuiltins].sort((a, b) => a.localeCompare(b, "en")); - let prefix = "__vf_builtin"; +export function prependRuntimeRequire(code: string, projectRoot: string): string { + let prefix = "__vf_runtime"; while (code.includes(prefix)) prefix += "_"; - + const host = [ + `getBuiltinModule: (id) => ${prefix}_process.getBuiltinModule(id)`, + `realpathSync: ${prefix}_fs.realpathSync`, + `readFileSync: ${prefix}_fs.readFileSync`, + `isAbsolute: ${prefix}_path.isAbsolute`, + `relative: ${prefix}_path.relative`, + `sep: ${prefix}_path.sep`, + ]; const lines = [ + `import ${prefix}_process from "node:process";`, `import * as ${prefix}_fs from "node:fs";`, `import * as ${prefix}_path from "node:path";`, + `const require = (${createRuntimeRequire.toString()})({ ${host.join(", ")} }, ${ + JSON.stringify(projectRoot) + }, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])});`, ]; - for (const [index, id] of ids.entries()) { - const specifier = JSON.stringify("node:" + builtinModuleName(id)); - lines.push(`import * as ${prefix}${index} from ${specifier};`); - } - const fields = ["__proto__: null"]; - for (const [index, id] of ids.entries()) { - fields.push(`${JSON.stringify(id)}: ${prefix}${index}.default ?? ${prefix}${index}`); - } - const root = JSON.stringify(projectRoot); - // Every id Node's require.resolve returns unchanged: a built-in Node accepts - // without the prefix, or any built-in with the node: prefix. - const builtinIds = [ - ...BUILTINS, - ...[...BUILTINS, ...PREFIX_ONLY_BUILTINS].map((name) => "node:" + name), - ].map((id) => JSON.stringify(id) + ": 1"); - lines.push( - // The table has no prototype, so `in` reads only its own keys and does - // not depend on built-ins route code could replace. - `const ${prefix}s = { ${fields.join(", ")} };`, - `const ${prefix}_ids = { __proto__: null, ${builtinIds.join(", ")} };`, - // JSON a computed require() loaded, by real path, as Node's module cache keeps it. - `const ${prefix}_json = { __proto__: null };`, - `const ${prefix}_missing = (label) => {`, - ' const error = new Error("Cannot find module " + JSON.stringify(label));', - ' error.code = "MODULE_NOT_FOUND";', - " return error;", - "};", - // A path is usable only when it exists and its real path is in the project. - `const ${prefix}_file = (id) => {`, - ` if (typeof id !== "string" || !${prefix}_path.isAbsolute(id)) return null;`, - " let real;", - ` try { real = ${prefix}_fs.realpathSync(id); } catch { return null; }`, - ` const relative = ${prefix}_path.relative(${root}, real);`, - ` const up = relative === ".." || relative.startsWith(".." + ${prefix}_path.sep);`, - ` if (relative === "" || up || ${prefix}_path.isAbsolute(relative)) return null;`, - " return { real, relative };", - "};", - `const ${prefix}_label = (id) => typeof id === "string" && ${prefix}_path.isAbsolute(id) ? "a path outside this project or missing" : String(id);`, - "const require = (id) => {", - ` if (typeof id === "string" && id in ${prefix}s) return ${prefix}s[id];`, - ` const file = ${prefix}_file(id);`, - ` if (file !== null && file.real.endsWith(".json")) {`, - ` if (!(file.real in ${prefix}_json)) ${prefix}_json[file.real] = JSON.parse(${prefix}_fs.readFileSync(file.real, "utf8"));`, - ` return ${prefix}_json[file.real];`, - " }", - ` throw ${prefix}_missing(file !== null ? file.relative : ${prefix}_label(id));`, - "};", - "require.resolve = (id) => {", - ` if (typeof id === "string" && (id in ${prefix}s || id in ${prefix}_ids)) return id;`, - ` const file = ${prefix}_file(id);`, - " if (file !== null) return id;", - ` throw ${prefix}_missing(${prefix}_label(id));`, - "};", - ); - // A hashbang is only valid as the first line. const hashbangEnd = code.startsWith("#!") ? code.indexOf("\n") + 1 : 0; return code.slice(0, hashbangEnd) + lines.join("\n") + "\n" + code.slice(hashbangEnd); diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 1cced184f6..939fa08a9a 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -53,6 +53,7 @@ import { createCompiledBuiltinPlugin, createDependencyFilePredicate, createDependencyResolvePlugin, + createImportExternalsPlugin, type IsFile, PackageTypeLookup, prepareDependencyFile, @@ -1961,8 +1962,6 @@ function createProjectBoundaryPlugin( packageType: await packageTypes.of(pathHelper.dirname(source.logicalPath)), isDependencyFile, isFile: hostIsFile, - rewriteImportMeta: (contents, moduleUrl) => - rewriteBundledImportMetaUrl(contents, moduleUrl, null), }), loader: getLoaderForFile(source.logicalPath), resolveDir: pathHelper.dirname(source.logicalPath), @@ -2177,14 +2176,12 @@ function buildTranspiledModuleSource( // When esbuild bundles CJS into ESM output, these become __require() shims that // fail at runtime. Inject a createRequire-based shim so require() works in ESM. // Use projectDir as the resolve base so require() finds the project's node_modules. - // The compiled binary binds `require` after the build instead, from the - // built-ins the bundle actually requires. + // The compiled binary binds `require` after the build instead. const safeProjectDir = JSON.stringify(projectDir + "/package.json"); const requireShim = compiled ? "" : [ 'import { createRequire as __vf_createRequire } from "node:module";', `var require = __vf_createRequire(${safeProjectDir});`, ].join("\n"); - const requiredBuiltins = new Set(); const isDependencyFile = compiled ? await createDependencyFilePredicate(projectDir, { realPath, @@ -2233,7 +2230,9 @@ function buildTranspiledModuleSource( "veryfront", "veryfront/*", "@opentelemetry/*", - ...userExternals, + // The compiled binary keeps them external through a plugin, which + // can tell an import from a require(). + ...(compiled ? [] : userExternals), ], stdin: { contents: await rewriteBundledImportMetaUrl(source, routeSourceUrl, denoImports), @@ -2244,7 +2243,7 @@ function buildTranspiledModuleSource( plugins: [ ...(compiled ? [ - createCompiledBuiltinPlugin(requiredBuiltins), + createCompiledBuiltinPlugin(), createDependencyResolvePlugin(isDependencyFile, hostIsFile), ] : []), @@ -2275,6 +2274,7 @@ function buildTranspiledModuleSource( workerImportMap, isDependencyFile, ), + ...(compiled ? [createImportExternalsPlugin(runtimeDeps.keys())] : []), createHTTPPlugin({ allowedHosts, projectDir, @@ -2307,7 +2307,7 @@ function buildTranspiledModuleSource( logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); const output = result.outputFiles?.[0]?.text ?? "export {}"; const js = compiled - ? prependRuntimeRequire(output, requiredBuiltins, await canonicalProjectRoot(projectDir)) + ? prependRuntimeRequire(output, await canonicalProjectRoot(projectDir)) : output; logger.debug(`transpiled size ${js.length} bytes`); diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 7e0a57547e..5b1554a141 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -11,6 +11,7 @@ import { createCompiledBuiltinPlugin, createDependencyFilePredicate, createDependencyResolvePlugin, + createImportExternalsPlugin, PackageTypeLookup, prepareDependencyFile, prependRuntimeRequire, @@ -60,8 +61,9 @@ function dependencyFilePlugin( return { name: "test-dependency-files", setup(build) { - build.onLoad({ filter: /\.[cm]?js$/ }, async (args) => { - if (!isDependencyFile(args.path)) return undefined; + // Every dependency file but JSON, as the route build's boundary loads them. + build.onLoad({ filter: /.*/ }, async (args) => { + if (!isDependencyFile(args.path) || args.path.endsWith(".json")) return undefined; const contents = await readFile(args.path, "utf8"); return { contents: await prepareDependencyFile({ @@ -72,7 +74,6 @@ function dependencyFilePlugin( packageType: await packageTypes.of(args.path.slice(0, args.path.lastIndexOf("/"))), isDependencyFile, isFile, - rewriteImportMeta: (source) => Promise.resolve(source), }), loader: "js", }; @@ -85,8 +86,8 @@ function dependencyFilePlugin( async function bundleAndImport( projectDir: string, route: string, + importExternals: string[] = [], ): Promise> { - const requiredBuiltins = new Set(); const isDependencyFile = await createDependencyFilePredicate(projectDir, hostFs); try { const result = await esbuild.build({ @@ -104,16 +105,13 @@ async function bundleAndImport( sourcefile: `${projectDir}/route.ts`, }, plugins: [ - createCompiledBuiltinPlugin(requiredBuiltins), + createCompiledBuiltinPlugin(), createDependencyResolvePlugin(isDependencyFile, isFile), + createImportExternalsPlugin(importExternals), dependencyFilePlugin(projectDir, isDependencyFile), ], }); - const code = prependRuntimeRequire( - result.outputFiles?.[0]?.text ?? "", - requiredBuiltins, - projectDir, - ); + const code = prependRuntimeRequire(result.outputFiles?.[0]?.text ?? "", projectDir); const file = `${projectDir}/handler.mjs`; await writeFile(file, code); return await import(pathToFileURL(file).href); @@ -192,6 +190,14 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { "var prefixOnly;", 'try { require.resolve("test"); } catch (error) { prefixOnly = error.code; }', "exports.prefixOnly = prefixOnly;", + // A computed built-in id is never seen by the bundler. + 'var fsId = ["f", "s"].join("");', + "exports.computedBuiltin = typeof require(fsId).readFileSync;", + 'exports.computedPrefixed = typeof require("node:" + "events");', + "exports.computedResolve = require.resolve(fsId);", + "var bareTest;", + 'try { require(["te", "st"].join("")); } catch (error) { bareTest = error.code; }', + "exports.bareTest = bareTest;", ].join("\n"), }); const mod = await bundleAndImport( @@ -212,6 +218,10 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { 'Cannot find module "a path outside this project or missing"', ], prefixOnly: "MODULE_NOT_FOUND", + computedBuiltin: "function", + computedPrefixed: "function", + computedResolve: "fs", + bareTest: "MODULE_NOT_FOUND", }); }); }); @@ -308,14 +318,22 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { "node_modules/modonly/package.json": JSON.stringify({ name: "modonly", module: "esm.js" }), "node_modules/modonly/index.js": 'module.exports = "index";', "node_modules/modonly/esm.js": 'export default "module";', + // An extensionless main and a default index pick .js over .ts too. + "node_modules/mainpkg/package.json": JSON.stringify({ name: "mainpkg", main: "impl" }), + "node_modules/mainpkg/impl.js": 'module.exports = "js";', + "node_modules/mainpkg/impl.ts": 'export default "ts";', + "node_modules/idxpkg/package.json": JSON.stringify({ name: "idxpkg" }), + "node_modules/idxpkg/index.js": 'module.exports = "js";', + "node_modules/idxpkg/index.ts": 'export default "ts";', }); await symlink("../packages/wsorder", `${projectDir}/node_modules/wsorder`); const mod = await bundleAndImport( projectDir, 'import order from "order";\nimport modonly from "modonly";\nimport wsorder from "wsorder";\n' + - "export const picked = [order.impl, order.data, modonly, wsorder];\n", + 'import mainpkg from "mainpkg";\nimport idxpkg from "idxpkg";\n' + + "export const picked = [order.impl, order.data, modonly, wsorder, mainpkg, idxpkg];\n", ); - assertEquals(mod.picked, ["js", "json", "index", "js"]); + assertEquals(mod.picked, ["js", "json", "index", "js", "js", "js"]); }); }); @@ -332,6 +350,104 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("prepares an extensionless CommonJS entry", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/bare/package.json": JSON.stringify({ name: "bare", main: "entry" }), + "node_modules/bare/entry": [ + "exports.dir = __dirname;", + "exports.file = __filename;", + 'exports.self = require.resolve("./entry");', + ].join("\n"), + }); + const mod = await bundleAndImport(projectDir, 'import bare from "bare";\nexport { bare };\n'); + const entry = `${projectDir}/node_modules/bare/entry`; + assertEquals(mod.bare, { dir: `${projectDir}/node_modules/bare`, file: entry, self: entry }); + }); + }); + + it("keeps import.meta locations of a dependency module its own", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/meta/package.json": JSON.stringify({ + name: "meta", + type: "module", + main: "index.js", + }), + "node_modules/meta/index.js": [ + "const meta = import.meta;", + "const resolve = import.meta.resolve;", + 'const name = ["./data", "json"].join(".");', + "export const url = import.meta.url;", + "export const dir = meta.dirname;", + "export const detached = resolve(name);", + 'export const builtin = import.meta.resolve("fs");', + 'export const computedBuiltin = resolve(["node", "path"].join(":"));', + 'export const sibling = import.meta.resolve("other");', + "let bare;", + 'try { resolve(["oth", "er"].join("")); } catch (error) { bare = error.code; }', + "export const computedBare = bare;", + ].join("\n"), + "node_modules/other/package.json": JSON.stringify({ name: "other", main: "main.js" }), + "node_modules/other/main.js": "module.exports = 1;", + }); + const mod = await bundleAndImport( + projectDir, + 'export * from "meta";\n', + ); + const base = pathToFileURL(`${projectDir}/node_modules`).href; + assertEquals({ ...mod }, { + url: `${base}/meta/index.js`, + dir: `${projectDir}/node_modules/meta`, + detached: `${base}/meta/data.json`, + builtin: "node:fs", + computedBuiltin: "node:path", + sibling: `${base}/other/main.js`, + computedBare: "ERR_MODULE_NOT_FOUND", + }); + }); + }); + + it("bundles a package whose name ends in .node", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/client.node/package.json": JSON.stringify({ + name: "client.node", + main: "index.js", + }), + "node_modules/client.node/index.js": 'module.exports = "client";', + }); + const mod = await bundleAndImport( + projectDir, + 'import client from "client.node";\nexport { client };\n', + ); + assertEquals(mod.client, "client"); + }); + }); + + it("bundles the CommonJS entry of a package it keeps external for imports", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/dual/package.json": JSON.stringify({ + name: "dual", + exports: { ".": { import: "./index.mjs", require: "./index.cjs" } }, + }), + "node_modules/dual/index.mjs": 'export default "esm";', + "node_modules/dual/index.cjs": 'module.exports = "cjs";', + "node_modules/user/package.json": JSON.stringify({ name: "user", main: "index.js" }), + "node_modules/user/index.js": 'module.exports = require("dual");', + }); + const mod = await bundleAndImport( + projectDir, + 'import user from "user";\nexport { user };\nexport const later = () => import("dual");\n', + ["dual"], + ); + assertEquals(mod.user, "cjs"); + const code = await readFile(`${projectDir}/handler.mjs`, "utf8"); + assert(code.includes('import("dual")'), code); + }); + }); + it("rejects a native addon at build time", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { @@ -343,7 +459,9 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { bundleAndImport(projectDir, 'import native from "native";\nexport { native };\n') ); assert( - String((error as Error).message).includes("Native addons are not supported"), + String((error as Error).message).includes( + 'Native addons are not supported in API routes: "binding.node"', + ), String(error), ); }); From 19c7713d0a1d330e8012e2bce9f6bbb84fea3035 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 22:17:26 +0100 Subject: [PATCH 12/29] test(api): unit-test dependency file parsing for compiled routes --- .../compiled-cjs-bundling.test.ts | 197 +++++++++++++++++- 1 file changed, 195 insertions(+), 2 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 5467116b1b..db55c29e60 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,7 +1,7 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals } from "#veryfront/testing/assert.ts"; -import { describe, it } from "#veryfront/testing/bdd.ts"; -import type { PluginBuild } from "veryfront/extensions/bundler"; +import { afterAll, describe, it } from "#veryfront/testing/bdd.ts"; +import { type PluginBuild, stop as stopBundler } from "veryfront/extensions/bundler"; import { builtinModuleName, createCompiledBuiltinPlugin, @@ -13,7 +13,9 @@ import { nodePreferredFile, nodeRelativeFile, PackageTypeLookup, + prepareCommonJSFile, prepareDependencyFile, + prepareModuleFile, prependRuntimeRequire, resolveLiteralCalls, type RuntimeRequireHost, @@ -416,4 +418,195 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(code.includes("import * as __vf_runtime___fs from"), code); }); }); + + // The bundler parses each file; its service process lives across tests. + describe("parsing dependency files", { sanitizeOps: false, sanitizeResources: false }, () => { + afterAll(async () => { + await stopBundler(); + }); + + const resolveRelative = (specifier: string) => + Promise.resolve(specifier.startsWith("./") ? "/d/p/" + specifier.slice(2) + ".js" : null); + + describe("prepareCommonJSFile()", () => { + it("binds __dirname and __filename and resolves require.resolve literals", async () => { + const code = await prepareCommonJSFile( + [ + "exports.d = __dirname;", + "exports.f = __filename;", + 'exports.a = require.resolve("./a");', + 'exports.b = require.resolve("fs");', + 'exports.c = require.resolve("missing");', + "exports.e = require.resolve(name);", + ].join("\n"), + "/d/p/index.cjs", + "module", + resolveRelative, + ); + assert(code !== null); + for ( + const part of [ + 'exports.d = "/d/p";', + 'exports.f = "/d/p/index.cjs";', + 'exports.a = "/d/p/a.js";', + 'exports.b = "fs";', + 'exports.c = require.resolve("missing");', + "exports.e = require.resolve(name);", + ] + ) { + assert(code.includes(part), code); + } + }); + + it("prepares an extensionless file and a .js file outside a module package", async () => { + const entry = await prepareCommonJSFile( + "exports.d = __dirname;", + "/d/p/entry", + "module", + resolveRelative, + ); + assert(entry?.includes('exports.d = "/d/p";'), String(entry)); + const js = await prepareCommonJSFile( + "exports.d = __dirname;", + "/d/p/a.js", + undefined, + resolveRelative, + ); + assert(js?.includes('exports.d = "/d/p";'), String(js)); + }); + + it("leaves files Node loads as ESM alone", async () => { + const source = "export const d = typeof __dirname;"; + assertEquals( + await prepareCommonJSFile(source, "/d/p/a.js", "module", resolveRelative), + null, + ); + assertEquals( + await prepareCommonJSFile(source, "/d/p/a.js", undefined, resolveRelative), + null, + ); + assertEquals( + await prepareCommonJSFile(source, "/d/p/entry", undefined, resolveRelative), + null, + ); + assertEquals( + await prepareCommonJSFile(source, "/d/p/a.mjs", undefined, resolveRelative), + null, + ); + }); + }); + + describe("prepareModuleFile()", () => { + it("makes import.meta locations constants and resolves literals", async () => { + const code = await prepareModuleFile( + 'export const u = import.meta.url;\nexport const f = import.meta.filename;\nexport const d = import.meta.dirname;\nexport const a = import.meta.resolve("./a");\n', + "/d/p/index.mjs", + resolveRelative, + ); + for ( + const part of [ + 'u = "file:///d/p/index.mjs"', + 'f = "/d/p/index.mjs"', + 'd = "/d/p"', + 'a = "/d/p/a.js"', + ] + ) { + assert(code.includes(part), code); + } + assert(!code.includes("function __vf_import_meta_resolve_"), code); + }); + + it("binds other uses to a resolver of the file, after any hashbang", async () => { + const code = await prepareModuleFile( + "#!/usr/bin/env node\nconst meta = import.meta;\nexport const r = import.meta.resolve(name);\nexport const m = meta;\n", + "/d/p/cli.ts", + resolveRelative, + ); + assert(code.startsWith("#!/usr/bin/env node\nfunction __vf_import_meta_resolve_"), code); + assert(code.includes(createImportMetaResolve.toString()), code); + assert( + /const __vf_import_meta_resolve_\w+_meta = \{ url: "file:\/\/\/d\/p\/cli\.ts"/.test(code), + code, + ); + }); + }); + + describe("prepareDependencyFile()", () => { + const resolved: Record = { + "./a": { path: "/d/node_modules/p/a.js" }, + pkg: { path: "/d/node_modules/pkg/main.js" }, + ext: { path: "ext", external: true }, + project: { path: "/d/src/project.js" }, + broken: { errors: [{ text: "x" }] }, + }; + const build = { + resolve: (specifier: string) => Promise.resolve(resolved[specifier] ?? { errors: [{}] }), + } as unknown as PluginBuild; + const input = (contents: string, logicalPath: string, withResolve = true) => ({ + build: withResolve ? build : ({} as PluginBuild), + contents, + logicalPath, + resolvedPath: logicalPath, + packageType: undefined, + isDependencyFile: (path: string) => path.startsWith("/d/node_modules/"), + isFile: () => Promise.resolve(false), + }); + + it("resolves require.resolve literals only to dependency files", async () => { + const code = await prepareDependencyFile( + input( + ["./a", "ext", "project", "broken"].map((id) => + `require.resolve(${JSON.stringify(id)});` + ) + .join("\n"), + "/d/node_modules/p/index.js", + ), + ); + for ( + const part of [ + '"/d/node_modules/p/a.js";', + 'require.resolve("ext");', + 'require.resolve("project");', + 'require.resolve("broken");', + ] + ) { + assert(code.includes(part), code); + } + const unresolved = await prepareDependencyFile( + input('require.resolve("./a");', "/d/node_modules/p/index.js", false), + ); + assert(unresolved.includes('require.resolve("./a");'), unresolved); + }); + + it("resolves import.meta.resolve literals in an ESM file", async () => { + const code = await prepareDependencyFile( + input( + ["fs", "./x.js", "https://e.test/m.js", "pkg", "ext"].map((id) => + `import.meta.resolve(${JSON.stringify(id)});` + ).join("\n"), + "/d/node_modules/p/index.mjs", + ), + ); + for ( + const part of [ + '"node:fs";', + '"file:///d/node_modules/p/x.js";', + '"https://e.test/m.js";', + '"file:///d/node_modules/pkg/main.js";', + "_resolve_", + ] + ) { + assert(code.includes(part), code); + } + }); + + it("leaves an ESM file that reads no import.meta unchanged", async () => { + const contents = "export const d = typeof __dirname;"; + assertEquals( + await prepareDependencyFile(input(contents, "/d/node_modules/p/a.mjs")), + contents, + ); + }); + }); + }); }); From 189546ebd445b115dcf241cc3c5a662c29d556a5 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 22:19:54 +0100 Subject: [PATCH 13/29] test(api): stop the bundler after each parsing test instead of disabling sanitizers --- .../api/module-loader/compiled-cjs-bundling.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index db55c29e60..06d91d4058 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,6 +1,6 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals } from "#veryfront/testing/assert.ts"; -import { afterAll, describe, it } from "#veryfront/testing/bdd.ts"; +import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; import { type PluginBuild, stop as stopBundler } from "veryfront/extensions/bundler"; import { builtinModuleName, @@ -419,9 +419,9 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); }); - // The bundler parses each file; its service process lives across tests. - describe("parsing dependency files", { sanitizeOps: false, sanitizeResources: false }, () => { - afterAll(async () => { + describe("parsing dependency files", () => { + // The bundler parses each file; its service process must not outlive a test. + afterEach(async () => { await stopBundler(); }); From d0532c29c1b91d8cbdd5443fccd50fd2d0953db3 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 23:02:29 +0100 Subject: [PATCH 14/29] fix(api): load computed dependency requires and keep compiled route output stable - The runtime require loads a dependency's JavaScript file that a computed path names, as CommonJS with its own require and a module cache. Project code still never loads this way. - Markers derive from the file, so the same source yields the same route module and its cache hits. - .cts files are CommonJS TypeScript; .cts and .mts load with the ts loader. - An exact extensionless require.resolve target is kept. - Project code's built-ins resolve after the import map, so a project can map a built-in name. - External import rewrites apply only to parsed import specifiers, never to bundled package text. --- .../compiled-cjs-bundling.test.ts | 223 +++++++++++--- .../module-loader/compiled-cjs-bundling.ts | 288 ++++++++++++++---- .../api/module-loader/loader-helpers.test.ts | 2 + .../api/module-loader/loader-helpers.ts | 2 + src/routing/api/module-loader/loader.ts | 34 ++- .../compiled-cjs-bundling.test.ts | 45 ++- 6 files changed, 484 insertions(+), 110 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 06d91d4058..9dc0f1493a 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -9,6 +9,7 @@ import { createDependencyResolvePlugin, createImportExternalsPlugin, createImportMetaResolve, + createNativeAddonPlugin, createRuntimeRequire, nodePreferredFile, nodeRelativeFile, @@ -18,6 +19,7 @@ import { prepareModuleFile, prependRuntimeRequire, resolveLiteralCalls, + rewriteImportSpecifiers, type RuntimeRequireHost, } from "./compiled-cjs-bundling.ts"; @@ -84,7 +86,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); describe("nodePreferredFile()", () => { - const files = new Set(["/d/impl.js", "/d/lib/index.js", "/d/data.json"]); + const files = new Set(["/d/impl.js", "/d/lib/index.js", "/d/data.json", "/d/entry.js"]); const isFile = (path: string) => Promise.resolve(files.has(path)); it("prefers Node's extensions over a TypeScript sibling the bundler probed", async () => { @@ -103,6 +105,8 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/d/node_modules/p/index.cjs", ); assertEquals(await nodePreferredFile("/d/only.ts", "./only", "/d", isFile), "/d/only.ts"); + // Node checks the exact path before adding an extension. + assertEquals(await nodePreferredFile("/d/entry", "./entry", "/d", isFile), "/d/entry"); }); }); @@ -195,21 +199,40 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { describe("createCompiledBuiltinPlugin()", () => { it("keeps built-ins external, with a required id unchanged", async () => { - const [builtins] = await resolveHandlers(createCompiledBuiltinPlugin()); - assertEquals(builtins!({ path: "fs", kind: "require-call" }), { + const [builtins] = await resolveHandlers(createCompiledBuiltinPlugin(() => true)); + assertEquals(builtins!({ path: "fs", kind: "require-call", importer: "/d/a.js" }), { path: "fs", external: true, }); - assertEquals(builtins!({ path: "path", kind: "import-statement" }), { + assertEquals(builtins!({ path: "path", kind: "import-statement", importer: "/d/a.js" }), { path: "node:path", external: true, }); - assertEquals(builtins!({ path: "left-pad", kind: "require-call" }), undefined); + assertEquals( + builtins!({ path: "left-pad", kind: "require-call", importer: "/d/a.js" }), + undefined, + ); }); + it("leaves other importers to the resolvers after it", async () => { + const [builtins] = await resolveHandlers( + createCompiledBuiltinPlugin((importer) => importer.includes("/node_modules/")), + ); + assertEquals( + builtins!({ path: "fs", kind: "import-statement", importer: "/d/src/route.ts" }), + undefined, + ); + assertEquals( + builtins!({ path: "fs", kind: "import-statement", importer: "/d/node_modules/p/a.js" }), + { path: "node:fs", external: true }, + ); + }); + }); + + describe("createNativeAddonPlugin()", () => { it("rejects a native addon once it is loaded, naming only the file", async () => { - assertEquals(await resolveHandlers(createCompiledBuiltinPlugin()).then((h) => h.length), 1); - const [native] = await handlers(createCompiledBuiltinPlugin(), "onLoad"); + assertEquals(await resolveHandlers(createNativeAddonPlugin()), []); + const [native] = await handlers(createNativeAddonPlugin(), "onLoad"); assertEquals(native!({ path: "/srv/app/node_modules/x/build/addon.node" }), { errors: [{ text: 'Native addons are not supported in API routes: "addon.node"' }], }); @@ -281,6 +304,48 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); }); + describe("rewriteImportSpecifiers()", () => { + it("rewrites parsed imports and leaves strings that read like imports", async () => { + const code = [ + 'import a from "fs";', + 'const later = () => import("fs");', + "const text = 'from \"fs\"';", + "const call = 'import(\"fs\")';", + 'export * from "dep/sub";', + ].join("\n"); + const probes: string[] = []; + const rewritten = await rewriteImportSpecifiers(code, (probe) => { + probes.push(probe); + return Promise.resolve( + probe.replaceAll('"fs"', '"node:fs"').replaceAll('"dep/sub"', '"file:///d/sub.js"'), + ); + }); + assertEquals(probes, ['export * from "fs";\nexport * from "dep/sub";']); + assertEquals( + rewritten, + [ + 'import a from "node:fs";', + 'const later = () => import("node:fs");', + "const text = 'from \"fs\"';", + "const call = 'import(\"fs\")';", + 'export * from "file:///d/sub.js";', + ].join("\n"), + ); + }); + + it("skips a module without imports and fails closed on a lost import", async () => { + assertEquals( + await rewriteImportSpecifiers("export const a = 1;", () => Promise.reject()), + "export const a = 1;", + ); + const error = await rewriteImportSpecifiers('import "fs";', () => Promise.resolve("")).then( + () => null, + (caught: Error) => caught.message, + ); + assertEquals(error, "[API] handler build failed: import specifiers could not be rewritten"); + }); + }); + describe("resolveLiteralCalls()", () => { it("replaces resolved literal calls once per specifier and keeps the rest", async () => { const asked: string[] = []; @@ -327,27 +392,57 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { const files: Record = { "/app/data.json": '{"a":1}', "/app/link.json": "", // a link to /outside.json - "/app/code.js": "", + "/app/code.js": "module.exports = 'project';", + "/app/node_modules/p/dialects/pg.js": [ + "exports.base = require('../base').name;", + "exports.fs = require('fs').kind;", + "exports.file = __filename;", + "exports.self = this === exports;", + ].join("\n"), + "/app/node_modules/p/base.js": "#!/usr/bin/env node\nexports.name = 'base';", + "/app/node_modules/p/dir/index.json": '{"dir":true}', + "/app/node_modules/p/cycle-a.js": + "exports.early = 1; exports.b = require('./cycle-b').seen; exports.late = 2;", + "/app/node_modules/p/cycle-b.js": "exports.seen = require('./cycle-a').early;", + "/app/node_modules/p/throws.js": "throw new Error('boom');", + "/app/node_modules/p/bare.js": "require('lodash');", }; let reads = 0; const host: RuntimeRequireHost = { // Like Deno's, it knows bare prefix-only names that Node does not. getBuiltinModule: (id) => id === "test" ? { kind: "test" } : builtins[id], - realpathSync: (path) => { - if (path === "/app/link.json") return "/outside.json"; - if (!(path in files)) throw new Error("ENOENT"); - return path; - }, + isFile: (path) => path in files, + realpathSync: (path) => path === "/app/link.json" ? "/outside.json" : path, readFileSync: (path) => { reads++; return files[path]!; }, + compile: (source) => + new Function( + "exports", + "require", + "module", + "__filename", + "__dirname", + source, + ) as ReturnType< + RuntimeRequireHost["compile"] + >, isAbsolute: (path) => path.startsWith("/"), relative: (from, to) => to.startsWith(from + "/") ? to.slice(from.length + 1) : to === from ? "" : "../" + to, + join: (...paths) => { + const parts: string[] = []; + for (const part of paths.join("/").split("/")) { + if (part === "..") parts.pop(); + else if (part !== "." && part !== "") parts.push(part); + } + return "/" + parts.join("/"); + }, + dirname: (path) => path.slice(0, path.lastIndexOf("/")), sep: "/", }; - const require = createRuntimeRequire(host, "/app", ["test"]); + const require = createRuntimeRequire(host, "/app", ["test"], ["/app/node_modules"]); it("serves any built-in, including a computed id", () => { assertEquals(require("fs"), { kind: "fs" }); @@ -366,34 +461,65 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(require("/app/data.json") === first); assertEquals(reads, 1); assertEquals(require.resolve("/app/code.js"), "/app/code.js"); + assertEquals(require.resolve("/app/data"), "/app/data.json"); + assertEquals(require("/app/node_modules/p/dir"), { dir: true }); }); - it("fails closed for packages, other files and paths outside the project", () => { - for (const id of ["left-pad", "./data.json", 42, "/app/code.js", "/app/link.json", "/app"]) { + it("runs a dependency's JavaScript as CommonJS, once, with its own require", () => { + const dialect = require("/app/node_modules/p/dialects/" + "pg"); + assertEquals(dialect, { + base: "base", + fs: "fs", + file: "/app/node_modules/p/dialects/pg.js", + self: true, + }); + assert(require("/app/node_modules/p/dialects/pg.js") === dialect); + }); + + it("hands a cycle the exports so far, as Node does", () => { + assertEquals(require("/app/node_modules/p/cycle-a.js"), { early: 1, b: 1, late: 2 }); + }); + + it("forgets a module whose body throws", () => { + for (let attempt = 0; attempt < 2; attempt++) { + let message: unknown; + try { + require("/app/node_modules/p/throws.js"); + } catch (error) { + message = (error as Error).message; + } + assertEquals(message, "boom"); + } + }); + + it("fails closed for packages, project code and paths outside the project", () => { + for ( + const id of [ + "left-pad", + "./data.json", + 42, + "/app/code.js", + "/app/link.json", + "/app", + "/app/node_modules/p/bare.js", + ] + ) { assertEquals(errorCode(() => require(id)), "MODULE_NOT_FOUND", String(id)); } assertEquals(errorCode(() => require.resolve("/app/missing.js")), "MODULE_NOT_FOUND"); }); it("names a project path relative to the project and hides any other", () => { + const message = (id: string) => { + try { + require(id); + } catch (error) { + return (error as Error).message; + } + }; + assertEquals(message("/app/code.js"), 'Cannot find module "code.js"'); assertEquals( - (() => { - try { - require("/app/code.js"); - } catch (error) { - return (error as Error).message; - } - })(), - 'Cannot find module "code.js"', - ); - assertEquals( - (() => { - try { - require("/app/link.json"); - } catch (error) { - return (error as Error).message; - } - })(), + message("/app/link.json"), 'Cannot find module "a path outside this project or missing"', ); }); @@ -401,20 +527,25 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { describe("prependRuntimeRequire()", () => { it("binds require to the runtime require with the host built-ins", () => { - const code = prependRuntimeRequire("export {};\n", "/app"); + const code = prependRuntimeRequire("export {};\n", "/app", ["/app/node_modules"]); assert(code.includes(createRuntimeRequire.toString()), code); assert(code.includes('import __vf_runtime_process from "node:process";'), code); - assert(code.includes('}, "/app", ["test","test/reporters","sqlite","sea"]);'), code); + assert( + code.includes( + '}, "/app", ["test","test/reporters","sqlite","sea"], ["/app/node_modules"]);', + ), + code, + ); assert(code.endsWith("\nexport {};\n"), code); }); it("keeps a hashbang on the first line", () => { - const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", "/app"); + const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", "/app", []); assert(code.startsWith("#!/usr/bin/env node\nimport __vf_runtime_process from "), code); }); it("picks helper names the bundle does not already use", () => { - const code = prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app"); + const code = prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app", []); assert(code.includes("import * as __vf_runtime___fs from"), code); }); }); @@ -475,6 +606,16 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(js?.includes('exports.d = "/d/p";'), String(js)); }); + it("prepares a .cts file as CommonJS TypeScript", async () => { + const code = await prepareCommonJSFile( + "const d: string = __dirname;\nexports.d = d;", + "/d/p/index.cts", + "module", + resolveRelative, + ); + assert(code?.includes('const d = "/d/p";'), String(code)); + }); + it("leaves files Node loads as ESM alone", async () => { const source = "export const d = typeof __dirname;"; assertEquals( @@ -516,6 +657,14 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(!code.includes("function __vf_import_meta_resolve_"), code); }); + it("yields the same output for the same file", async () => { + const source = "export const r = import.meta.resolve(name);\n"; + assertEquals( + await prepareModuleFile(source, "/d/p/a.mjs", resolveRelative), + await prepareModuleFile(source, "/d/p/a.mjs", resolveRelative), + ); + }); + it("binds other uses to a resolver of the file, after any hashbang", async () => { const code = await prepareModuleFile( "#!/usr/bin/env node\nconst meta = import.meta;\nexport const r = import.meta.resolve(name);\nexport const m = meta;\n", diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index be02ee6118..3b8850b662 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -29,6 +29,7 @@ import { build as bundle } from "veryfront/extensions/bundler"; import type { Plugin, PluginBuild } from "veryfront/extensions/bundler"; import * as pathHelper from "#veryfront/compat/path"; import { isWithinDirectory } from "#veryfront/security/path-validation.ts"; +import { parseImports, replaceSpecifiers } from "#veryfront/transforms/esm/lexer.ts"; // Names Node accepts only with the `node:` prefix. A bare `test` is an npm package. const PREFIX_ONLY_BUILTINS = new Set(["test", "test/reporters", "sqlite", "sea"]); @@ -62,21 +63,35 @@ function nativeAddonError(path: string) { } /** - * Keeps built-ins external. A `require()` of one keeps its id, for the runtime - * `require` of {@link prependRuntimeRequire}. A native addon fails the build - * once it resolves to a file: a package whose name ends in `.node` is fine. - * Must run before any plugin that resolves built-ins or loads files itself. + * Keeps built-ins external for the importers `appliesTo` accepts. A + * `require()` of one keeps its id, for the runtime `require` of + * {@link prependRuntimeRequire}. A dependency's built-ins must be claimed + * before the project's import map, which does not apply to it; project code's + * after it, so that a project can map a built-in name to its own module. */ -export function createCompiledBuiltinPlugin(): Plugin { +export function createCompiledBuiltinPlugin(appliesTo: (importer: string) => boolean): Plugin { return { name: "vf-compiled-builtins", setup(build) { build.onResolve({ filter: /.*/ }, (args) => { const name = builtinModuleName(args.path); - if (name === undefined) return undefined; + if (name === undefined || !appliesTo(args.importer)) return undefined; if (args.kind === "require-call") return { path: args.path, external: true }; return { path: `node:${name}`, external: true }; }); + }, + }; +} + +/** + * Fails the build on a native addon once a path resolves to one: a package + * whose name ends in `.node` is fine. Must run before any plugin that loads + * files itself. + */ +export function createNativeAddonPlugin(): Plugin { + return { + name: "vf-native-addons", + setup(build) { build.onLoad({ filter: NATIVE_ADDON }, (args) => nativeAddonError(args.path)); }, }; @@ -117,7 +132,10 @@ export async function nodePreferredFile( const relative = specifier.startsWith("./") || specifier.startsWith("../") || specifier === "." || specifier === ".." || pathHelper.isAbsolute(specifier); const extension = pathHelper.extname(resolvedPath); - if (!relative || NODE_REQUIRE_EXTENSIONS.includes(extension)) return resolvedPath; + // An extensionless path is the file the import named exactly; Node checks it first. + if (!relative || extension === "" || NODE_REQUIRE_EXTENSIONS.includes(extension)) { + return resolvedPath; + } const stem = resolvedPath.slice(0, resolvedPath.length - extension.length); const named = pathHelper.resolve(resolveDir, specifier); if (stem !== named && stem !== pathHelper.join(named, "index")) return resolvedPath; @@ -311,15 +329,15 @@ async function linkedWorkspaceDirectories( } /** - * Whether a path the bundler resolved is an installed dependency's file: a - * file under the project's `node_modules`, under either the given or the - * canonical root (esbuild reports paths with symlinks followed), or under a - * workspace package that `node_modules` links to inside the project. + * The directories that hold installed dependencies' files: the project's + * `node_modules`, under either the given or the canonical root (esbuild + * reports paths with symlinks followed), and each workspace package that + * `node_modules` links to inside the project. */ -export async function createDependencyFilePredicate( +export async function dependencyDirectories( projectDir: string, fs: DependencyRootFileSystem, -): Promise<(path: string) => boolean> { +): Promise { const given = pathHelper.resolve(projectDir); // null: an adapter-backed project may have no host directory const canonical = await fs.realPath(given).catch(() => null); @@ -330,7 +348,12 @@ export async function createDependencyFilePredicate( ...await linkedWorkspaceDirectories(fs, canonical), ]), ]; - const prefixes = [...new Set(directories)].map((directory) => directory + pathHelper.sep); + return [...new Set(directories)]; +} + +/** Whether a path is a file inside one of the {@link dependencyDirectories}. */ +export function dependencyFilePredicate(directories: readonly string[]): (path: string) => boolean { + const prefixes = directories.map((directory) => directory + pathHelper.sep); return (path) => { if (path === "") return false; const resolved = pathHelper.resolve(path); @@ -338,12 +361,46 @@ export async function createDependencyFilePredicate( }; } +/** {@link dependencyFilePredicate} of the project's {@link dependencyDirectories}. */ +export async function createDependencyFilePredicate( + projectDir: string, + fs: DependencyRootFileSystem, +): Promise<(path: string) => boolean> { + return dependencyFilePredicate(await dependencyDirectories(projectDir, fs)); +} + +// The loader that parses a dependency file, by extension. +const LOADERS: Record = { + ".ts": "ts", + ".mts": "ts", + ".cts": "ts", + ".tsx": "tsx", + ".jsx": "jsx", +}; + +/** + * A name `contents` does not contain, derived from the file alone, so that the + * same file always yields the same output: the route module is cached by a + * hash of its code. + */ +async function markerFor(prefix: string, filePath: string, contents: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(filePath + "\0" + contents), + ); + let marker = prefix + [...new Uint8Array(digest).slice(0, 16)] + .map((byte) => byte.toString(16).padStart(2, "0")).join(""); + while (contents.includes(marker)) marker += "_"; + return marker; +} + /** Resolves a `require.resolve` literal to a path, or null to leave it for runtime. */ export type ResolveRequireLiteral = (specifier: string) => Promise; /** - * A dependency file as Node would run it when Node loads it as CommonJS, or - * null when Node loads it as ESM: a `.mjs` file, a `.js` file in a + * A dependency file as Node would run it when Node loads it as CommonJS (a + * `.cjs` or `.cts` file, or a `.js` or extensionless one as below), or null + * when Node loads it as ESM: a `.mjs` file, a `.js` file in a * `"type": "module"` package, or a `.js` or extensionless file with ESM * syntax, which esbuild bundles as ESM whatever its package says. * @@ -362,18 +419,22 @@ export async function prepareCommonJSFile( packageType: string | undefined, resolve: ResolveRequireLiteral, ): Promise { - const cjsExtension = /\.cjs$/i.test(filePath); + const cjsExtension = /\.c[jt]s$/i.test(filePath); const jsExtension = /\.js$/i.test(filePath); // Node's require() loads a file without an extension as CommonJS, whatever // its package declares; a package `main` may name one. const extensionless = pathHelper.extname(filePath) === ""; if (!cjsExtension && !extensionless && !(jsExtension && packageType !== "module")) return null; - // A fresh name per file: the source cannot contain it, so every occurrence - // in the output is one the parser put there. It never reaches the bundle. - const marker = `__vf_require_resolve_${crypto.randomUUID().replaceAll("-", "")}`; + // The source does not contain the name, so every occurrence in the output + // is one the parser put there. It never reaches the bundle. + const marker = await markerFor("__vf_require_resolve_", filePath, contents); const result = await bundle({ - stdin: { contents, loader: "js", sourcefile: filePath }, + stdin: { + contents, + loader: LOADERS[pathHelper.extname(filePath).toLowerCase()] ?? "js", + sourcefile: filePath, + }, bundle: false, write: false, metafile: true, @@ -426,14 +487,6 @@ export async function resolveLiteralCalls( /** Resolves an `import.meta.resolve` literal to a URL, or null to leave it for runtime. */ export type ResolveImportMetaLiteral = (specifier: string) => Promise; -const LOADERS: Record = { - ".ts": "ts", - ".mts": "ts", - ".cts": "ts", - ".tsx": "tsx", - ".jsx": "jsx", -}; - /** * An ES module dependency file with its `import.meta` locations pointing at * the file itself, not at the route bundle it ends up in. Unbound reads of @@ -451,7 +504,7 @@ export async function prepareModuleFile( resolve: ResolveImportMetaLiteral, ): Promise { const url = pathHelper.toFileUrl(filePath).href; - const marker = `__vf_import_meta_resolve_${crypto.randomUUID().replaceAll("-", "")}`; + const marker = await markerFor("__vf_import_meta_resolve_", filePath, contents); const result = await bundle({ stdin: { contents, @@ -581,16 +634,60 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise }); } +/** + * Applies a text rewrite of import specifiers to the parsed imports of `code` + * only. A bundle holds package code whose strings may read like imports, such + * as `'from "fs"'`; those stay as written. `rewrite` sees one + * `export * from "specifier";` line per distinct specifier and must keep one + * import per line. + */ +export async function rewriteImportSpecifiers( + code: string, + rewrite: (probe: string) => Promise, +): Promise { + const specifiers = [ + ...new Set( + (await parseImports(code)) + .map((entry) => entry.n) + .filter((specifier): specifier is string => typeof specifier === "string"), + ), + ]; + if (specifiers.length === 0) return code; + const probe = specifiers.map((specifier) => `export * from ${JSON.stringify(specifier)};`); + const rewritten = (await parseImports(await rewrite(probe.join("\n")))).map((entry) => entry.n); + if (rewritten.length !== specifiers.length || rewritten.some((entry) => entry === undefined)) { + throw new Error("[API] handler build failed: import specifiers could not be rewritten"); + } + const replacements = new Map( + specifiers.map((specifier, index) => [specifier, rewritten[index]!]), + ); + return await replaceSpecifiers(code, (specifier) => replacements.get(specifier)); +} + /** Host operations the runtime `require` uses; see {@link createRuntimeRequire}. */ export interface RuntimeRequireHost { getBuiltinModule(id: string): unknown; + isFile(path: string): boolean; realpathSync(path: string): string; readFileSync(path: string, encoding: "utf8"): string; + /** Compiles a CommonJS module body. */ + compile(source: string, filename: string): CommonJSModuleBody; isAbsolute(path: string): boolean; relative(from: string, to: string): string; + join(...paths: string[]): string; + dirname(path: string): string; readonly sep: string; } +/** A compiled CommonJS module body, called with `this` set to `exports`. */ +export type CommonJSModuleBody = ( + exports: unknown, + require: RuntimeRequire, + module: { exports: unknown }, + filename: string, + dirname: string, +) => void; + /** The runtime `require` of a bundled route; see {@link createRuntimeRequire}. */ export interface RuntimeRequire { (id: unknown): unknown; @@ -598,14 +695,20 @@ export interface RuntimeRequire { } /** - * The `require` that esbuild's output calls for anything it did not bundle: a - * built-in, under any id Node accepts, including a computed one; and, for a - * computed id, an existing file inside the project, which `require.resolve` - * returns and `require` loads when it is JSON, cached as Node's module cache - * keeps it. Anything else throws `MODULE_NOT_FOUND`, so optional dependencies - * probed in a `try` still fall back. Errors name a path relative to the - * project, never the host path. + * The `require` that esbuild's output calls for anything it did not bundle: + * a computed id, or a built-in. * + * - A built-in loads under any id Node accepts. + * - An absolute path, or a relative one inside a module this loaded, finds a + * file in Node's order: the path, then with `.js` or `.json`, then its + * `index.js` or `index.json`. Its real path must be inside the project. + * JSON loads anywhere in the project; JavaScript only from the project's + * dependencies, as CommonJS, with its own `require`. Each loads once, as in + * Node's module cache. `require.resolve` returns the path it found. + * - Anything else, a package name included, throws `MODULE_NOT_FOUND`, so + * optional dependencies probed in a `try` still fall back. + * + * Errors name a path relative to the project, never the host path. * {@link prependRuntimeRequire} embeds this function's source in the route * module, so it must not refer to anything outside its own body. */ @@ -613,12 +716,14 @@ export function createRuntimeRequire( host: RuntimeRequireHost, projectRoot: string, prefixOnlyNames: readonly string[], + dependencyDirectories: readonly string[], ): RuntimeRequire { // Tables without a prototype: `in` reads only their own keys and does not // depend on built-ins that route code could replace later. const prefixOnly: Record = Object.create(null); for (const name of prefixOnlyNames) prefixOnly[name] = true; - const json: Record = Object.create(null); + const modules: Record = Object.create(null); + const dependencyPrefixes = dependencyDirectories.map((directory) => directory + host.sep); const builtin = (id: unknown) => { // A bare prefix-only name such as `test` is an npm package, as in Node. if (typeof id !== "string" || id in prefixOnly) return undefined; @@ -631,42 +736,92 @@ export function createRuntimeRequire( error.code = "MODULE_NOT_FOUND"; return error; }; - // A path is usable only when it exists and its real path is in the project. - const file = (id: unknown) => { - if (typeof id !== "string" || !host.isAbsolute(id)) return null; + // An existing file whose real path is in the project. + const projectFile = (path: string) => { + if (!host.isFile(path)) return null; let real: string; try { - real = host.realpathSync(id); + real = host.realpathSync(path); } catch { return null; } const relative = host.relative(projectRoot, real); const up = relative === ".." || relative.startsWith(".." + host.sep); if (relative === "" || up || host.isAbsolute(relative)) return null; - return { real, relative }; + return { path, real, relative }; + }; + const relativeId = (id: string) => + id === "." || id === ".." || id.startsWith("./") || id.startsWith("../"); + const find = (id: unknown, directory: string | undefined) => { + if (typeof id !== "string") return null; + let base: string; + if (host.isAbsolute(id)) base = id; + else if (directory !== undefined && relativeId(id)) base = host.join(directory, id); + else return null; + const candidates = [ + base, + base + ".js", + base + ".json", + host.join(base, "index.js"), + host.join(base, "index.json"), + ]; + for (const candidate of candidates) { + const found = projectFile(candidate); + if (found !== null) return found; + } + return null; }; const label = (id: unknown) => - typeof id === "string" && host.isAbsolute(id) + typeof id === "string" && (host.isAbsolute(id) || relativeId(id)) ? "a path outside this project or missing" : String(id); - const require = ((id: unknown) => { - const module = builtin(id); - if (module !== undefined) return module; - const found = file(id); - if (found !== null && found.real.endsWith(".json")) { - if (!(found.real in json)) { - json[found.real] = JSON.parse(host.readFileSync(found.real, "utf8")); - } - return json[found.real]; + const evaluate = (real: string) => { + const module = { exports: {} as unknown }; + modules[real] = module; + const directory = host.dirname(real); + const source = host.readFileSync(real, "utf8"); + try { + // A hashbang is not JavaScript; Node ignores it too. + host.compile(source.startsWith("#!") ? "//" + source : source, real).call( + module.exports, + module.exports, + create(directory), + module, + real, + directory, + ); + } catch (error) { + delete modules[real]; + throw error; } - throw missing(found !== null ? found.relative : label(id)); - }) as RuntimeRequire; - require.resolve = (id: unknown) => { - if (builtin(id) !== undefined) return id as string; - if (file(id) !== null) return id as string; - throw missing(label(id)); + return module.exports; + }; + const create = (directory: string | undefined) => { + const require = ((id: unknown) => { + const module = builtin(id); + if (module !== undefined) return module; + const found = find(id, directory); + if (found === null) throw missing(label(id)); + if (found.real in modules) return modules[found.real]!.exports; + if (found.real.endsWith(".json")) { + const exports = JSON.parse(host.readFileSync(found.real, "utf8")); + modules[found.real] = { exports }; + return exports; + } + if (dependencyPrefixes.some((prefix) => found.real.startsWith(prefix))) { + return evaluate(found.real); + } + throw missing(found.relative); + }) as RuntimeRequire; + require.resolve = (id: unknown) => { + if (builtin(id) !== undefined) return id as string; + const found = find(id, directory); + if (found === null) throw missing(label(id)); + return found.path; + }; + return require; }; - return require; + return create(undefined); } /** @@ -675,15 +830,24 @@ export function createRuntimeRequire( * declares a top-level `require` (it renames a route's own binding of that * name), so the prologue cannot collide with route code. */ -export function prependRuntimeRequire(code: string, projectRoot: string): string { +export function prependRuntimeRequire( + code: string, + projectRoot: string, + dependencyDirectories: readonly string[], +): string { let prefix = "__vf_runtime"; while (code.includes(prefix)) prefix += "_"; const host = [ `getBuiltinModule: (id) => ${prefix}_process.getBuiltinModule(id)`, + `isFile: (path) => { try { return ${prefix}_fs.statSync(path).isFile(); } catch { return false; } }`, `realpathSync: ${prefix}_fs.realpathSync`, `readFileSync: ${prefix}_fs.readFileSync`, + // Captured now: route code runs after the prologue. + `compile: ((F) => (source) => new F("exports", "require", "module", "__filename", "__dirname", source))(Function)`, `isAbsolute: ${prefix}_path.isAbsolute`, `relative: ${prefix}_path.relative`, + `join: ${prefix}_path.join`, + `dirname: ${prefix}_path.dirname`, `sep: ${prefix}_path.sep`, ]; const lines = [ @@ -692,7 +856,7 @@ export function prependRuntimeRequire(code: string, projectRoot: string): string `import * as ${prefix}_path from "node:path";`, `const require = (${createRuntimeRequire.toString()})({ ${host.join(", ")} }, ${ JSON.stringify(projectRoot) - }, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])});`, + }, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])}, ${JSON.stringify(dependencyDirectories)});`, ]; // A hashbang is only valid as the first line. const hashbangEnd = code.startsWith("#!") ? code.indexOf("\n") + 1 : 0; diff --git a/src/routing/api/module-loader/loader-helpers.test.ts b/src/routing/api/module-loader/loader-helpers.test.ts index 91badec5dc..d14289ffe9 100644 --- a/src/routing/api/module-loader/loader-helpers.test.ts +++ b/src/routing/api/module-loader/loader-helpers.test.ts @@ -44,6 +44,8 @@ describe("routing/api/module-loader helpers", () => { assertEquals(getLoaderForFile("file.jsx"), "jsx", "jsx files must use the jsx loader"); assertEquals(getLoaderForFile("file.json"), "json", "json files must use the json loader"); assertEquals(getLoaderForFile("file.mjs"), "js", "mjs files must use the js loader"); + assertEquals(getLoaderForFile("file.cts"), "ts", "cts files are CommonJS TypeScript"); + assertEquals(getLoaderForFile("file.mts"), "ts", "mts files are ESM TypeScript"); assertEquals(getLoaderForFile("noext"), "js", "unmapped extensions fall back to the js loader"); }); }); diff --git a/src/routing/api/module-loader/loader-helpers.ts b/src/routing/api/module-loader/loader-helpers.ts index 1663624ccb..2b7dba6ee2 100644 --- a/src/routing/api/module-loader/loader-helpers.ts +++ b/src/routing/api/module-loader/loader-helpers.ts @@ -6,6 +6,8 @@ const EXT_TO_LOADER: Record = { tsx: "tsx", jsx: "jsx", ts: "ts", + mts: "ts", + cts: "ts", json: "json", }; diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 939fa08a9a..6d959174db 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -51,13 +51,16 @@ import { import { COMPILED_RESOLVE_OPTIONS, createCompiledBuiltinPlugin, - createDependencyFilePredicate, createDependencyResolvePlugin, createImportExternalsPlugin, + createNativeAddonPlugin, + dependencyDirectories, + dependencyFilePredicate, type IsFile, PackageTypeLookup, prepareDependencyFile, prependRuntimeRequire, + rewriteImportSpecifiers, } from "./compiled-cjs-bundling.ts"; import { MAX_WORKER_MODULE_SOURCE_BYTES, @@ -2182,11 +2185,14 @@ function buildTranspiledModuleSource( 'import { createRequire as __vf_createRequire } from "node:module";', `var require = __vf_createRequire(${safeProjectDir});`, ].join("\n"); - const isDependencyFile = compiled - ? await createDependencyFilePredicate(projectDir, { + const dependencyDirs = compiled + ? await dependencyDirectories(projectDir, { realPath, readDir: (path) => createFileSystem().readDir(path), }) + : []; + const isDependencyFile = compiled + ? dependencyFilePredicate(dependencyDirs) : noDependencyFiles; const routeSourceUrl = pathHelper.toFileUrl(resolvedPath).href; const workerShim = [ @@ -2243,7 +2249,8 @@ function buildTranspiledModuleSource( plugins: [ ...(compiled ? [ - createCompiledBuiltinPlugin(), + createNativeAddonPlugin(), + createCompiledBuiltinPlugin(isDependencyFile), createDependencyResolvePlugin(isDependencyFile, hostIsFile), ] : []), @@ -2274,7 +2281,13 @@ function buildTranspiledModuleSource( workerImportMap, isDependencyFile, ), - ...(compiled ? [createImportExternalsPlugin(runtimeDeps.keys())] : []), + ...(compiled + ? [ + // After the import map, so that a project can map a built-in name. + createCompiledBuiltinPlugin(() => true), + createImportExternalsPlugin(runtimeDeps.keys()), + ] + : []), createHTTPPlugin({ allowedHosts, projectDir, @@ -2307,13 +2320,16 @@ function buildTranspiledModuleSource( logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); const output = result.outputFiles?.[0]?.text ?? "export {}"; const js = compiled - ? prependRuntimeRequire(output, await canonicalProjectRoot(projectDir)) + ? prependRuntimeRequire(output, await canonicalProjectRoot(projectDir), dependencyDirs) : output; logger.debug(`transpiled size ${js.length} bytes`); - // Only dependencies left external are rewritten: bundled package code - // may contain text that looks like an import of a bundled dependency. - return await rewriteExternalImports(js, projectDir, sourceSnapshot, runtimeDeps); + // Only dependencies left external are rewritten. The compiled bundle + // holds package code whose strings may read like imports, so there only + // parsed import specifiers are. + const rewrite = (code: string) => + rewriteExternalImports(code, projectDir, sourceSnapshot, runtimeDeps); + return compiled ? await rewriteImportSpecifiers(js, rewrite) : await rewrite(js); }, { "api.modulePath": modulePath, "api.projectDir": projectDir }, ); diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 5b1554a141..adaa449fcd 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -12,6 +12,8 @@ import { createDependencyFilePredicate, createDependencyResolvePlugin, createImportExternalsPlugin, + createNativeAddonPlugin, + dependencyDirectories, PackageTypeLookup, prepareDependencyFile, prependRuntimeRequire, @@ -105,13 +107,19 @@ async function bundleAndImport( sourcefile: `${projectDir}/route.ts`, }, plugins: [ - createCompiledBuiltinPlugin(), + createNativeAddonPlugin(), + createCompiledBuiltinPlugin(isDependencyFile), createDependencyResolvePlugin(isDependencyFile, isFile), + createCompiledBuiltinPlugin(() => true), createImportExternalsPlugin(importExternals), dependencyFilePlugin(projectDir, isDependencyFile), ], }); - const code = prependRuntimeRequire(result.outputFiles?.[0]?.text ?? "", projectDir); + const code = prependRuntimeRequire( + result.outputFiles?.[0]?.text ?? "", + projectDir, + await dependencyDirectories(projectDir, hostFs), + ); const file = `${projectDir}/handler.mjs`; await writeFile(file, code); return await import(pathToFileURL(file).href); @@ -350,6 +358,39 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("runs a dependency file a computed path requires", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/dialects/package.json": JSON.stringify({ + name: "dialects", + main: "index.js", + }), + "node_modules/dialects/index.js": [ + 'var path = require("path");', + "exports.load = function (name) {", + ' return require(path.join(__dirname, "dialects", name + ".js"));', + "};", + ].join("\n"), + "node_modules/dialects/dialects/pg.js": + 'var base = require("../base");\nexports.name = base.prefix + "pg";\nexports.sep = require("path").sep;', + "node_modules/dialects/base.js": 'exports.prefix = "dialect:";', + "src/secret.js": 'module.exports = "project code";', + }); + const mod = await bundleAndImport( + projectDir, + 'import dialects from "dialects";\n' + + "export const pg = dialects.load('pg');\n" + + "export const again = dialects.load('pg') === pg;\n" + + "let project;\n" + + 'try { dialects.load("../../../src/secret"); } catch (error) { project = (error as { code?: string }).code; }\n' + + "export { project };\n", + ); + assertEquals(mod.pg, { name: "dialect:pg", sep: "/" }); + assertEquals(mod.again, true); + assertEquals(mod.project, "MODULE_NOT_FOUND"); + }); + }); + it("prepares an extensionless CommonJS entry", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { From c799b566422ebbc85506afbb6d312e80103444ef Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 6 Nov 2026 23:35:29 +0100 Subject: [PATCH 15/29] fix(api): keep require.ensure on the compiled runtime require --- .../api/module-loader/compiled-cjs-bundling.test.ts | 11 +++++++++++ .../api/module-loader/compiled-cjs-bundling.ts | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 9dc0f1493a..e4c2eaa362 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -406,6 +406,8 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/app/node_modules/p/cycle-b.js": "exports.seen = require('./cycle-a').early;", "/app/node_modules/p/throws.js": "throw new Error('boom');", "/app/node_modules/p/bare.js": "require('lodash');", + "/app/node_modules/p/split.js": + "require.ensure(['./base'], function (r) { exports.base = r('./base').name; });", }; let reads = 0; const host: RuntimeRequireHost = { @@ -476,6 +478,15 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(require("/app/node_modules/p/dialects/pg.js") === dialect); }); + it("runs a require.ensure callback at once, at the top and in a loaded module", () => { + let ensured: unknown; + require.ensure([], (inner) => { + ensured = inner("fs"); + }); + assertEquals(ensured, { kind: "fs" }); + assertEquals(require("/app/node_modules/p/split.js"), { base: "base" }); + }); + it("hands a cycle the exports so far, as Node does", () => { assertEquals(require("/app/node_modules/p/cycle-a.js"), { early: 1, b: 1, late: 2 }); }); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 3b8850b662..e147bdb5d6 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -692,6 +692,8 @@ export type CommonJSModuleBody = ( export interface RuntimeRequire { (id: unknown): unknown; resolve(id: unknown): string; + /** webpack's `require.ensure`: everything is loaded already. */ + ensure(ids: unknown, callback: (require: RuntimeRequire) => void): void; } /** @@ -707,6 +709,7 @@ export interface RuntimeRequire { * Node's module cache. `require.resolve` returns the path it found. * - Anything else, a package name included, throws `MODULE_NOT_FOUND`, so * optional dependencies probed in a `try` still fall back. + * - `require.ensure`, which code written for webpack calls, runs its callback. * * Errors name a path relative to the project, never the host path. * {@link prependRuntimeRequire} embeds this function's source in the route @@ -819,6 +822,10 @@ export function createRuntimeRequire( if (found === null) throw missing(label(id)); return found.path; }; + // Code written for webpack; the callback runs at once, as nothing is split. + require.ensure = (_ids: unknown, callback: (require: RuntimeRequire) => void) => { + callback(require); + }; return require; }; return create(undefined); From e6431f37f76c93b883d4e9f13a5b45b3a35c27cd Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 00:11:04 +0100 Subject: [PATCH 16/29] fix(api): resolve packages from runtime-loaded dependencies - A dependency file loaded through a computed require resolves package names from its own directory upwards, through exports (node, require, default) or main, never above the project. - .jsx dependency files get CommonJS preparation like .js files. - A linked @scope directory counts as a dependency directory. - The package-type lookup walks under the canonical project root too. --- .../compiled-cjs-bundling.test.ts | 68 +++++++++ .../module-loader/compiled-cjs-bundling.ts | 134 +++++++++++++++--- src/routing/api/module-loader/loader.ts | 12 +- .../compiled-cjs-bundling.test.ts | 16 ++- 4 files changed, 205 insertions(+), 25 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index e4c2eaa362..d7c6d8dd77 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -79,6 +79,19 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(await lookup().of("/app/node_modules/typed/lib"), "commonjs"); }); + it("walks up from a path under the canonical root as well", async () => { + const both = new PackageTypeLookup( + (path) => + Promise.resolve( + path === "/real/app/node_modules/esm/package.json" + ? JSON.stringify({ type: "module" }) + : null, + ), + ["/app", "/real/app"], + ); + assertEquals(await both.of("/real/app/node_modules/esm/lib/deep"), "module"); + }); + it("stops at node_modules, as Node does, so a package without a manifest is CommonJS", async () => { assertEquals(await lookup().of("/app/node_modules/bare/lib"), undefined); assertEquals(await lookup().of("/app/node_modules/@scope/bare"), undefined); @@ -143,6 +156,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/app": "/real/app", "/real/app/node_modules/linked": "/real/app/packages/linked", "/real/app/node_modules/@s/w": "/real/app/packages/w", + "/real/app/node_modules/@ls": "/real/app/packages/ls", "/real/app/node_modules/out": "/elsewhere/out", "/real/app/node_modules/self": "/real/app", "/real/app/node_modules/inner": "/real/app/node_modules/plain", @@ -151,6 +165,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/real/app/node_modules": [ { name: "linked", isDirectory: false, isSymlink: true }, { name: "@s", isDirectory: true, isSymlink: false }, + { name: "@ls", isDirectory: false, isSymlink: true }, { name: "plain", isDirectory: true, isSymlink: false }, { name: "out", isDirectory: false }, { name: "self", isDirectory: false, isSymlink: true }, @@ -158,6 +173,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { { name: "dead", isDirectory: false, isSymlink: true }, ], "/real/app/node_modules/@s": [{ name: "w", isDirectory: false, isSymlink: true }], + "/real/app/node_modules/@ls": [{ name: "pkg", isDirectory: true, isSymlink: false }], }; const fs = { realPath(path: string) { @@ -177,6 +193,8 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(isDependency("/real/app/node_modules/x/index.js")); assert(isDependency("/real/app/packages/linked/index.js")); assert(isDependency("/real/app/packages/w/index.js")); + // A linked scope directory holds plain package directories. + assert(isDependency("/real/app/packages/ls/pkg/index.js")); }); it("never counts project code, the project root or targets outside the project", async () => { @@ -406,6 +424,29 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/app/node_modules/p/cycle-b.js": "exports.seen = require('./cycle-a').early;", "/app/node_modules/p/throws.js": "throw new Error('boom');", "/app/node_modules/p/bare.js": "require('lodash');", + "/app/node_modules/p/dialects/mysql.js": [ + "exports.driver = require('driver').name;", + "exports.sub = require('driver/extra').name;", + "exports.scoped = require('@s/conds');", + "exports.mainOnly = require('mainonly');", + "exports.local = require('nested');", + ].join("\n"), + "/app/node_modules/driver/package.json": JSON.stringify({ + exports: { + ".": { import: "./esm.mjs", require: "./cjs.js" }, + "./extra": "./extra.js", + }, + }), + "/app/node_modules/driver/cjs.js": "exports.name = 'driver';", + "/app/node_modules/driver/extra.js": "exports.name = 'extra';", + "/app/node_modules/@s/conds/package.json": JSON.stringify({ + exports: [{ browser: "./browser.js", default: "./node.js" }], + }), + "/app/node_modules/@s/conds/node.js": "module.exports = 'conds';", + "/app/node_modules/mainonly/package.json": JSON.stringify({ main: "lib/entry" }), + "/app/node_modules/mainonly/lib/entry.js": "module.exports = 'main';", + "/app/node_modules/p/node_modules/nested/index.js": "module.exports = 'nested';", + "/app/node_modules/p/sealed.js": "require('driver/hidden');", "/app/node_modules/p/split.js": "require.ensure(['./base'], function (r) { exports.base = r('./base').name; });", }; @@ -487,6 +528,23 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(require("/app/node_modules/p/split.js"), { base: "base" }); }); + it("looks a package up from a loaded module, as Node's require() does", () => { + assertEquals(require("/app/node_modules/p/dialects/mysql.js"), { + driver: "driver", + sub: "extra", + scoped: "conds", + mainOnly: "main", + local: "nested", + }); + for (const file of ["bare", "sealed"]) { + assertEquals( + errorCode(() => require(`/app/node_modules/p/${file}.js`)), + "MODULE_NOT_FOUND", + file, + ); + } + }); + it("hands a cycle the exports so far, as Node does", () => { assertEquals(require("/app/node_modules/p/cycle-a.js"), { early: 1, b: 1, late: 2 }); }); @@ -617,6 +675,16 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(js?.includes('exports.d = "/d/p";'), String(js)); }); + it("prepares a .jsx file as Node's .js loader would", async () => { + const code = await prepareCommonJSFile( + "exports.d = __dirname;", + "/d/p/view.jsx", + undefined, + resolveRelative, + ); + assert(code?.includes('exports.d = "/d/p";'), String(code)); + }); + it("prepares a .cts file as CommonJS TypeScript", async () => { const code = await prepareCommonJSFile( "const d: string = __dirname;\nexports.d = d;", diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index e147bdb5d6..e34cd42e61 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -232,10 +232,18 @@ export type ReadManifest = (path: string) => Promise; */ export class PackageTypeLookup { readonly #byDirectory = new Map>(); - readonly #root: string; - - constructor(private readonly readManifest: ReadManifest, projectDir: string) { - this.#root = pathHelper.resolve(projectDir); + readonly #roots: string[]; + + /** + * `projectDirs` are the project root as given and as canonical: the bundler + * reports paths with symlinks followed. + */ + constructor( + private readonly readManifest: ReadManifest, + projectDirs: string | readonly string[], + ) { + this.#roots = (typeof projectDirs === "string" ? [projectDirs] : [...projectDirs]) + .map((directory) => pathHelper.resolve(directory)); } of(directory: string): Promise { @@ -260,7 +268,9 @@ export class PackageTypeLookup { } } const parent = pathHelper.dirname(directory); - if (parent === directory || !isWithinDirectory(this.#root, parent)) return undefined; + if (parent === directory || !this.#roots.some((root) => isWithinDirectory(root, parent))) { + return undefined; + } return await this.of(parent); } } @@ -293,6 +303,8 @@ async function possibleLinks(fs: DependencyRootFileSystem, modules: string): Pro const links: string[] = []; for (const entry of await directoryEntries(fs, modules)) { const path = pathHelper.join(modules, entry.name); + // A linked `@scope` directory holds plain package directories. + if (entry.name.startsWith("@") && entry.isSymlink !== false) links.push(path); const scoped = entry.name.startsWith("@") ? (await directoryEntries(fs, path)).map((inner) => ({ ...inner, @@ -400,9 +412,9 @@ export type ResolveRequireLiteral = (specifier: string) => Promise { const cjsExtension = /\.c[jt]s$/i.test(filePath); - const jsExtension = /\.js$/i.test(filePath); + // Node's require() loads `.jsx` through its `.js` loader too. + const jsExtension = /\.jsx?$/i.test(filePath); // Node's require() loads a file without an extension as CommonJS, whatever // its package declares; a package `main` may name one. const extensionless = pathHelper.extname(filePath) === ""; @@ -703,12 +716,16 @@ export interface RuntimeRequire { * - A built-in loads under any id Node accepts. * - An absolute path, or a relative one inside a module this loaded, finds a * file in Node's order: the path, then with `.js` or `.json`, then its - * `index.js` or `index.json`. Its real path must be inside the project. + * `index.js` or `index.json`. A package name inside a module this loaded + * is looked up in `node_modules` from that module upwards, through the + * package's `exports` (`node`, `require` and `default` conditions) or + * `main`. Its real path must be inside the project. * JSON loads anywhere in the project; JavaScript only from the project's * dependencies, as CommonJS, with its own `require`. Each loads once, as in * Node's module cache. `require.resolve` returns the path it found. - * - Anything else, a package name included, throws `MODULE_NOT_FOUND`, so - * optional dependencies probed in a `try` still fall back. + * - Anything else, a package name in route code included, throws + * `MODULE_NOT_FOUND`, so optional dependencies probed in a `try` still + * fall back. * - `require.ensure`, which code written for webpack calls, runs its callback. * * Errors name a path relative to the project, never the host path. @@ -755,12 +772,8 @@ export function createRuntimeRequire( }; const relativeId = (id: string) => id === "." || id === ".." || id.startsWith("./") || id.startsWith("../"); - const find = (id: unknown, directory: string | undefined) => { - if (typeof id !== "string") return null; - let base: string; - if (host.isAbsolute(id)) base = id; - else if (directory !== undefined && relativeId(id)) base = host.join(directory, id); - else return null; + // A file at `base` in Node's order: exact, `.js`, `.json`, then the index. + const findFile = (base: string) => { const candidates = [ base, base + ".js", @@ -774,6 +787,91 @@ export function createRuntimeRequire( } return null; }; + // The target a package's `exports` gives `require()`: the `node`, `require` + // and `default` conditions, in the order the package lists them. + const exportTarget = (value: unknown): string | null => { + if (typeof value === "string") return value; + if (Array.isArray(value)) { + for (const item of value) { + const target = exportTarget(item); + if (target !== null) return target; + } + return null; + } + if (value === null || typeof value !== "object") return null; + for (const [condition, item] of Object.entries(value)) { + if (condition !== "node" && condition !== "require" && condition !== "default") continue; + const target = exportTarget(item); + if (target !== null) return target; + } + return null; + }; + // A package's file for `subpath` ("" or "/sub"), as Node's require() picks it. + const packageFile = (packageDir: string, subpath: string) => { + const manifestPath = host.join(packageDir, "package.json"); + let manifest: { exports?: unknown; main?: unknown } = {}; + if (host.isFile(manifestPath)) { + try { + manifest = JSON.parse(host.readFileSync(manifestPath, "utf8")); + } catch { + return null; // Node rejects a malformed package.json + } + } + const exports = manifest.exports; + if (exports !== undefined && exports !== null) { + const key = "." + subpath; + const keyed = typeof exports === "object" && !Array.isArray(exports) && + Object.keys(exports).some((name) => name.startsWith(".")); + const entry = keyed + ? (exports as Record)[key] + : key === "." + ? exports + : null; + const target = exportTarget(entry); + return target === null ? null : projectFile(host.join(packageDir, target)); + } + if (subpath !== "") return findFile(host.join(packageDir, subpath)); + if (typeof manifest.main === "string") { + const main = findFile(host.join(packageDir, manifest.main)); + if (main !== null) return main; + } + return findFile(packageDir); + }; + // A package name looked up in `node_modules` from `directory` upwards, as + // Node's require() does, never above the project. + const findPackage = (id: string, directory: string) => { + const parts = id.split("/"); + const nameLength = id.startsWith("@") ? 2 : 1; + if ( + parts.length < nameLength || + parts.some((part) => part === "" || part === "." || part === "..") + ) { + return null; + } + const name = parts.slice(0, nameLength).join("/"); + const subpath = parts.length > nameLength ? "/" + parts.slice(nameLength).join("/") : ""; + let current = directory; + for (;;) { + const relative = host.relative(projectRoot, current); + if (relative === ".." || relative.startsWith(".." + host.sep) || host.isAbsolute(relative)) { + return null; + } + if (!current.endsWith(host.sep + "node_modules")) { + const found = packageFile(host.join(current, "node_modules", name), subpath); + if (found !== null) return found; + } + const parent = host.dirname(current); + if (parent === current) return null; + current = parent; + } + }; + const find = (id: unknown, directory: string | undefined) => { + if (typeof id !== "string") return null; + if (host.isAbsolute(id)) return findFile(id); + if (directory === undefined) return null; + if (relativeId(id)) return findFile(host.join(directory, id)); + return findPackage(id, directory); + }; const label = (id: unknown) => typeof id === "string" && (host.isAbsolute(id) || relativeId(id)) ? "a path outside this project or missing" diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 6d959174db..08d1deb429 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -1923,8 +1923,12 @@ function createProjectBoundaryPlugin( allowedHosts: string[], workerImportMap: DenoImportMap | null, isDependencyFile: DependencyFilePredicate = noDependencyFiles, + canonicalProjectDir: string = projectDir, ): Plugin { - const packageTypes = new PackageTypeLookup(snapshotManifestReader(sourceSnapshot), projectDir); + const packageTypes = new PackageTypeLookup(snapshotManifestReader(sourceSnapshot), [ + projectDir, + canonicalProjectDir, + ]); return { name: "vf-project-boundary", setup(build) { @@ -2185,6 +2189,7 @@ function buildTranspiledModuleSource( 'import { createRequire as __vf_createRequire } from "node:module";', `var require = __vf_createRequire(${safeProjectDir});`, ].join("\n"); + const canonicalRoot = compiled ? await canonicalProjectRoot(projectDir) : projectDir; const dependencyDirs = compiled ? await dependencyDirectories(projectDir, { realPath, @@ -2301,6 +2306,7 @@ function buildTranspiledModuleSource( allowedHosts, workerImportMap, isDependencyFile, + canonicalRoot, ), ], // Only the opt-in decorator transform needs a working directory: adding @@ -2319,9 +2325,7 @@ function buildTranspiledModuleSource( logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); const output = result.outputFiles?.[0]?.text ?? "export {}"; - const js = compiled - ? prependRuntimeRequire(output, await canonicalProjectRoot(projectDir), dependencyDirs) - : output; + const js = compiled ? prependRuntimeRequire(output, canonicalRoot, dependencyDirs) : output; logger.debug(`transpiled size ${js.length} bytes`); // Only dependencies left external are rewritten. The compiled bundle diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index adaa449fcd..951470b630 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -371,8 +371,18 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { ' return require(path.join(__dirname, "dialects", name + ".js"));', "};", ].join("\n"), - "node_modules/dialects/dialects/pg.js": - 'var base = require("../base");\nexports.name = base.prefix + "pg";\nexports.sep = require("path").sep;', + "node_modules/dialects/dialects/pg.js": [ + 'var base = require("../base");', + 'exports.name = base.prefix + "pg";', + 'exports.sep = require("path").sep;', + // Never seen by the bundler: only this computed load reaches it. + 'exports.driver = require("driver");', + ].join("\n"), + "node_modules/driver/package.json": JSON.stringify({ + name: "driver", + exports: { ".": { import: "./index.mjs", require: "./index.cjs" } }, + }), + "node_modules/driver/index.cjs": 'module.exports = "driver";', "node_modules/dialects/base.js": 'exports.prefix = "dialect:";', "src/secret.js": 'module.exports = "project code";', }); @@ -385,7 +395,7 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { 'try { dialects.load("../../../src/secret"); } catch (error) { project = (error as { code?: string }).code; }\n' + "export { project };\n", ); - assertEquals(mod.pg, { name: "dialect:pg", sep: "/" }); + assertEquals(mod.pg, { name: "dialect:pg", sep: "/", driver: "driver" }); assertEquals(mod.again, true); assertEquals(mod.project, "MODULE_NOT_FOUND"); }); From 48640eea474d4495934a5bd7886f850d9ddd260f Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 00:57:31 +0100 Subject: [PATCH 17/29] fix(api): share the CommonJS module cache and match Node's resolution in compiled routes - Bundled CommonJS files and dependency JSON register their module with the runtime require, so a computed require of the same file returns it; require.cache exposes the cache and a deleted entry reloads. - Runtime require: exports/imports patterns, #imports, the paths option of require.resolve, Node's module object, a directory's package.json main, .node in the extension order, a JSON byte order mark, and Node's error codes for native addons and ES modules. The root require resolves from the project. - A linked workspace package's package imports resolve from node_modules, past the project's tsconfig paths. - An ESM dependency's import.meta.url keeps its import query or fragment. --- .../compiled-cjs-bundling.test.ts | 315 ++++++++++++- .../module-loader/compiled-cjs-bundling.ts | 416 ++++++++++++++---- src/routing/api/module-loader/loader.ts | 11 + .../compiled-cjs-bundling.test.ts | 77 +++- 4 files changed, 716 insertions(+), 103 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index d7c6d8dd77..980e19862e 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,5 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assert, assertEquals } from "#veryfront/testing/assert.ts"; +import { assert, assertEquals, assertThrows } from "#veryfront/testing/assert.ts"; import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; import { type PluginBuild, stop as stopBundler } from "veryfront/extensions/bundler"; import { @@ -16,8 +16,10 @@ import { PackageTypeLookup, prepareCommonJSFile, prepareDependencyFile, + prepareDependencyJson, prepareModuleFile, prependRuntimeRequire, + registerBundledModule, resolveLiteralCalls, rewriteImportSpecifiers, type RuntimeRequireHost, @@ -269,16 +271,100 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); it("resolves a dependency's relative import in Node's order", async () => { - const [handler] = await resolveHandlers(createDependencyResolvePlugin(isDependency, isFile)); + const [, handler] = await resolveHandlers( + createDependencyResolvePlugin(isDependency, isFile), + ); assertEquals(await handler!(args("./impl")), { path: "/d/node_modules/p/impl.js" }); assertEquals(await handler!(args("./missing")), undefined); assertEquals(await handler!(args("./impl", "/d/pages/api/route.ts")), undefined); }); it("leaves a native addon found by Node's extension order to the loader", async () => { - const [handler] = await resolveHandlers(createDependencyResolvePlugin(isDependency, isFile)); + const [, handler] = await resolveHandlers( + createDependencyResolvePlugin(isDependency, isFile), + ); assertEquals(await handler!(args("./addon")), { path: "/d/node_modules/p/addon.node" }); }); + + it("resolves a linked workspace package's package imports from its node_modules", async () => { + const calls: unknown[] = []; + const build = { + onResolve: (options: { filter: RegExp }, handler: ResolveHandler) => { + filters.push(options.filter); + registered.push(handler); + }, + onLoad: () => {}, + resolve: (path: string, options: Record) => { + calls.push({ path, ...options }); + // Only /d/node_modules exists, as the bundler needs. + return Promise.resolve( + path === "missing" || options.resolveDir !== "/d/node_modules" + ? { errors: [{ text: "Could not resolve" }] } + : { path: "/d/node_modules/pg/index.js", errors: [], suffix: "?q" }, + ); + }, + }; + const registered: ResolveHandler[] = []; + const filters: RegExp[] = []; + const workspace = (path: string) => path.startsWith("/d/packages/") || isDependency(path); + await createDependencyResolvePlugin(workspace, isFile).setup( + build as unknown as PluginBuild, + ); + const [bare] = registered; + // A package's own `#import` resolves from its package.json, as before. + assertEquals(["pg", "@s/p", "#own", "./x", "/x"].map((id) => filters[0]!.test(id)), [ + true, + true, + false, + false, + false, + ]); + const result = await bare!({ ...args("pg", "/d/packages/w/index.js") }); + assertEquals((result as { path: string }).path, "/d/node_modules/pg/index.js"); + assertEquals((result as { suffix: string }).suffix, "?q"); + const tried = (path: string, ...dirs: string[]) => + dirs.map((resolveDir) => ({ + path, + kind: "require-call", + importer: "/d/packages/w/index.js", + resolveDir, + pluginData: { vfFromNodeModules: true }, + })); + assertEquals( + calls, + tried("pg", "/d/packages/w/node_modules", "/d/packages/node_modules", "/d/node_modules"), + ); + calls.length = 0; + assertEquals(await bare!({ ...args("missing", "/d/packages/w/index.js") }), { + errors: [{ text: "Could not resolve" }], + }); + // Its own second resolution, files in node_modules, project code and + // built-ins resolve as before. + assertEquals( + await bare!({ + ...args("pg", "/d/packages/w/index.js"), + pluginData: { vfFromNodeModules: true }, + }), + undefined, + ); + assertEquals(await bare!(args("pg")), undefined); + assertEquals(await bare!(args("pg", "/d/pages/api/route.ts")), undefined); + assertEquals(await bare!(args("fs", "/d/packages/w/index.js")), undefined); + assertEquals( + await bare!(args("data:text/javascript,1", "/d/packages/w/index.js")), + undefined, + ); + assertEquals( + calls, + tried( + "missing", + "/d/packages/w/node_modules", + "/d/packages/node_modules", + "/d/node_modules", + "/node_modules", + ), + ); + }); }); describe("createImportExternalsPlugin()", () => { @@ -311,9 +397,56 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { isFile: () => Promise.resolve(false), }); - it("returns a file that reads no CommonJS names and no import.meta unchanged", async () => { + it("registers a file that reads no CommonJS names and no import.meta, unparsed", async () => { const contents = 'module.exports = require("./other");'; - assertEquals(await prepareDependencyFile(input(contents, "/d/a.js")), contents); + assertEquals( + await prepareDependencyFile(input(contents, "/d/a.js")), + registerBundledModule(contents, "/d/a.js"), + ); + assertEquals( + await prepareDependencyFile(input(contents, "/d/bin/tool")), + registerBundledModule(contents, "/d/bin/tool"), + ); + assertEquals( + await prepareDependencyFile(input(contents, "/d/a.cjs", "module")), + registerBundledModule(contents, "/d/a.cjs"), + ); + // An ES module has no `module` to register. + assertEquals(await prepareDependencyFile(input(contents, "/d/a.js", "module")), contents); + assertEquals(await prepareDependencyFile(input(contents, "/d/a.mjs")), contents); + }); + + it("registers a module object by its path after the module's own code", () => { + const code = registerBundledModule("module.exports = 1;", '/d/"q".js'); + const registered: unknown[] = []; + new Function("module", "__vf_register_bundled_module", code)( + { exports: 2 }, + (path: string, module: unknown) => registered.push([path, module]), + ); + assertEquals(registered, [['/d/"q".js', { exports: 1 }]]); + // Outside a CommonJS module, or without the registry, it does nothing. + const bare = registerBundledModule("", "/d/esm.js"); + new Function("module", "__vf_register_bundled_module", bare)(undefined, () => { + throw new Error("registered"); + }); + new Function("module", bare)(null); + new Function("module", code)({ exports: 0 }); + }); + + it("turns dependency JSON into a module that registers itself and rejects bad JSON", () => { + const code = prepareDependencyJson('{"a":[1]}', "/d/data.json"); + const registered: Array<{ exports: unknown }> = []; + const module = { exports: {} }; + new Function("module", "__vf_register_bundled_module", code)( + module, + (_path: string, value: { exports: unknown }) => registered.push(value), + ); + assertEquals(module.exports, { a: [1] }); + assert(registered[0] === module); + assertThrows(() => prepareDependencyJson("{", "/d/bad.json"), SyntaxError); + const bom = { exports: {} }; + new Function("module", prepareDependencyJson('\uFEFF{"b":2}', "/d/bom.json"))(bom); + assertEquals(bom.exports, { b: 2 }); }); it("leaves files that are neither JavaScript nor CommonJS alone", async () => { @@ -406,7 +539,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); describe("createRuntimeRequire()", () => { - const builtins: Record = { fs: { kind: "fs" }, "node:fs": { kind: "fs" } }; + const builtins: Record = { + fs: { kind: "fs" }, + "node:fs": { kind: "fs" }, + "vf-state": { loads: 0 }, + }; const files: Record = { "/app/data.json": '{"a":1}', "/app/link.json": "", // a link to /outside.json @@ -449,6 +586,63 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/app/node_modules/p/sealed.js": "require('driver/hidden');", "/app/node_modules/p/split.js": "require.ensure(['./base'], function (r) { exports.base = r('./base').name; });", + "/app/node_modules/p/meta.js": [ + "exports.id = module.id;", + "exports.filename = module.filename;", + "exports.path = module.path;", + "exports.loadedDuring = module.loaded;", + "exports.ownRequire = module.require === require;", + "exports.cached = require.cache[__filename] === module;", + ].join("\n"), + "/app/node_modules/p/counter.js": + "var state = require('vf-state'); state.loads++; module.exports = state.loads;", + "/app/node_modules/p/addon.node": "", + "/app/node_modules/p/esm.mjs": "export {};", + "/app/node_modules/p/patterns.js": [ + "exports.feature = require('patterned/feature/a');", + "exports.longest = require('patterned/feature/deep/b');", + "exports.exact = require('patterned/exact');", + "exports.own = require('#own');", + "exports.ownPattern = require('#lib/x');", + "exports.ownPackage = require('#dep');", + ].join("\n"), + "/app/node_modules/p/package.json": JSON.stringify({ + imports: { + "#own": "./base.js", + "#lib/*": "./lib/*.js", + "#dep": "driver/extra", + "#outside": "../escape.js", + }, + }), + "/app/node_modules/p/lib/x.js": "module.exports = 'lib x';", + "/app/node_modules/p/no-import.js": "require('#missing');", + "/app/node_modules/p/escape-import.js": "require('#outside');", + "/app/node_modules/patterned/package.json": JSON.stringify({ + exports: { + "./feature/*": "./features/*.js", + "./feature/deep/*": { node: "./deep/*.js" }, + "./exact": "./exact.js", + "./*.js": null, + }, + }), + "/app/node_modules/patterned/features/a.js": "module.exports = 'feature a';", + "/app/node_modules/patterned/deep/b.js": "module.exports = 'deep b';", + "/app/node_modules/patterned/exact.js": "module.exports = 'exact';", + "/app/node_modules/p/broken-pkg.js": "require('broken');", + "/app/node_modules/broken/package.json": "{", + "/app/node_modules/broken/index.js": "module.exports = 'broken';", + "/app/node_modules/p/outside-pattern.js": "require('patterned/feature/../../escape');", + "/app/node_modules/p/bom.json": '\uFEFF{"bom":true}', + "/app/node_modules/p/plugin/package.json": JSON.stringify({ main: "lib/main" }), + "/app/node_modules/p/plugin/lib/main.js": "module.exports = 'plugin main';", + "/app/node_modules/p/plugin/index.js": "module.exports = 'plugin index';", + "/app/node_modules/p/dirmain/package.json": JSON.stringify({ main: "lib" }), + "/app/node_modules/p/dirmain/lib/index.js": "module.exports = 'main dir index';", + "/app/node_modules/p/badmain/package.json": JSON.stringify({ main: "missing" }), + "/app/node_modules/p/badmain/index.js": "module.exports = 'fallback index';", + "/app/node_modules/p/binding.node": "", + "/app/node_modules/sibling.js": "module.exports = 'file beside the package';", + "/app/node_modules/sibling/index.js": "module.exports = 'package';", }; let reads = 0; const host: RuntimeRequireHost = { @@ -485,7 +679,12 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { dirname: (path) => path.slice(0, path.lastIndexOf("/")), sep: "/", }; - const require = createRuntimeRequire(host, "/app", ["test"], ["/app/node_modules"]); + const { require, register } = createRuntimeRequire( + host, + "/app", + ["test"], + ["/app/node_modules"], + ); it("serves any built-in, including a computed id", () => { assertEquals(require("fs"), { kind: "fs" }); @@ -565,7 +764,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { for ( const id of [ "left-pad", - "./data.json", + "./missing.json", 42, "/app/code.js", "/app/link.json", @@ -578,6 +777,102 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(errorCode(() => require.resolve("/app/missing.js")), "MODULE_NOT_FOUND"); }); + it("resolves a relative id from the project root", () => { + assert(require("./data.json") === require("/app/data.json")); + assertEquals(require.resolve("./data"), "/app/data.json"); + assertEquals(require.resolve("driver"), "/app/node_modules/driver/cjs.js"); + }); + + it("gives each module Node's module object and shares one cache", () => { + const meta = require("/app/node_modules/p/meta.js"); + assertEquals(meta, { + id: "/app/node_modules/p/meta.js", + filename: "/app/node_modules/p/meta.js", + path: "/app/node_modules/p", + loadedDuring: false, + ownRequire: true, + cached: true, + }); + const cached = require.cache["/app/node_modules/p/meta.js"] as unknown as { loaded: boolean }; + assertEquals(cached.loaded, true); + }); + + it("reloads a module deleted from require.cache", () => { + const first = require("/app/node_modules/p/counter.js"); + assertEquals(require("/app/node_modules/p/counter.js"), first); + delete require.cache["/app/node_modules/p/counter.js"]; + assertEquals(require("/app/node_modules/p/counter.js"), (first as number) + 1); + }); + + it("returns a module the bundle registered, so both loads share it", () => { + const bundled = { exports: { bundled: true } }; + register("/app/node_modules/p/registered.js", bundled); + register("/app/node_modules/p/registered.js", { exports: { second: true } }); + // A path through a link is keyed by its real path. + const linked = { exports: { linked: true } }; + register("/app/link.json", linked); + assert(require.cache["/outside.json"] === linked); + delete require.cache["/outside.json"]; + files["/app/node_modules/p/registered.js"] = "module.exports = { evaluated: true };"; + assert(require("/app/node_modules/p/" + "registered.js") === bundled.exports); + }); + + it("takes Node's paths option in require.resolve", () => { + assertEquals( + require.resolve("driver", { paths: ["/app/node_modules/p", 7, "relative"] }), + "/app/node_modules/driver/cjs.js", + ); + assertEquals( + require.resolve("nested", { paths: ["/app/node_modules/p"] }), + "/app/node_modules/p/node_modules/nested/index.js", + ); + assertEquals( + errorCode(() => require.resolve("nested", { paths: ["/app"] })), + "MODULE_NOT_FOUND", + ); + assertEquals( + require.resolve("./base", { paths: ["/app/node_modules/p"] }), + "/app/node_modules/p/base.js", + ); + }); + + it("loads a directory and an extension as Node does", () => { + assertEquals(require("/app/node_modules/p/bom.json"), { bom: true }); + assertEquals(require("/app/node_modules/p/" + "plugin"), "plugin main"); + assertEquals(require("/app/node_modules/p/dirmain"), "main dir index"); + assertEquals(require("/app/node_modules/p/badmain"), "fallback index"); + assertEquals(errorCode(() => require("/app/node_modules/p/binding")), "ERR_DLOPEN_FAILED"); + assertEquals( + require.resolve("/app/node_modules/p/binding"), + "/app/node_modules/p/binding.node", + ); + // A file named like the package comes first, as in Node. + assertEquals(require("sibling"), "file beside the package"); + }); + + it("rejects a native addon and an ES module with Node's codes", () => { + assertEquals(errorCode(() => require("/app/node_modules/p/addon.node")), "ERR_DLOPEN_FAILED"); + assertEquals(errorCode(() => require("/app/node_modules/p/esm.mjs")), "ERR_REQUIRE_ESM"); + }); + + it("matches exports patterns and the package's own imports", () => { + assertEquals(require("/app/node_modules/p/patterns.js"), { + feature: "feature a", + longest: "deep b", + exact: "exact", + own: { name: "base" }, + ownPattern: "lib x", + ownPackage: { name: "extra" }, + }); + for (const file of ["no-import", "escape-import", "broken-pkg", "outside-pattern"]) { + assertEquals( + errorCode(() => require(`/app/node_modules/p/${file}.js`)), + "MODULE_NOT_FOUND", + file, + ); + } + }); + it("names a project path relative to the project and hides any other", () => { const message = (id: string) => { try { @@ -605,6 +900,10 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { ), code, ); + assert( + code.includes("const { require, register: __vf_register_bundled_module } = ("), + code, + ); assert(code.endsWith("\nexport {};\n"), code); }); diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index e34cd42e61..db8e2fd286 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -146,6 +146,13 @@ export async function nodePreferredFile( } const RELATIVE_SPECIFIER = /^(?:\.\.?(?:\/|$)|\/)/; +// A package name: not a path and not a package's own `#import`, which +// resolves from the importer's package. A URL is told apart in the handler +// (the bundler's filters are Go regular expressions, without lookahead). +const BARE_SPECIFIER = /^[^./#]/; +const URL_SPECIFIER = /^[a-z][a-z0-9+.-]*:/i; +const IN_NODE_MODULES = /[\\/]node_modules[\\/]/; +const FROM_NODE_MODULES = "vfFromNodeModules"; /** * The file Node's require() loads for a relative or absolute extensionless @@ -174,10 +181,11 @@ export async function nodeRelativeFile( } /** - * Resolves the relative imports of dependency files the way Node does: the - * bundler's extension order serves project code and would load a TypeScript - * sibling first. Only relative imports differ, so only they are probed, with a - * few file checks and no second resolution. Must run after + * Resolves the imports of dependency files the way Node does. The bundler's + * extension order serves project code and would load a TypeScript sibling + * first, so relative imports are probed in Node's order, with a few file + * checks and no second resolution. Package imports of a linked workspace + * package skip the project's tsconfig `paths`. Must run after * {@link createCompiledBuiltinPlugin} and before the project's own resolvers. */ export function createDependencyResolvePlugin( @@ -187,6 +195,45 @@ export function createDependencyResolvePlugin( return { name: "vf-dependency-resolve", setup(build) { + // A workspace package linked from node_modules lives outside it, where + // the bundler would apply the project's tsconfig `paths` to its package + // imports. Resolving from its own node_modules directory instead looks + // packages up as Node does, and the bundler applies no tsconfig there. + build.onResolve({ filter: BARE_SPECIFIER }, async (args) => { + const data = args.pluginData as Record | undefined; + if (data?.[FROM_NODE_MODULES] === true || !build.resolve) return undefined; + if (!args.importer || !isDependencyFile(args.importer)) return undefined; + if ( + IN_NODE_MODULES.test(args.importer) || URL_SPECIFIER.test(args.path) || + builtinModuleName(args.path) !== undefined + ) { + return undefined; + } + // The bundler needs a directory that exists; from the nearest + // `node_modules` it looks further up itself, as Node does. + let result: Awaited>> | undefined; + let directory = args.resolveDir; + for (;;) { + result = await build.resolve(args.path, { + kind: args.kind, + importer: args.importer, + resolveDir: pathHelper.join(directory, "node_modules"), + pluginData: { [FROM_NODE_MODULES]: true }, + }); + if (!result.errors?.length) break; + const parent = pathHelper.dirname(directory); + if (parent === directory) return { errors: result.errors }; + directory = parent; + } + return { + path: result.path, + external: result.external, + namespace: result.namespace, + suffix: result.suffix, + sideEffects: result.sideEffects, + pluginData: result.pluginData, + }; + }); build.onResolve({ filter: RELATIVE_SPECIFIER }, async (args) => { if (!args.importer || !isDependencyFile(args.importer)) return undefined; const file = await nodeRelativeFile(args.path, args.resolveDir, isFile); @@ -515,8 +562,10 @@ export async function prepareModuleFile( contents: string, filePath: string, resolve: ResolveImportMetaLiteral, + suffix = "", ): Promise { - const url = pathHelper.toFileUrl(filePath).href; + // The module's URL keeps the query or fragment it was imported with. + const url = pathHelper.toFileUrl(filePath).href + suffix; const marker = await markerFor("__vf_import_meta_resolve_", filePath, contents); const result = await bundle({ stdin: { @@ -588,6 +637,37 @@ export function createImportMetaResolve( }; } +/** + * The name a bundled dependency module registers itself under, so that the + * runtime `require` shares one module cache with the bundle. The route + * module's prologue binds it; see {@link prependRuntimeRequire}. + */ +export const BUNDLED_MODULE_REGISTRY = "__vf_register_bundled_module"; + +/** + * `code` with a last statement that registers its CommonJS `module` by + * `path`. In an ES module `module` is unbound and nothing happens. A file + * that leaves early with a top-level `return` is not registered, and a + * computed `require()` of it runs it again. + */ +export function registerBundledModule(code: string, path: string): string { + const registry = BUNDLED_MODULE_REGISTRY; + return `${code}\n;typeof module === "object" && module !== null && ` + + `typeof ${registry} === "function" && ${registry}(${JSON.stringify(path)}, module);\n`; +} + +/** + * A dependency's JSON file as a CommonJS module that registers itself, so a + * computed `require()` of the same file returns the same object, as in Node. + * Malformed JSON fails the build, as the bundler's JSON loader would. + */ +export function prepareDependencyJson(contents: string, path: string): string { + // Node drops a byte order mark before parsing JSON. + const json = contents.charCodeAt(0) === 0xfeff ? contents.slice(1) : contents; + JSON.parse(json); + return registerBundledModule(`module.exports = JSON.parse(${JSON.stringify(json)});`, path); +} + /** Input for {@link prepareDependencyFile}. */ export interface DependencyFileInput { readonly build: PluginBuild; @@ -600,6 +680,8 @@ export interface DependencyFileInput { readonly packageType: string | undefined; readonly isDependencyFile: (path: string) => boolean; readonly isFile: IsFile; + /** The query or fragment the file was imported with, as the bundler reports it. */ + readonly suffix?: string; } /** @@ -610,13 +692,19 @@ export interface DependencyFileInput { */ export async function prepareDependencyFile(input: DependencyFileInput): Promise { const { build, contents, logicalPath, resolvedPath, isDependencyFile } = input; + // A file Node may load as CommonJS registers its module for the runtime + // `require`; an ES module has no `module` to register. + const mayBeCommonJS = /\.c[jt]s$/i.test(logicalPath) || + (/(?:\.jsx?|[\\/][^.\\/]+)$/i.test(logicalPath) && input.packageType !== "module"); + const register = (code: string) => + mayBeCommonJS ? registerBundledModule(code, resolvedPath) : code; // Most package files read none of these names; parsing them would change // nothing. The check is textual, so it can only send extra files to the parser. const importMeta = /\bimport\.meta\b/.test(contents); const readsCommonJSNames = contents.includes("__dirname") || contents.includes("__filename") || (contents.includes("require") && contents.includes("resolve")); - if (!readsCommonJSNames && !importMeta) return contents; + if (!readsCommonJSNames && !importMeta) return register(contents); const resolveDir = pathHelper.dirname(resolvedPath); const dependencyPath = async ( specifier: string, @@ -634,17 +722,21 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise input.packageType, (specifier) => dependencyPath(specifier, "require-resolve"), ); - if (commonJS !== null) return commonJS; + if (commonJS !== null) return registerBundledModule(commonJS, resolvedPath); if (!importMeta) return contents; + const moduleUrl = pathHelper.toFileUrl(logicalPath).href + (input.suffix ?? ""); return await prepareModuleFile(contents, logicalPath, async (specifier) => { const builtin = builtinModuleName(specifier); if (builtin !== undefined) return "node:" + builtin; - if (/^[a-z][a-z0-9+.-]*:/i.test(specifier) || RELATIVE_SPECIFIER.test(specifier)) { - return new URL(specifier, pathHelper.toFileUrl(logicalPath)).href; + if ( + URL_SPECIFIER.test(specifier) || RELATIVE_SPECIFIER.test(specifier) || + specifier.startsWith("?") || specifier.startsWith("#") + ) { + return new URL(specifier, moduleUrl).href; } const path = await dependencyPath(specifier, "import-statement"); return path === null ? null : pathHelper.toFileUrl(path).href; - }); + }, input.suffix ?? ""); } /** @@ -696,37 +788,67 @@ export interface RuntimeRequireHost { export type CommonJSModuleBody = ( exports: unknown, require: RuntimeRequire, - module: { exports: unknown }, + module: RuntimeModule, filename: string, dirname: string, ) => void; +/** The `module` a CommonJS file the runtime `require` loads sees. */ +export interface RuntimeModule { + exports: unknown; + id: string; + filename: string; + path: string; + loaded: boolean; + children: unknown[]; + require: RuntimeRequire; +} + +/** {@link createRuntimeRequire}'s result. */ +export interface RuntimeRequireBinding { + readonly require: RuntimeRequire; + /** + * Records a module the bundler bundled, by real path, so that a computed + * `require()` of the same file returns it instead of running it again. + */ + readonly register: (path: string, module: { exports: unknown }) => void; +} + /** The runtime `require` of a bundled route; see {@link createRuntimeRequire}. */ export interface RuntimeRequire { (id: unknown): unknown; - resolve(id: unknown): string; + /** Node's `require.resolve`, including its `paths` option. */ + resolve(id: unknown, options?: { paths?: unknown }): string; + /** Loaded modules by real path; deleting one makes the next load run it again. */ + cache: Record; /** webpack's `require.ensure`: everything is loaded already. */ ensure(ids: unknown, callback: (require: RuntimeRequire) => void): void; } /** * The `require` that esbuild's output calls for anything it did not bundle: - * a computed id, or a built-in. + * a computed id, or a built-in. Ids resolve as Node's `require()` resolves + * them, from the project for route code and from its own directory for a + * module this loaded: * * - A built-in loads under any id Node accepts. - * - An absolute path, or a relative one inside a module this loaded, finds a - * file in Node's order: the path, then with `.js` or `.json`, then its - * `index.js` or `index.json`. A package name inside a module this loaded - * is looked up in `node_modules` from that module upwards, through the - * package's `exports` (`node`, `require` and `default` conditions) or - * `main`. Its real path must be inside the project. - * JSON loads anywhere in the project; JavaScript only from the project's - * dependencies, as CommonJS, with its own `require`. Each loads once, as in - * Node's module cache. `require.resolve` returns the path it found. - * - Anything else, a package name in route code included, throws - * `MODULE_NOT_FOUND`, so optional dependencies probed in a `try` still - * fall back. - * - `require.ensure`, which code written for webpack calls, runs its callback. + * - A path finds a file in Node's order: the path, then with `.js` or + * `.json`, then its `index.js` or `index.json`. + * - A package name is looked up in `node_modules` upwards, through the + * package's `exports` (exact keys and `*` patterns; `node`, `require` and + * `default` conditions) or its `main`. A `#name` reads the `imports` of the + * package that holds the module. + * + * Whatever it finds must have its real path inside the project, and nothing + * looks above it. JSON loads from anywhere in the project. JavaScript loads + * only from the project's dependencies, as CommonJS, with its own `require` + * and `module`. A native addon or an ES module throws. Each file loads once, + * and a module the bundle already ran is the same module; see + * {@link RuntimeRequireBinding.register}. `require.cache` holds them by real + * path, `require.resolve` returns the path it found and takes Node's `paths` + * option, and `require.ensure`, which code written for webpack calls, runs + * its callback. Anything else throws `MODULE_NOT_FOUND`, so optional + * dependencies probed in a `try` still fall back. * * Errors name a path relative to the project, never the host path. * {@link prependRuntimeRequire} embeds this function's source in the route @@ -737,25 +859,25 @@ export function createRuntimeRequire( projectRoot: string, prefixOnlyNames: readonly string[], dependencyDirectories: readonly string[], -): RuntimeRequire { +): RuntimeRequireBinding { // Tables without a prototype: `in` reads only their own keys and does not // depend on built-ins that route code could replace later. const prefixOnly: Record = Object.create(null); for (const name of prefixOnlyNames) prefixOnly[name] = true; const modules: Record = Object.create(null); + const failure = (message: string, code: string) => { + const error = new Error(message) as Error & { code?: string }; + error.code = code; + return error; + }; const dependencyPrefixes = dependencyDirectories.map((directory) => directory + host.sep); const builtin = (id: unknown) => { // A bare prefix-only name such as `test` is an npm package, as in Node. if (typeof id !== "string" || id in prefixOnly) return undefined; return host.getBuiltinModule(id); }; - const missing = (label: string) => { - const error = new Error("Cannot find module " + JSON.stringify(label)) as Error & { - code?: string; - }; - error.code = "MODULE_NOT_FOUND"; - return error; - }; + const missing = (label: string) => + failure("Cannot find module " + JSON.stringify(label), "MODULE_NOT_FOUND"); // An existing file whose real path is in the project. const projectFile = (path: string) => { if (!host.isFile(path)) return null; @@ -772,28 +894,36 @@ export function createRuntimeRequire( }; const relativeId = (id: string) => id === "." || id === ".." || id.startsWith("./") || id.startsWith("../"); - // A file at `base` in Node's order: exact, `.js`, `.json`, then the index. - const findFile = (base: string) => { - const candidates = [ - base, - base + ".js", - base + ".json", - host.join(base, "index.js"), - host.join(base, "index.json"), - ]; + const first = (candidates: string[]) => { for (const candidate of candidates) { const found = projectFile(candidate); if (found !== null) return found; } return null; }; - // The target a package's `exports` gives `require()`: the `node`, `require` - // and `default` conditions, in the order the package lists them. - const exportTarget = (value: unknown): string | null => { + const asFile = (base: string) => first([base, base + ".js", base + ".json", base + ".node"]); + const asIndex = (directory: string) => + first(["index.js", "index.json", "index.node"].map((name) => host.join(directory, name))); + // A file at `base` in Node's order: the file with Node's extensions, then + // the directory's package.json `main`, then its index. + const findFile = (base: string) => { + const file = asFile(base); + if (file !== null) return file; + const main = readManifest(base)?.main; + if (typeof main === "string") { + const target = host.join(base, main); + const found = asFile(target) ?? asIndex(target); + if (found !== null) return found; + } + return asIndex(base); + }; + // The target a package's `exports` or `imports` gives `require()`: the + // `node`, `require` and `default` conditions, in the order it lists them. + const conditionalTarget = (value: unknown): string | null => { if (typeof value === "string") return value; if (Array.isArray(value)) { for (const item of value) { - const target = exportTarget(item); + const target = conditionalTarget(item); if (target !== null) return target; } return null; @@ -801,41 +931,69 @@ export function createRuntimeRequire( if (value === null || typeof value !== "object") return null; for (const [condition, item] of Object.entries(value)) { if (condition !== "node" && condition !== "require" && condition !== "default") continue; - const target = exportTarget(item); + const target = conditionalTarget(item); if (target !== null) return target; } return null; }; - // A package's file for `subpath` ("" or "/sub"), as Node's require() picks it. - const packageFile = (packageDir: string, subpath: string) => { - const manifestPath = host.join(packageDir, "package.json"); - let manifest: { exports?: unknown; main?: unknown } = {}; - if (host.isFile(manifestPath)) { - try { - manifest = JSON.parse(host.readFileSync(manifestPath, "utf8")); - } catch { - return null; // Node rejects a malformed package.json + // A key of a package's `exports` or `imports` map, as Node matches it: an + // exact key, else the `*` pattern with the longest prefix, whose match + // replaces each `*` in the target. + const mapTarget = (map: Record, key: string): string | null => { + if (!key.includes("*") && Object.hasOwn(map, key)) return conditionalTarget(map[key]); + let best: { pattern: string; match: string } | null = null; + for (const pattern of Object.keys(map)) { + const star = pattern.indexOf("*"); + if (star === -1 || pattern.indexOf("*", star + 1) !== -1) continue; + const prefix = pattern.slice(0, star); + const suffix = pattern.slice(star + 1); + if ( + key.length < pattern.length || !key.startsWith(prefix) || !key.endsWith(suffix) || + key === prefix + ) { + continue; } + const longer = best === null || prefix.length > best.pattern.indexOf("*") || + (prefix.length === best.pattern.indexOf("*") && pattern.length > best.pattern.length); + if (longer) best = { pattern, match: key.slice(prefix.length, key.length - suffix.length) }; + } + if (best === null) return null; + const target = conditionalTarget(map[best.pattern]); + return target === null ? null : target.split("*").join(best.match); + }; + const readManifest = (packageDir: string): Record | null | undefined => { + const manifestPath = host.join(packageDir, "package.json"); + if (!host.isFile(manifestPath)) return undefined; + try { + const manifest = JSON.parse(host.readFileSync(manifestPath, "utf8")); + return manifest !== null && typeof manifest === "object" ? manifest : null; + } catch { + return null; // Node rejects a malformed package.json } - const exports = manifest.exports; + }; + const inProject = (directory: string) => { + const relative = host.relative(projectRoot, directory); + return !(relative === ".." || relative.startsWith(".." + host.sep) || + host.isAbsolute(relative)); + }; + // A package's file for `subpath` ("" or "/sub"), as Node's require() picks it. + const packageFile = (packageDir: string, subpath: string) => { + const manifest = readManifest(packageDir); + if (manifest === null) return null; + const exports = manifest?.exports; if (exports !== undefined && exports !== null) { const key = "." + subpath; const keyed = typeof exports === "object" && !Array.isArray(exports) && Object.keys(exports).some((name) => name.startsWith(".")); - const entry = keyed - ? (exports as Record)[key] + const target = keyed + ? mapTarget(exports as Record, key) : key === "." - ? exports + ? conditionalTarget(exports) : null; - const target = exportTarget(entry); - return target === null ? null : projectFile(host.join(packageDir, target)); + if (target === null || !target.startsWith("./")) return null; + return projectFile(host.join(packageDir, target)); } - if (subpath !== "") return findFile(host.join(packageDir, subpath)); - if (typeof manifest.main === "string") { - const main = findFile(host.join(packageDir, manifest.main)); - if (main !== null) return main; - } - return findFile(packageDir); + return findFile(host.join(packageDir, subpath)); }; // A package name looked up in `node_modules` from `directory` upwards, as // Node's require() does, never above the project. @@ -851,11 +1009,7 @@ export function createRuntimeRequire( const name = parts.slice(0, nameLength).join("/"); const subpath = parts.length > nameLength ? "/" + parts.slice(nameLength).join("/") : ""; let current = directory; - for (;;) { - const relative = host.relative(projectRoot, current); - if (relative === ".." || relative.startsWith(".." + host.sep) || host.isAbsolute(relative)) { - return null; - } + while (inProject(current)) { if (!current.endsWith(host.sep + "node_modules")) { const found = packageFile(host.join(current, "node_modules", name), subpath); if (found !== null) return found; @@ -864,12 +1018,35 @@ export function createRuntimeRequire( if (parent === current) return null; current = parent; } + return null; + }; + // A `#name` from the `imports` of the package that holds `directory`. + const packageImport = (id: string, directory: string) => { + let current = directory; + while (inProject(current)) { + const manifest = readManifest(current); + if (manifest !== undefined) { + const imports = manifest?.imports; + if (imports === null || typeof imports !== "object" || Array.isArray(imports)) return null; + const target = mapTarget(imports as Record, id); + if (target === null) return null; + if (target.startsWith("./")) return projectFile(host.join(current, target)); + return target.startsWith("#") || target.startsWith("/") || target.startsWith(".") + ? null + : findPackage(target, current); + } + const parent = host.dirname(current); + if (parent === current) return null; + current = parent; + } + return null; }; const find = (id: unknown, directory: string | undefined) => { if (typeof id !== "string") return null; if (host.isAbsolute(id)) return findFile(id); if (directory === undefined) return null; if (relativeId(id)) return findFile(host.join(directory, id)); + if (id.startsWith("#")) return packageImport(id, directory); return findPackage(id, directory); }; const label = (id: unknown) => @@ -877,16 +1054,25 @@ export function createRuntimeRequire( ? "a path outside this project or missing" : String(id); const evaluate = (real: string) => { - const module = { exports: {} as unknown }; - modules[real] = module; const directory = host.dirname(real); + const require = create(directory); + const module: RuntimeModule = { + exports: {}, + id: real, + filename: real, + path: directory, + loaded: false, + children: [], + require, + }; + modules[real] = module; const source = host.readFileSync(real, "utf8"); try { // A hashbang is not JavaScript; Node ignores it too. host.compile(source.startsWith("#!") ? "//" + source : source, real).call( module.exports, module.exports, - create(directory), + require, module, real, directory, @@ -895,38 +1081,80 @@ export function createRuntimeRequire( delete modules[real]; throw error; } + module.loaded = true; return module.exports; }; + const isDependency = (real: string) => + dependencyPrefixes.some((prefix) => real.startsWith(prefix)); + const load = (found: { real: string; relative: string }) => { + const real = found.real; + if (real in modules) return modules[real]!.exports; + if (real.endsWith(".json")) { + const text = host.readFileSync(real, "utf8"); + // Node drops a byte order mark before parsing JSON. + const exports = JSON.parse(text.charCodeAt(0) === 0xfeff ? text.slice(1) : text); + modules[real] = { exports }; + return exports; + } + if (!isDependency(real)) throw missing(found.relative); + const name = JSON.stringify(real.slice(real.lastIndexOf(host.sep) + 1)); + if (/\.node$/i.test(real)) { + throw failure("Native addons are not supported in API routes: " + name, "ERR_DLOPEN_FAILED"); + } + if (/\.m[jt]s$/i.test(real)) { + throw failure("require() of an ES module is not supported here: " + name, "ERR_REQUIRE_ESM"); + } + return evaluate(real); + }; + // Node's `paths` option: each entry is a directory to resolve from. + const findFrom = (id: unknown, directory: string | undefined, options: unknown) => { + const paths = options !== null && typeof options === "object" + ? (options as { paths?: unknown }).paths + : undefined; + if (!Array.isArray(paths)) return find(id, directory); + for (const path of paths) { + if (typeof path !== "string" || !host.isAbsolute(path)) continue; + const found = find(id, path); + if (found !== null) return found; + } + return null; + }; const create = (directory: string | undefined) => { const require = ((id: unknown) => { const module = builtin(id); if (module !== undefined) return module; const found = find(id, directory); if (found === null) throw missing(label(id)); - if (found.real in modules) return modules[found.real]!.exports; - if (found.real.endsWith(".json")) { - const exports = JSON.parse(host.readFileSync(found.real, "utf8")); - modules[found.real] = { exports }; - return exports; - } - if (dependencyPrefixes.some((prefix) => found.real.startsWith(prefix))) { - return evaluate(found.real); - } - throw missing(found.relative); + return load(found); }) as RuntimeRequire; - require.resolve = (id: unknown) => { + require.resolve = (id: unknown, options?: { paths?: unknown }) => { if (builtin(id) !== undefined) return id as string; - const found = find(id, directory); + const found = findFrom(id, directory, options); if (found === null) throw missing(label(id)); return found.path; }; + require.cache = modules; // Code written for webpack; the callback runs at once, as nothing is split. require.ensure = (_ids: unknown, callback: (require: RuntimeRequire) => void) => { callback(require); }; return require; }; - return create(undefined); + return { + // Route code's computed ids resolve from the project, as Node's would. + require: create(projectRoot), + // The bundler names files by their real path; this keys the module the + // same way even where a symlink above the project was kept. + register: (path, module) => { + let real = path; + try { + real = host.realpathSync(path); + } catch { + // A file the bundler read is there; keep its own name if it is not. + } + if (!(real in modules)) modules[real] = module; + }, + }; } /** @@ -959,9 +1187,11 @@ export function prependRuntimeRequire( `import ${prefix}_process from "node:process";`, `import * as ${prefix}_fs from "node:fs";`, `import * as ${prefix}_path from "node:path";`, - `const require = (${createRuntimeRequire.toString()})({ ${host.join(", ")} }, ${ - JSON.stringify(projectRoot) - }, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])}, ${JSON.stringify(dependencyDirectories)});`, + `const { require, register: ${BUNDLED_MODULE_REGISTRY} } = (${createRuntimeRequire.toString()})({ ${ + host.join(", ") + } }, ${JSON.stringify(projectRoot)}, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])}, ${ + JSON.stringify(dependencyDirectories) + });`, ]; // A hashbang is only valid as the first line. const hashbangEnd = code.startsWith("#!") ? code.indexOf("\n") + 1 : 0; diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 08d1deb429..31bbce1c1f 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -59,6 +59,7 @@ import { type IsFile, PackageTypeLookup, prepareDependencyFile, + prepareDependencyJson, prependRuntimeRequire, rewriteImportSpecifiers, } from "./compiled-cjs-bundling.ts"; @@ -1959,6 +1960,15 @@ function createProjectBoundaryPlugin( importMap: workerImportMap, }); } + // A dependency's JSON registers itself, so that a computed require() + // of the same file returns the same object. + if (!executableModule && dependency) { + return { + contents: prepareDependencyJson(source.contents, args.path), + loader: "js", + resolveDir: pathHelper.dirname(source.logicalPath), + }; + } if (executableModule && dependency) { return { contents: await prepareDependencyFile({ @@ -1969,6 +1979,7 @@ function createProjectBoundaryPlugin( packageType: await packageTypes.of(pathHelper.dirname(source.logicalPath)), isDependencyFile, isFile: hostIsFile, + suffix: args.suffix, }), loader: getLoaderForFile(source.logicalPath), resolveDir: pathHelper.dirname(source.logicalPath), diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 951470b630..c1a490f900 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -16,6 +16,7 @@ import { dependencyDirectories, PackageTypeLookup, prepareDependencyFile, + prepareDependencyJson, prependRuntimeRequire, } from "#veryfront/routing/api/module-loader/compiled-cjs-bundling.ts"; @@ -63,10 +64,13 @@ function dependencyFilePlugin( return { name: "test-dependency-files", setup(build) { - // Every dependency file but JSON, as the route build's boundary loads them. + // Every dependency file, as the route build's boundary loads them. build.onLoad({ filter: /.*/ }, async (args) => { - if (!isDependencyFile(args.path) || args.path.endsWith(".json")) return undefined; + if (!isDependencyFile(args.path)) return undefined; const contents = await readFile(args.path, "utf8"); + if (args.path.endsWith(".json")) { + return { contents: prepareDependencyJson(contents, args.path), loader: "js" }; + } return { contents: await prepareDependencyFile({ build, @@ -76,6 +80,7 @@ function dependencyFilePlugin( packageType: await packageTypes.of(args.path.slice(0, args.path.lastIndexOf("/"))), isDependencyFile, isFile, + suffix: args.suffix, }), loader: "js", }; @@ -401,6 +406,74 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("shares one module between a bundled require and a computed one", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/plugins/package.json": JSON.stringify({ name: "plugins", main: "index.js" }), + "node_modules/plugins/index.js": [ + 'var path = require("path");', + 'var bundled = require("./plugin");', + 'var data = require("./data.json");', + "bundled.count++;", + 'exports.same = require(path.join(__dirname, "plugin.js")) === bundled;', + 'exports.count = require(path.join(__dirname, "plu" + "gin")).count;', + 'exports.sameJson = require(path.join(__dirname, "data.json")) === data;', + "exports.cached = require.cache[require.resolve(path.join(__dirname, 'plugin'))].exports === bundled;", + ].join("\n"), + "node_modules/plugins/plugin.js": "exports.count = 0;", + "node_modules/plugins/data.json": '{"name":"data"}', + }); + const mod = await bundleAndImport( + projectDir, + 'import plugins from "plugins";\nexport { plugins };\n', + ); + assertEquals(mod.plugins, { same: true, count: 1, sameJson: true, cached: true }); + }); + }); + + it("resolves a workspace package's imports past the project's tsconfig paths", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "tsconfig.json": JSON.stringify({ + compilerOptions: { baseUrl: ".", paths: { "pg": ["./mock/pg.js"] } }, + }), + "mock/pg.js": 'module.exports = "mock";', + "node_modules/pg/package.json": JSON.stringify({ name: "pg", main: "index.js" }), + "node_modules/pg/index.js": 'module.exports = "pg";', + "packages/ws/package.json": JSON.stringify({ name: "ws", main: "index.js" }), + "packages/ws/index.js": 'module.exports = require("pg");', + }); + await mkdir(`${projectDir}/node_modules`, { recursive: true }); + await symlink("../packages/ws", `${projectDir}/node_modules/ws`); + const mod = await bundleAndImport( + projectDir, + 'import ws from "ws";\nimport pg from "pg";\nexport const picked = [ws, pg];\n', + ); + // The route's own import still follows the project's paths. + assertEquals(mod.picked, ["pg", "mock"]); + }); + }); + + it("keeps the query of a dependency module in its import.meta.url", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/queried/package.json": JSON.stringify({ + name: "queried", + type: "module", + main: "index.js", + }), + "node_modules/queried/index.js": 'export { url, sibling } from "./child.js?v=1";', + "node_modules/queried/child.js": [ + "export const url = import.meta.url;", + 'export const sibling = import.meta.resolve("./other.js");', + ].join("\n"), + }); + const mod = await bundleAndImport(projectDir, 'export * from "queried";\n'); + const base = pathToFileURL(`${projectDir}/node_modules/queried`).href; + assertEquals({ ...mod }, { url: `${base}/child.js?v=1`, sibling: `${base}/other.js` }); + }); + }); + it("prepares an extensionless CommonJS entry", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { From 3d25226ab3bc6116dde29b1fb507fae7c72401df Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 01:07:47 +0100 Subject: [PATCH 18/29] test(api): avoid a fixture package name Bun treats as built-in --- .../api/module-loader/compiled-cjs-bundling.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index c1a490f900..a239546e1c 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -440,14 +440,14 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { "mock/pg.js": 'module.exports = "mock";', "node_modules/pg/package.json": JSON.stringify({ name: "pg", main: "index.js" }), "node_modules/pg/index.js": 'module.exports = "pg";', - "packages/ws/package.json": JSON.stringify({ name: "ws", main: "index.js" }), - "packages/ws/index.js": 'module.exports = require("pg");', + "packages/workspace/package.json": JSON.stringify({ name: "workspace", main: "index.js" }), + "packages/workspace/index.js": 'module.exports = require("pg");', }); await mkdir(`${projectDir}/node_modules`, { recursive: true }); - await symlink("../packages/ws", `${projectDir}/node_modules/ws`); + await symlink("../packages/workspace", `${projectDir}/node_modules/workspace`); const mod = await bundleAndImport( projectDir, - 'import ws from "ws";\nimport pg from "pg";\nexport const picked = [ws, pg];\n', + 'import workspace from "workspace";\nimport pg from "pg";\nexport const picked = [workspace, pg];\n', ); // The route's own import still follows the project's paths. assertEquals(mod.picked, ["pg", "mock"]); From 37f518a364e321c3d498dff7c747cbf0adbb9b70 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 01:52:53 +0100 Subject: [PATCH 19/29] fix(api): resolve residual requires, package targets and nested externals as Node does - A computed require, or require.resolve, in a bundled dependency resolves from that file's directory instead of the project root. - A null condition in exports/imports blocks the path; package-map targets cannot leave their package. - require.resolve returns the real path that keys require.cache. - import.meta.resolve("#name") goes through the package's imports. - A dependency's nested copy of a package the project keeps external is bundled for it instead of the project's copy. --- .../compiled-cjs-bundling.test.ts | 182 +++++++++++++++-- .../module-loader/compiled-cjs-bundling.ts | 183 +++++++++++++++--- src/routing/api/module-loader/loader.ts | 18 +- .../compiled-cjs-bundling.test.ts | 104 +++++++++- 4 files changed, 443 insertions(+), 44 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 980e19862e..7a4366f62b 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -371,14 +371,74 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { it("keeps imports of the packages external and leaves require() to the bundler", async () => { const [handler] = await resolveHandlers(createImportExternalsPlugin(["dual", "@s/p.js"])); for (const kind of ["import-statement", "dynamic-import"]) { - assertEquals(handler!({ path: "dual/sub", kind }), { path: "dual/sub", external: true }); + assertEquals(await handler!({ path: "dual/sub", kind }), { + path: "dual/sub", + external: true, + }); } - assertEquals(handler!({ path: "@s/p.js", kind: "import-statement" }), { + assertEquals(await handler!({ path: "@s/p.js", kind: "import-statement" }), { path: "@s/p.js", external: true, }); - assertEquals(handler!({ path: "dual", kind: "require-call" }), undefined); - assertEquals(handler!({ path: "dual", kind: "require-resolve" }), undefined); + assertEquals(await handler!({ path: "dual", kind: "require-call" }), undefined); + assertEquals(await handler!({ path: "dual", kind: "require-resolve" }), undefined); + }); + + it("bundles a dependency's nested copy instead of the project's external one", async () => { + // /d/node_modules/dual links to /d/store/dual; p holds its own copy. + const resolved: Record = { + "/d/src/route.ts": "/d/store/dual/index.js", + "/d/node_modules/p/index.js": "/d/node_modules/p/node_modules/dual/index.js", + "/d/node_modules/q/index.js": "/d/store/dual/index.js", + }; + const registered: ResolveHandler[] = []; + const seen: unknown[] = []; + const build = { + onResolve: (_options: unknown, handler: ResolveHandler) => registered.push(handler), + onLoad: () => {}, + resolve: (path: string, options: { importer: string; pluginData: unknown }) => { + seen.push(options.pluginData); + const target = resolved[options.importer]; + return Promise.resolve( + path === "dual/missing" || target === undefined + ? { errors: [{ text: "Could not resolve" }] } + : { path: target, errors: [], namespace: "file", suffix: "" }, + ); + }, + }; + await createImportExternalsPlugin(["dual"], { + isDependencyFile: (path) => path.startsWith("/d/node_modules/"), + nodeModules: ["/d/node_modules", "/d/node_modules"], + realPath: (path) => + Promise.resolve(path === "/d/node_modules/dual" ? "/d/store/dual" : path), + }).setup(build as unknown as PluginBuild); + const [handler] = registered; + const args = (importer: string, path = "dual") => ({ + path, + importer, + kind: "import-statement", + resolveDir: importer.slice(0, importer.lastIndexOf("/")), + }); + const external = { path: "dual", external: true }; + assertEquals(await handler!(args("/d/node_modules/p/index.js")), { + path: "/d/node_modules/p/node_modules/dual/index.js", + namespace: "file", + suffix: "", + sideEffects: undefined, + pluginData: undefined, + }); + assertEquals(await handler!(args("/d/node_modules/q/index.js")), external); + // Route code, a failed resolution and the plugin's own lookup stay as before. + assertEquals(await handler!(args("/d/src/route.ts")), external); + assertEquals(await handler!(args("/d/node_modules/p/index.js", "dual/missing")), { + path: "dual/missing", + external: true, + }); + assertEquals( + await handler!({ ...args("/d/node_modules/p/index.js"), pluginData: seen[0] }), + undefined, + ); + assertEquals(seen.length, 3); }); it("registers nothing without packages", async () => { @@ -612,6 +672,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "#lib/*": "./lib/*.js", "#dep": "driver/extra", "#outside": "../escape.js", + "#up": "./../sibling.js", }, }), "/app/node_modules/p/lib/x.js": "module.exports = 'lib x';", @@ -643,13 +704,37 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/app/node_modules/p/binding.node": "", "/app/node_modules/sibling.js": "module.exports = 'file beside the package';", "/app/node_modules/sibling/index.js": "module.exports = 'package';", + "/app/node_modules/blocked/package.json": JSON.stringify({ + exports: { + ".": { node: null, default: "./fallback.js" }, + "./later": [null, "./later.js"], + "./escape": "./../sibling/index.js", + "./nested": "./node_modules/x.js", + "./encoded": "./%2e%2e/sibling/index.js", + "./empty": "./a//b.js", + "./ok": { import: "./esm.mjs", node: { require: "./ok.js" } }, + }, + }), + "/app/node_modules/blocked/fallback.js": "module.exports = 'fallback';", + "/app/node_modules/blocked/later.js": "module.exports = 'later';", + "/app/node_modules/blocked/node_modules/x.js": "module.exports = 'x';", + "/app/node_modules/blocked/a/b.js": "module.exports = 'b';", + "/app/node_modules/blocked/ok.js": "module.exports = 'ok';", + "/app/node_modules/p/escape-imports.js": "require('#up');", + "/app/node_modules/linked/index.js": "module.exports = 'linked';", + "/app/node_modules/.store/linked/index.js": "module.exports = 'linked';", }; let reads = 0; const host: RuntimeRequireHost = { // Like Deno's, it knows bare prefix-only names that Node does not. getBuiltinModule: (id) => id === "test" ? { kind: "test" } : builtins[id], isFile: (path) => path in files, - realpathSync: (path) => path === "/app/link.json" ? "/outside.json" : path, + realpathSync: (path) => + path === "/app/link.json" + ? "/outside.json" + : path === "/app/node_modules/linked/index.js" + ? "/app/node_modules/.store/linked/index.js" + : path, readFileSync: (path) => { reads++; return files[path]!; @@ -679,7 +764,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { dirname: (path) => path.slice(0, path.lastIndexOf("/")), sep: "/", }; - const { require, register } = createRuntimeRequire( + const { require, register, requireFrom } = createRuntimeRequire( host, "/app", ["test"], @@ -850,6 +935,36 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(require("sibling"), "file beside the package"); }); + it("stops at a null condition and keeps package targets inside the package", () => { + assertEquals(require("blocked/ok"), "ok"); + for (const id of ["blocked", "blocked/later", "blocked/escape", "blocked/nested"]) { + assertEquals(errorCode(() => require(id)), "MODULE_NOT_FOUND", id); + } + for (const id of ["blocked/encoded", "blocked/empty"]) { + assertEquals(errorCode(() => require(id)), "MODULE_NOT_FOUND", id); + } + assertEquals( + errorCode(() => require("/app/node_modules/p/escape-imports.js")), + "MODULE_NOT_FOUND", + ); + }); + + it("resolves to the real path that keys require.cache", () => { + const resolved = require.resolve("linked"); + assertEquals(resolved, "/app/node_modules/.store/linked/index.js"); + assertEquals(require("linked"), "linked"); + assertEquals((require.cache[resolved] as { exports: unknown }).exports, "linked"); + }); + + it("gives a bundled file the require of its own directory, once per directory", () => { + const own = requireFrom("/app/node_modules/p/dialects"); + assert(requireFrom("/app/node_modules/p/dialects") === own); + assertEquals((own("./" + "pg") as { base: string }).base, "base"); + assertEquals(own.resolve("../" + "base"), "/app/node_modules/p/base.js"); + assertEquals(own("nested"), "nested"); + assertEquals(errorCode(() => requireFrom(7)("./pg")), "MODULE_NOT_FOUND"); + }); + it("rejects a native addon and an ES module with Node's codes", () => { assertEquals(errorCode(() => require("/app/node_modules/p/addon.node")), "ERR_DLOPEN_FAILED"); assertEquals(errorCode(() => require("/app/node_modules/p/esm.mjs")), "ERR_REQUIRE_ESM"); @@ -901,7 +1016,9 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { code, ); assert( - code.includes("const { require, register: __vf_register_bundled_module } = ("), + code.includes( + "const { require, register: __vf_register_bundled_module, requireFrom: __vf_require_from } = (", + ), code, ); assert(code.endsWith("\nexport {};\n"), code); @@ -937,6 +1054,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { 'exports.b = require.resolve("fs");', 'exports.c = require.resolve("missing");', "exports.e = require.resolve(name);", + 'exports.g = require("./g");', + "exports.h = require(`./h`);", + 'exports.i = require("./dialects/" + name);', + "exports.j = typeof require;", + "exports.k = function (require) { return require(name); };", ].join("\n"), "/d/p/index.cjs", "module", @@ -949,8 +1071,13 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { 'exports.f = "/d/p/index.cjs";', 'exports.a = "/d/p/a.js";', 'exports.b = "fs";', - 'exports.c = require.resolve("missing");', - "exports.e = require.resolve(name);", + 'exports.c = (__vf_require_from("/d/p")).resolve("missing");', + 'exports.e = (__vf_require_from("/d/p")).resolve(name);', + 'exports.g = require("./g");', + 'exports.h = require("./h");', + 'exports.i = (__vf_require_from("/d/p"))("./dialects/" + name);', + 'exports.j = typeof (__vf_require_from("/d/p"));', + "return require2(name);", ] ) { assert(code.includes(part), code); @@ -1092,9 +1219,9 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { for ( const part of [ '"/d/node_modules/p/a.js";', - 'require.resolve("ext");', - 'require.resolve("project");', - 'require.resolve("broken");', + '(__vf_require_from("/d/node_modules/p")).resolve("ext");', + '(__vf_require_from("/d/node_modules/p")).resolve("project");', + '(__vf_require_from("/d/node_modules/p")).resolve("broken");', ] ) { assert(code.includes(part), code); @@ -1102,7 +1229,36 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { const unresolved = await prepareDependencyFile( input('require.resolve("./a");', "/d/node_modules/p/index.js", false), ); - assert(unresolved.includes('require.resolve("./a");'), unresolved); + assert( + unresolved.includes('(__vf_require_from("/d/node_modules/p")).resolve("./a");'), + unresolved, + ); + }); + + it("parses a file whose only CommonJS name is a computed require", async () => { + const computed = await prepareDependencyFile( + input('module.exports = require("./dialects/" + name);', "/d/node_modules/p/index.js"), + ); + assert( + computed.includes('(__vf_require_from("/d/node_modules/p"))("./dialects/" + name)'), + computed, + ); + // An assignment to require keeps the name, which stays a valid target. + const assigned = await prepareDependencyFile( + input( + 'require = require("esm")(module);\nmodule.exports = require(name);', + "/d/node_modules/p/index.js", + ), + ); + assert(assigned.includes('require = require("esm")(module);'), assigned); + assert(assigned.includes("module.exports = require(name);"), assigned); + assert(!assigned.includes("__vf_require_from"), assigned); + // Literal calls alone leave nothing for runtime, so nothing is parsed. + const literal = "module.exports = [require(\"./a\"), require( 'b' )];"; + assertEquals( + await prepareDependencyFile(input(literal, "/d/node_modules/p/index.js")), + registerBundledModule(literal, "/d/node_modules/p/index.js"), + ); }); it("resolves import.meta.resolve literals in an ESM file", async () => { diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index db8e2fd286..8c2114e6d3 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -151,6 +151,7 @@ const RELATIVE_SPECIFIER = /^(?:\.\.?(?:\/|$)|\/)/; // (the bundler's filters are Go regular expressions, without lookahead). const BARE_SPECIFIER = /^[^./#]/; const URL_SPECIFIER = /^[a-z][a-z0-9+.-]*:/i; +const LITERAL_REQUIRE_CALL = /\brequire\s*\(\s*(["'])[^"'\\\n]*\1\s*\)/g; const IN_NODE_MODULES = /[\\/]node_modules[\\/]/; const FROM_NODE_MODULES = "vfFromNodeModules"; @@ -243,31 +244,92 @@ export function createDependencyResolvePlugin( }; } +/** How {@link createImportExternalsPlugin} tells a nested copy of a package apart. */ +export interface ImportExternalsOptions { + /** Whether an importer is an installed dependency file. */ + readonly isDependencyFile: (path: string) => boolean; + /** The project's top-level `node_modules` directories, as given and canonical. */ + readonly nodeModules: readonly string[]; + /** A path with symlinks followed, or null when it does not exist. */ + readonly realPath: (path: string) => Promise; +} + /** * Keeps the given packages external for `import` and `import()` only. A * package that loads from disk as ESM may also export CommonJS to `require`; * a `require()` of it from a bundled dependency bundles that CommonJS entry, - * since the runtime `require` cannot load packages. Must run where esbuild's - * own resolution would apply: after the project's import map and aliases. + * since the runtime `require` cannot load packages. With `options`, an import + * from a dependency stays external only if it resolves to the project's + * top-level copy; a nested copy the dependency holds is bundled instead. + * Must run where esbuild's own resolution would apply: after the project's + * import map and aliases. */ -export function createImportExternalsPlugin(names: Iterable): Plugin { +export function createImportExternalsPlugin( + names: Iterable, + options?: ImportExternalsOptions, +): Plugin { const escaped = [...names].map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); const filter = new RegExp(`^(?:${escaped.join("|")})(?:/.*)?$`); + // The real directory of each top-level package, which the external import + // loads at runtime. + const topLevel = new Map>(); + const topLevelDirectories = (name: string) => { + let directories = topLevel.get(name); + if (directories === undefined) { + directories = Promise.all( + (options?.nodeModules ?? []).map((root) => options!.realPath(pathHelper.join(root, name))), + ).then((paths) => paths.filter((path): path is string => path !== null)); + topLevel.set(name, directories); + } + return directories; + }; return { name: "vf-import-externals", setup(build) { if (escaped.length === 0) return; - build.onResolve( - { filter }, - (args) => - args.kind === "import-statement" || args.kind === "dynamic-import" - ? { path: args.path, external: true } - : undefined, - ); + build.onResolve({ filter }, async (args) => { + if (args.kind !== "import-statement" && args.kind !== "dynamic-import") return undefined; + const data = args.pluginData as Record | undefined; + if (data?.[RESOLVING_EXTERNAL] === true) return undefined; + const external = { path: args.path, external: true }; + // A dependency may hold its own copy in a nested node_modules, which + // Node loads for it instead of the project's. + if ( + !options || !build.resolve || !args.importer || !options.isDependencyFile(args.importer) + ) { + return external; + } + const result = await build.resolve(args.path, { + kind: args.kind, + importer: args.importer, + resolveDir: args.resolveDir, + pluginData: { [RESOLVING_EXTERNAL]: true }, + }); + if (result.errors?.length || !result.path || result.external) return external; + const name = packageName(args.path); + const real = (await options.realPath(result.path)) ?? result.path; + const directories = await topLevelDirectories(name); + if (directories.some((directory) => isWithinDirectory(directory, real))) return external; + return { + path: result.path, + namespace: result.namespace, + suffix: result.suffix, + sideEffects: result.sideEffects, + pluginData: result.pluginData, + }; + }); }, }; } +// The package a bare specifier names: its first segment, or two when scoped. +function packageName(specifier: string): string { + const parts = specifier.split("/"); + return parts.slice(0, specifier.startsWith("@") ? 2 : 1).join("/"); +} + +const RESOLVING_EXTERNAL = "vfResolvingExternal"; + /** Reads a package.json as text, or null when the directory has none. */ export type ReadManifest = (path: string) => Promise; @@ -489,6 +551,7 @@ export async function prepareCommonJSFile( // The source does not contain the name, so every occurrence in the output // is one the parser put there. It never reaches the bundle. const marker = await markerFor("__vf_require_resolve_", filePath, contents); + const requireMarker = await markerFor("__vf_require_call_", filePath, contents); const result = await bundle({ stdin: { contents, @@ -502,6 +565,7 @@ export async function prepareCommonJSFile( logLevel: "silent", define: { "require.resolve": marker, + require: requireMarker, __dirname: JSON.stringify(pathHelper.dirname(filePath)), __filename: JSON.stringify(filePath), }, @@ -517,8 +581,32 @@ export async function prepareCommonJSFile( async (specifier) => builtinModuleName(specifier) === undefined ? await resolve(specifier) : specifier, ); - // A call left unresolved throws at runtime, as in Node. - return rewritten.replaceAll(marker, "require.resolve"); + // A literal require() goes back to the bundler, which bundles it. Any other + // use of `require`, such as a computed id, runs at runtime with the require + // of this file's directory, as in Node; so does a require.resolve call left + // unresolved, which throws there as in Node. + // A file that assigns to `require` keeps the name: a call is no target. + const assigned = new RegExp(`${requireMarker}\\s*=(?!=)`).test(rewritten); + const ownRequire = assigned + ? "require" + : `(${BUNDLED_REQUIRE_FROM}(${JSON.stringify(pathHelper.dirname(filePath))}))`; + return rewritten + .replace( + literalRequire(requireMarker), + (_whole, double, single, template) => + `require(${JSON.stringify(double ?? single ?? template)})`, + ) + .replaceAll(marker, `${requireMarker}.resolve`) + .replaceAll(requireMarker, ownRequire); +} + +// A call of `marker` with one string literal as the bundler prints it; a +// template literal only without substitutions. +function literalRequire(marker: string): RegExp { + return new RegExp( + `${marker}\\((?:"([^"\\\\\\n]*)"|'([^'\\\\\\n]*)'|\`([^\`\\\\\\n$]*)\`)\\)`, + "g", + ); } /** @@ -644,6 +732,12 @@ export function createImportMetaResolve( */ export const BUNDLED_MODULE_REGISTRY = "__vf_register_bundled_module"; +/** + * The name a bundled CommonJS dependency calls for the `require` of its own + * directory; see {@link RuntimeRequireBinding.requireFrom}. + */ +export const BUNDLED_REQUIRE_FROM = "__vf_require_from"; + /** * `code` with a last statement that registers its CommonJS `module` by * `path`. In an ES module `module` is unbound and nothing happens. A file @@ -703,7 +797,9 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise const importMeta = /\bimport\.meta\b/.test(contents); const readsCommonJSNames = contents.includes("__dirname") || contents.includes("__filename") || - (contents.includes("require") && contents.includes("resolve")); + // Any use of require but a call with a string literal, which the bundler + // bundles: a computed id must resolve from this file's directory. + /\brequire\b/.test(contents.replace(LITERAL_REQUIRE_CALL, "")); if (!readsCommonJSNames && !importMeta) return register(contents); const resolveDir = pathHelper.dirname(resolvedPath); const dependencyPath = async ( @@ -728,10 +824,8 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise return await prepareModuleFile(contents, logicalPath, async (specifier) => { const builtin = builtinModuleName(specifier); if (builtin !== undefined) return "node:" + builtin; - if ( - URL_SPECIFIER.test(specifier) || RELATIVE_SPECIFIER.test(specifier) || - specifier.startsWith("?") || specifier.startsWith("#") - ) { + // A `#name` resolves through the package's own `imports`, below. + if (URL_SPECIFIER.test(specifier) || RELATIVE_SPECIFIER.test(specifier)) { return new URL(specifier, moduleUrl).href; } const path = await dependencyPath(specifier, "import-statement"); @@ -812,6 +906,11 @@ export interface RuntimeRequireBinding { * `require()` of the same file returns it instead of running it again. */ readonly register: (path: string, module: { exports: unknown }) => void; + /** + * The `require` of a bundled dependency file in `directory`, for the calls + * the bundler left to runtime, such as a computed relative id. + */ + readonly requireFrom: (directory: unknown) => RuntimeRequire; } /** The runtime `require` of a bundled route; see {@link createRuntimeRequire}. */ @@ -919,12 +1018,15 @@ export function createRuntimeRequire( }; // The target a package's `exports` or `imports` gives `require()`: the // `node`, `require` and `default` conditions, in the order it lists them. - const conditionalTarget = (value: unknown): string | null => { + // null blocks the path, even when a later condition would match; undefined + // means no condition matched. + const conditionalTarget = (value: unknown): string | null | undefined => { if (typeof value === "string") return value; if (Array.isArray(value)) { + // As in Node, the first item that resolves wins, a null one included. for (const item of value) { const target = conditionalTarget(item); - if (target !== null) return target; + if (target !== undefined) return target; } return null; } @@ -932,15 +1034,15 @@ export function createRuntimeRequire( for (const [condition, item] of Object.entries(value)) { if (condition !== "node" && condition !== "require" && condition !== "default") continue; const target = conditionalTarget(item); - if (target !== null) return target; + if (target !== undefined) return target; } - return null; + return undefined; }; // A key of a package's `exports` or `imports` map, as Node matches it: an // exact key, else the `*` pattern with the longest prefix, whose match // replaces each `*` in the target. const mapTarget = (map: Record, key: string): string | null => { - if (!key.includes("*") && Object.hasOwn(map, key)) return conditionalTarget(map[key]); + if (!key.includes("*") && Object.hasOwn(map, key)) return conditionalTarget(map[key]) ?? null; let best: { pattern: string; match: string } | null = null; for (const pattern of Object.keys(map)) { const star = pattern.indexOf("*"); @@ -959,7 +1061,7 @@ export function createRuntimeRequire( } if (best === null) return null; const target = conditionalTarget(map[best.pattern]); - return target === null ? null : target.split("*").join(best.match); + return typeof target === "string" ? target.split("*").join(best.match) : null; }; const readManifest = (packageDir: string): Record | null | undefined => { const manifestPath = host.join(packageDir, "package.json"); @@ -971,6 +1073,24 @@ export function createRuntimeRequire( return null; // Node rejects a malformed package.json } }; + // A `./` target of a package map, which Node rejects if any later segment + // is empty, `.`, `..` or `node_modules`, so it cannot leave its package. + const packageTarget = (packageDir: string, target: string) => { + if (!target.startsWith("./")) return null; + const segments = target.slice(2).split(/[\\/]/); + const invalid = segments.some((segment) => { + let decoded = segment; + try { + decoded = decodeURIComponent(segment); + } catch { + return true; + } + const lower = decoded.toLowerCase(); + return lower === "" || lower === "." || lower === ".." || lower === "node_modules" || + /[\\/]/.test(decoded); + }); + return invalid ? null : projectFile(host.join(packageDir, target)); + }; const inProject = (directory: string) => { const relative = host.relative(projectRoot, directory); return !(relative === ".." || relative.startsWith(".." + host.sep) || @@ -988,10 +1108,9 @@ export function createRuntimeRequire( const target = keyed ? mapTarget(exports as Record, key) : key === "." - ? conditionalTarget(exports) + ? conditionalTarget(exports) ?? null : null; - if (target === null || !target.startsWith("./")) return null; - return projectFile(host.join(packageDir, target)); + return target === null ? null : packageTarget(packageDir, target); } return findFile(host.join(packageDir, subpath)); }; @@ -1030,7 +1149,7 @@ export function createRuntimeRequire( if (imports === null || typeof imports !== "object" || Array.isArray(imports)) return null; const target = mapTarget(imports as Record, id); if (target === null) return null; - if (target.startsWith("./")) return projectFile(host.join(current, target)); + if (target.startsWith("./")) return packageTarget(current, target); return target.startsWith("#") || target.startsWith("/") || target.startsWith(".") ? null : findPackage(target, current); @@ -1131,7 +1250,8 @@ export function createRuntimeRequire( if (builtin(id) !== undefined) return id as string; const found = findFrom(id, directory, options); if (found === null) throw missing(label(id)); - return found.path; + // Node returns the real path, which also keys require.cache. + return found.real; }; require.cache = modules; // Code written for webpack; the callback runs at once, as nothing is split. @@ -1140,9 +1260,14 @@ export function createRuntimeRequire( }; return require; }; + const byDirectory: Record = Object.create(null); return { // Route code's computed ids resolve from the project, as Node's would. require: create(projectRoot), + requireFrom: (directory) => { + if (typeof directory !== "string") return create(undefined); + return byDirectory[directory] ??= create(directory); + }, // The bundler names files by their real path; this keys the module the // same way even where a symlink above the project was kept. register: (path, module) => { @@ -1187,7 +1312,7 @@ export function prependRuntimeRequire( `import ${prefix}_process from "node:process";`, `import * as ${prefix}_fs from "node:fs";`, `import * as ${prefix}_path from "node:path";`, - `const { require, register: ${BUNDLED_MODULE_REGISTRY} } = (${createRuntimeRequire.toString()})({ ${ + `const { require, register: ${BUNDLED_MODULE_REGISTRY}, requireFrom: ${BUNDLED_REQUIRE_FROM} } = (${createRuntimeRequire.toString()})({ ${ host.join(", ") } }, ${JSON.stringify(projectRoot)}, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])}, ${ JSON.stringify(dependencyDirectories) diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 31bbce1c1f..457f3f04ba 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -1328,6 +1328,15 @@ async function canonicalProjectRoot(projectDir: string): Promise { } /** Whether a path is a regular file on the host. */ +/** A path with symlinks followed, or null when it does not exist. */ +async function hostRealPath(path: string): Promise { + try { + return await realPath(path); + } catch { + return null; // expected: no such file + } +} + const hostIsFile: IsFile = async (path) => { try { return (await createFileSystem().stat(path)).isFile; @@ -2301,7 +2310,14 @@ function buildTranspiledModuleSource( ? [ // After the import map, so that a project can map a built-in name. createCompiledBuiltinPlugin(() => true), - createImportExternalsPlugin(runtimeDeps.keys()), + createImportExternalsPlugin(runtimeDeps.keys(), { + isDependencyFile, + nodeModules: [ + pathHelper.join(projectDir, "node_modules"), + pathHelper.join(canonicalRoot, "node_modules"), + ], + realPath: hostRealPath, + }), ] : []), createHTTPPlugin({ diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index a239546e1c..0962126b41 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -116,7 +116,17 @@ async function bundleAndImport( createCompiledBuiltinPlugin(isDependencyFile), createDependencyResolvePlugin(isDependencyFile, isFile), createCompiledBuiltinPlugin(() => true), - createImportExternalsPlugin(importExternals), + createImportExternalsPlugin(importExternals, { + isDependencyFile, + nodeModules: [`${projectDir}/node_modules`], + realPath: async (path) => { + try { + return await realpath(path); + } catch { + return null; // expected: no such package + } + }, + }), dependencyFilePlugin(projectDir, isDependencyFile), ], }); @@ -572,6 +582,98 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("bundles a dependency's own copy of a package the project keeps external", async () => { + await inProject(async (projectDir) => { + const esm = (value: string) => ({ + "package.json": JSON.stringify({ name: "esmpkg", type: "module", main: "index.js" }), + "index.js": `export default ${JSON.stringify(value)};`, + }); + const files: Record = { + "node_modules/user/package.json": JSON.stringify({ + name: "user", + type: "module", + main: "index.js", + }), + "node_modules/user/index.js": 'export { default } from "esmpkg";', + "node_modules/other/package.json": JSON.stringify({ + name: "other", + type: "module", + main: "index.js", + }), + "node_modules/other/index.js": 'export const load = () => import("esmpkg");', + }; + for (const [name, contents] of Object.entries(esm("top"))) { + files[`node_modules/esmpkg/${name}`] = contents; + } + for (const [name, contents] of Object.entries(esm("nested"))) { + files[`node_modules/user/node_modules/esmpkg/${name}`] = contents; + } + await writeFiles(projectDir, files); + const mod = await bundleAndImport( + projectDir, + 'import user from "user";\nexport { user };\nexport { load } from "other";\n', + ["esmpkg"], + ); + assertEquals(mod.user, "nested"); + const code = await readFile(`${projectDir}/handler.mjs`, "utf8"); + // The dependency without its own copy still imports the project's. + assert(code.includes('import("esmpkg")'), code); + assert(!code.includes('"top"'), code); + }); + }); + + it("resolves a computed relative require from the bundled file's directory", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/loader/package.json": JSON.stringify({ + name: "loader", + main: "lib/index.js", + }), + "node_modules/loader/lib/index.js": [ + "var name = ['p', 'g'].join('');", + 'exports.dialect = require("./dialects/" + name).name;', + 'exports.path = require.resolve("./dialects/" + name);', + "exports.kind = typeof require;", + 'exports.static = require("./dialects/base").name;', + ].join("\n"), + "node_modules/loader/lib/dialects/pg.js": 'exports.name = "pg";', + "node_modules/loader/lib/dialects/base.js": 'exports.name = "base";', + // A project file at the same relative path must not be picked. + "dialects/pg.js": 'exports.name = "project";', + }); + const mod = await bundleAndImport( + projectDir, + 'import loader from "loader";\nexport { loader };\n', + ); + assertEquals(mod.loader, { + dialect: "pg", + path: `${projectDir}/node_modules/loader/lib/dialects/pg.js`, + kind: "function", + static: "base", + }); + }); + }); + + it("resolves a dependency's import.meta.resolve of its own #import", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/hashed/package.json": JSON.stringify({ + name: "hashed", + type: "module", + main: "index.js", + imports: { "#worker": "./worker.js" }, + }), + "node_modules/hashed/index.js": 'export const worker = import.meta.resolve("#worker");', + "node_modules/hashed/worker.js": "export {};", + }); + const mod = await bundleAndImport(projectDir, 'export * from "hashed";\n'); + assertEquals( + mod.worker, + pathToFileURL(`${projectDir}/node_modules/hashed/worker.js`).href, + ); + }); + }); + it("rejects a native addon at build time", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { From 05ea7df4e8183e10e2a2f82407473ac4e17e2367 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 02:30:32 +0100 Subject: [PATCH 20/29] fix(api): follow Node's package resolution in the runtime require - A null item in an exports/imports array falls through to the next item. - A package-map target's percent escapes name the file. - The nearest copy of a package ends the lookup when its exports or a malformed package.json refuse the path. - A .js file in a type: module package throws ERR_REQUIRE_ESM. - A dependency file using direct eval declares require, __dirname and __filename for its own directory. - A dependency's import of an external package subpath names the file the package's exports map it to. --- .../compiled-cjs-bundling.test.ts | 97 +++++++++++++++- .../module-loader/compiled-cjs-bundling.ts | 104 +++++++++++++----- .../compiled-cjs-bundling.test.ts | 53 +++++++++ 3 files changed, 223 insertions(+), 31 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 7a4366f62b..541e9fe0d6 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -398,7 +398,9 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { onLoad: () => {}, resolve: (path: string, options: { importer: string; pluginData: unknown }) => { seen.push(options.pluginData); - const target = resolved[options.importer]; + const target = path === "dual/feature" + ? "/d/store/dual/dist/feature.js" + : resolved[options.importer]; return Promise.resolve( path === "dual/missing" || target === undefined ? { errors: [{ text: "Could not resolve" }] } @@ -428,6 +430,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { pluginData: undefined, }); assertEquals(await handler!(args("/d/node_modules/q/index.js")), external); + // A subpath names the file the package's exports map it to. + assertEquals(await handler!(args("/d/node_modules/q/index.js", "dual/feature")), { + path: "file:///d/node_modules/dual/dist/feature.js", + external: true, + }); // Route code, a failed resolution and the plugin's own lookup stay as before. assertEquals(await handler!(args("/d/src/route.ts")), external); assertEquals(await handler!(args("/d/node_modules/p/index.js", "dual/missing")), { @@ -438,7 +445,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { await handler!({ ...args("/d/node_modules/p/index.js"), pluginData: seen[0] }), undefined, ); - assertEquals(seen.length, 3); + assertEquals(seen.length, 4); }); it("registers nothing without packages", async () => { @@ -713,8 +720,29 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "./encoded": "./%2e%2e/sibling/index.js", "./empty": "./a//b.js", "./ok": { import: "./esm.mjs", node: { require: "./ok.js" } }, + "./space": "./a%20b.js", + "./badescape": "./%E0%A4%A.js", }, }), + "/app/node_modules/blocked/a b.js": "module.exports = 'space';", + "/app/node_modules/shadow/package.json": JSON.stringify({ + exports: { ".": "./main.js", "./sub": "./sub.js" }, + }), + "/app/node_modules/shadow/main.js": "module.exports = 'top main';", + "/app/node_modules/shadow/sub.js": "module.exports = 'top sub';", + "/app/node_modules/p/node_modules/shadow/package.json": JSON.stringify({ + exports: { ".": "./main.js" }, + }), + "/app/node_modules/p/node_modules/shadow/main.js": "module.exports = 'nested main';", + "/app/node_modules/p/node_modules/badnested/package.json": "{", + "/app/node_modules/badnested/package.json": JSON.stringify({ main: "index.js" }), + "/app/node_modules/badnested/index.js": "module.exports = 'top';", + "/app/node_modules/p/shadowed.js": "module.exports = require('shadow');", + "/app/node_modules/p/shadow-sub.js": "require('shadow/sub');", + "/app/node_modules/p/bad-nested.js": "require('badnested');", + "/app/node_modules/esmtype/package.json": JSON.stringify({ type: "module" }), + "/app/node_modules/esmtype/index.js": "export default 1;", + "/app/node_modules/esmtype/legacy.cjs": "module.exports = 'cjs';", "/app/node_modules/blocked/fallback.js": "module.exports = 'fallback';", "/app/node_modules/blocked/later.js": "module.exports = 'later';", "/app/node_modules/blocked/node_modules/x.js": "module.exports = 'x';", @@ -937,7 +965,13 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { it("stops at a null condition and keeps package targets inside the package", () => { assertEquals(require("blocked/ok"), "ok"); - for (const id of ["blocked", "blocked/later", "blocked/escape", "blocked/nested"]) { + // In an array a null falls through to the next item, as in Node. + assertEquals(require("blocked/later"), "later"); + // A target's escapes name the file. + assertEquals(require("blocked/space"), "space"); + for ( + const id of ["blocked", "blocked/escape", "blocked/nested", "blocked/badescape"] + ) { assertEquals(errorCode(() => require(id)), "MODULE_NOT_FOUND", id); } for (const id of ["blocked/encoded", "blocked/empty"]) { @@ -949,6 +983,28 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { ); }); + it("stops at the nearest copy of a package that refuses the path", () => { + assertEquals(require("/app/node_modules/p/shadowed.js"), "nested main"); + for (const file of ["shadow-sub", "bad-nested"]) { + assertEquals( + errorCode(() => require(`/app/node_modules/p/${file}.js`)), + "MODULE_NOT_FOUND", + file, + ); + } + // From elsewhere the top-level copy still serves both. + assertEquals(require("shadow/sub"), "top sub"); + assertEquals(require("badnested"), "top"); + }); + + it("rejects a .js file its package declares an ES module", () => { + assertEquals( + errorCode(() => require("/app/node_modules/esmtype/index.js")), + "ERR_REQUIRE_ESM", + ); + assertEquals(require("/app/node_modules/esmtype/legacy.cjs"), "cjs"); + }); + it("resolves to the real path that keys require.cache", () => { const resolved = require.resolve("linked"); assertEquals(resolved, "/app/node_modules/.store/linked/index.js"); @@ -1084,6 +1140,36 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { } }); + it("declares the CommonJS names in a file a direct eval may read", async () => { + const code = await prepareCommonJSFile( + [ + '"use strict";', + 'exports.p = eval("require")("./p");', + 'exports.s = require("./s");', + 'exports.d = eval("__dirname");', + ].join("\n"), + "/d/p/index.cjs", + undefined, + resolveRelative, + ); + assert(code !== null); + assert( + code.startsWith( + '"use strict";\nvar require = (__vf_require_from("/d/p")), __dirname = "/d/p", ' + + '__filename = "/d/p/index.cjs";\n', + ), + code, + ); + assert(code.includes('exports.s = require("./s");'), code); + const hashbang = await prepareCommonJSFile( + '#!/usr/bin/env node\neval("x");', + "/d/p/bin.cjs", + undefined, + resolveRelative, + ); + assert(hashbang?.startsWith("#!/usr/bin/env node\nvar require = "), String(hashbang)); + }); + it("prepares an extensionless file and a .js file outside a module package", async () => { const entry = await prepareCommonJSFile( "exports.d = __dirname;", @@ -1243,6 +1329,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { computed.includes('(__vf_require_from("/d/node_modules/p"))("./dialects/" + name)'), computed, ); + // A direct eval may read a name the text never spells out. + const evaluated = await prepareDependencyFile( + input('exports.d = eval("__dir" + "name");', "/d/node_modules/p/index.js"), + ); + assert(evaluated.includes('__dirname = "/d/node_modules/p"'), evaluated); // An assignment to require keeps the name, which stays a valid target. const assigned = await prepareDependencyFile( input( diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 8c2114e6d3..74f7a55cc4 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -272,13 +272,18 @@ export function createImportExternalsPlugin( const filter = new RegExp(`^(?:${escaped.join("|")})(?:/.*)?$`); // The real directory of each top-level package, which the external import // loads at runtime. - const topLevel = new Map>(); + type Copy = { readonly logical: string; readonly real: string }; + const topLevel = new Map>(); const topLevelDirectories = (name: string) => { let directories = topLevel.get(name); if (directories === undefined) { directories = Promise.all( - (options?.nodeModules ?? []).map((root) => options!.realPath(pathHelper.join(root, name))), - ).then((paths) => paths.filter((path): path is string => path !== null)); + (options?.nodeModules ?? []).map(async (root) => { + const logical = pathHelper.join(root, name); + const real = await options!.realPath(logical); + return real === null ? null : { logical, real }; + }), + ).then((copies) => copies.filter((copy): copy is Copy => copy !== null)); topLevel.set(name, directories); } return directories; @@ -308,8 +313,17 @@ export function createImportExternalsPlugin( if (result.errors?.length || !result.path || result.external) return external; const name = packageName(args.path); const real = (await options.realPath(result.path)) ?? result.path; - const directories = await topLevelDirectories(name); - if (directories.some((directory) => isWithinDirectory(directory, real))) return external; + const copy = (await topLevelDirectories(name)).find((directory) => + isWithinDirectory(directory.real, real) + ); + if (copy !== undefined) { + // The import rewrite maps a subpath onto the package directory as + // written; the file this one resolved to honours the package's + // `exports`, so the import names it. + if (args.path === name) return external; + const file = pathHelper.join(copy.logical, pathHelper.relative(copy.real, real)); + return { path: pathHelper.toFileUrl(file).href, external: true }; + } return { path: result.path, namespace: result.namespace, @@ -585,19 +599,28 @@ export async function prepareCommonJSFile( // use of `require`, such as a computed id, runs at runtime with the require // of this file's directory, as in Node; so does a require.resolve call left // unresolved, which throws there as in Node. - // A file that assigns to `require` keeps the name: a call is no target. + const directory = pathHelper.dirname(filePath); + const boundRequire = `(${BUNDLED_REQUIRE_FROM}(${JSON.stringify(directory)}))`; + // Code in a direct eval sees the file's own bindings, which no rewrite of + // the source reaches; the file declares them, as Node's wrapper does. Its + // requires then all run at runtime. A file that assigns to `require` also + // keeps the name: a call is no assignment target. + const directEval = /\beval\s*\(/.test(contents); const assigned = new RegExp(`${requireMarker}\\s*=(?!=)`).test(rewritten); - const ownRequire = assigned - ? "require" - : `(${BUNDLED_REQUIRE_FROM}(${JSON.stringify(pathHelper.dirname(filePath))}))`; - return rewritten + const prepared = rewritten .replace( literalRequire(requireMarker), (_whole, double, single, template) => `require(${JSON.stringify(double ?? single ?? template)})`, ) .replaceAll(marker, `${requireMarker}.resolve`) - .replaceAll(requireMarker, ownRequire); + .replaceAll(requireMarker, directEval || assigned ? "require" : boundRequire); + if (!directEval) return prepared; + // After a hashbang and the directive prologue, so "use strict" still applies. + const head = /^(?:#![^\n]*\n)?(?:(["'])[^"'\n]*\1;\n)*/.exec(prepared)?.[0] ?? ""; + const bindings = `var require = ${boundRequire}, __dirname = ${JSON.stringify(directory)}, ` + + `__filename = ${JSON.stringify(filePath)};\n`; + return head + bindings + prepared.slice(head.length); } // A call of `marker` with one string literal as the bundler prints it; a @@ -799,7 +822,9 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise contents.includes("__filename") || // Any use of require but a call with a string literal, which the bundler // bundles: a computed id must resolve from this file's directory. - /\brequire\b/.test(contents.replace(LITERAL_REQUIRE_CALL, "")); + /\brequire\b/.test(contents.replace(LITERAL_REQUIRE_CALL, "")) || + // A direct eval may read any of them. + /\beval\s*\(/.test(contents); if (!readsCommonJSNames && !importMeta) return register(contents); const resolveDir = pathHelper.dirname(resolvedPath); const dependencyPath = async ( @@ -1023,10 +1048,11 @@ export function createRuntimeRequire( const conditionalTarget = (value: unknown): string | null | undefined => { if (typeof value === "string") return value; if (Array.isArray(value)) { - // As in Node, the first item that resolves wins, a null one included. + // As in Node, the first item that resolves to a target wins; a null or + // unmatched item falls through to the next. for (const item of value) { const target = conditionalTarget(item); - if (target !== undefined) return target; + if (typeof target === "string") return target; } return null; } @@ -1075,31 +1101,52 @@ export function createRuntimeRequire( }; // A `./` target of a package map, which Node rejects if any later segment // is empty, `.`, `..` or `node_modules`, so it cannot leave its package. + // A target is a URL path, so its escapes name the file: `%20` is a space. const packageTarget = (packageDir: string, target: string) => { if (!target.startsWith("./")) return null; - const segments = target.slice(2).split(/[\\/]/); - const invalid = segments.some((segment) => { - let decoded = segment; + const decoded: string[] = []; + for (const segment of target.slice(2).split(/[\\/]/)) { + let name: string; try { - decoded = decodeURIComponent(segment); + name = decodeURIComponent(segment); } catch { - return true; + return null; + } + const lower = name.toLowerCase(); + if ( + lower === "" || lower === "." || lower === ".." || lower === "node_modules" || + /[\\/]/.test(name) + ) { + return null; } - const lower = decoded.toLowerCase(); - return lower === "" || lower === "." || lower === ".." || lower === "node_modules" || - /[\\/]/.test(decoded); - }); - return invalid ? null : projectFile(host.join(packageDir, target)); + decoded.push(name); + } + return projectFile(host.join(packageDir, ...decoded)); + }; + // The `type` of the package.json that scopes a file, as Node reads it: the + // nearest one, never past a `node_modules` directory or the project. + const packageType = (file: string) => { + let current = host.dirname(file); + while (inProject(current) && !current.endsWith(host.sep + "node_modules")) { + const manifest = readManifest(current); + if (manifest !== undefined) return manifest?.type; + const parent = host.dirname(current); + if (parent === current) return undefined; + current = parent; + } + return undefined; }; const inProject = (directory: string) => { const relative = host.relative(projectRoot, directory); return !(relative === ".." || relative.startsWith(".." + host.sep) || host.isAbsolute(relative)); }; - // A package's file for `subpath` ("" or "/sub"), as Node's require() picks it. + // A package's file for `subpath` ("" or "/sub"), as Node's require() picks + // it; false when the package is there but refuses the subpath (its + // `exports` or a malformed package.json), which ends Node's search. const packageFile = (packageDir: string, subpath: string) => { const manifest = readManifest(packageDir); - if (manifest === null) return null; + if (manifest === null) return false; const exports = manifest?.exports; if (exports !== undefined && exports !== null) { const key = "." + subpath; @@ -1110,7 +1157,7 @@ export function createRuntimeRequire( : key === "." ? conditionalTarget(exports) ?? null : null; - return target === null ? null : packageTarget(packageDir, target); + return (target === null ? null : packageTarget(packageDir, target)) ?? false; } return findFile(host.join(packageDir, subpath)); }; @@ -1131,6 +1178,7 @@ export function createRuntimeRequire( while (inProject(current)) { if (!current.endsWith(host.sep + "node_modules")) { const found = packageFile(host.join(current, "node_modules", name), subpath); + if (found === false) return null; if (found !== null) return found; } const parent = host.dirname(current); @@ -1220,7 +1268,7 @@ export function createRuntimeRequire( if (/\.node$/i.test(real)) { throw failure("Native addons are not supported in API routes: " + name, "ERR_DLOPEN_FAILED"); } - if (/\.m[jt]s$/i.test(real)) { + if (/\.m[jt]s$/i.test(real) || (/\.js$/i.test(real) && packageType(real) === "module")) { throw failure("require() of an ES module is not supported here: " + name, "ERR_REQUIRE_ESM"); } return evaluate(real); diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 0962126b41..af3c7ae40b 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -654,6 +654,59 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("gives a dependency that uses direct eval its own CommonJS names", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/evaler/package.json": JSON.stringify({ name: "evaler", main: "index.js" }), + "node_modules/evaler/index.js": [ + '"use strict";', + 'exports.plugin = eval("require")("./plugin").name;', + 'exports.dir = eval("__dirname");', + 'exports.file = eval("__filename");', + 'exports.json = require("./data.json").v;', + "exports.strict = (function () { return this; })() === undefined;", + ].join("\n"), + "node_modules/evaler/plugin.js": 'exports.name = "plugin";', + "node_modules/evaler/data.json": '{"v":1}', + "plugin.js": 'exports.name = "project";', + }); + const mod = await bundleAndImport( + projectDir, + 'import evaler from "evaler";\nexport { evaler };\n', + ); + assertEquals(mod.evaler, { + plugin: "plugin", + dir: `${projectDir}/node_modules/evaler`, + file: `${projectDir}/node_modules/evaler/index.js`, + json: 1, + strict: true, + }); + }); + }); + + it("imports the file an external package's exports map a subpath to", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/esmpkg/package.json": JSON.stringify({ + name: "esmpkg", + type: "module", + exports: { ".": "./index.js", "./feature": "./dist/feature.js" }, + }), + "node_modules/esmpkg/index.js": 'export default "root";', + "node_modules/esmpkg/dist/feature.js": 'export default "feature";', + "node_modules/other/package.json": JSON.stringify({ + name: "other", + type: "module", + main: "index.js", + }), + "node_modules/other/index.js": 'export const load = () => import("esmpkg/feature");', + }); + const mod = await bundleAndImport(projectDir, 'export { load } from "other";\n', ["esmpkg"]); + const feature = await (mod.load as () => Promise<{ default: string }>)(); + assertEquals(feature.default, "feature"); + }); + }); + it("resolves a dependency's import.meta.resolve of its own #import", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { From dd15aac3799be49bd9d8df020865d35724f8aa74 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 03:21:00 +0100 Subject: [PATCH 21/29] fix(module-loader): match Node for exports fallbacks, Windows ids and helper names Array fallbacks skip invalid targets, targets strip query and fragment, mixed exports keys are rejected, backslash-relative ids resolve on Windows hosts, .ts dependency files are prepared, external resolution errors surface, import-map and alias dependency targets are prepared, and the runtime helper names never collide with route code. --- .../compiled-cjs-bundling.test.ts | 147 ++++++++++++++++-- .../module-loader/compiled-cjs-bundling.ts | 127 +++++++++++---- src/routing/api/module-loader/loader.ts | 48 +++++- .../compiled-cjs-bundling.test.ts | 39 ++++- 4 files changed, 316 insertions(+), 45 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 541e9fe0d6..abb72a18b2 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -435,11 +435,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { path: "file:///d/node_modules/dual/dist/feature.js", external: true, }); - // Route code, a failed resolution and the plugin's own lookup stay as before. + // Route code and the plugin's own lookup stay as before. assertEquals(await handler!(args("/d/src/route.ts")), external); + // A copy that refuses the path fails the build, as Node would fail. assertEquals(await handler!(args("/d/node_modules/p/index.js", "dual/missing")), { - path: "dual/missing", - external: true, + errors: [{ text: "Could not resolve" }], }); assertEquals( await handler!({ ...args("/d/node_modules/p/index.js"), pluginData: seen[0] }), @@ -478,6 +478,10 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { await prepareDependencyFile(input(contents, "/d/a.cjs", "module")), registerBundledModule(contents, "/d/a.cjs"), ); + assertEquals( + await prepareDependencyFile(input(contents, "/d/a.ts")), + registerBundledModule(contents, "/d/a.ts"), + ); // An ES module has no `module` to register. assertEquals(await prepareDependencyFile(input(contents, "/d/a.js", "module")), contents); assertEquals(await prepareDependencyFile(input(contents, "/d/a.mjs")), contents); @@ -517,8 +521,9 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); it("leaves files that are neither JavaScript nor CommonJS alone", async () => { + // Node loads no `.tsx` file. const contents = "export const d = __dirname;"; - assertEquals(await prepareDependencyFile(input(contents, "/d/a.ts")), contents); + assertEquals(await prepareDependencyFile(input(contents, "/d/a.tsx")), contents); }); }); @@ -680,6 +685,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "#dep": "driver/extra", "#outside": "../escape.js", "#up": "./../sibling.js", + "#fallback": ["../invalid.js", "./base.js"], }, }), "/app/node_modules/p/lib/x.js": "module.exports = 'lib x';", @@ -721,10 +727,24 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "./empty": "./a//b.js", "./ok": { import: "./esm.mjs", node: { require: "./ok.js" } }, "./space": "./a%20b.js", + "./arrinvalid": ["../invalid", "./later.js"], + "./query": "./ok.js?mode=x", + "./fragment": "./ok.js#part", "./badescape": "./%E0%A4%A.js", }, }), "/app/node_modules/blocked/a b.js": "module.exports = 'space';", + "/app/node_modules/mixed/package.json": JSON.stringify({ + exports: { ".": "./ok.js", default: "./ok.js" }, + }), + "/app/node_modules/mixed/ok.js": "module.exports = 'ok';", + "/app/node_modules/pat/package.json": JSON.stringify({ + exports: { "./feature*": "./feature*.js" }, + }), + "/app/node_modules/pat/feature.js": "module.exports = 'feature';", + "/app/node_modules/pat/featureX.js": "module.exports = 'featureX';", + "/app/node_modules/p/badplug/package.json": "{", + "/app/node_modules/p/badplug/index.js": "module.exports = 'index';", "/app/node_modules/shadow/package.json": JSON.stringify({ exports: { ".": "./main.js", "./sub": "./sub.js" }, }), @@ -967,8 +987,20 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(require("blocked/ok"), "ok"); // In an array a null falls through to the next item, as in Node. assertEquals(require("blocked/later"), "later"); - // A target's escapes name the file. + // A target's escapes name the file; its query or fragment does not. assertEquals(require("blocked/space"), "space"); + assertEquals(require("blocked/query"), "ok"); + assertEquals(require.resolve("blocked/fragment"), "/app/node_modules/blocked/ok.js"); + // An invalid item in an array falls through too. + assertEquals(require("blocked/arrinvalid"), "later"); + assertEquals(requireFrom("/app/node_modules/p")("#fallback"), { name: "base" }); + // Node's `*` never matches the empty string. + assertEquals(require("pat/featureX"), "featureX"); + assertEquals(errorCode(() => require("pat/feature")), "MODULE_NOT_FOUND"); + // Nor does Node accept exports that mix subpaths and conditions, or a + // directory whose package.json is malformed. + assertEquals(errorCode(() => require("mixed")), "MODULE_NOT_FOUND"); + assertEquals(errorCode(() => require("/app/node_modules/p/badplug")), "MODULE_NOT_FOUND"); for ( const id of ["blocked", "blocked/escape", "blocked/nested", "blocked/badescape"] ) { @@ -1021,6 +1053,39 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(errorCode(() => requireFrom(7)("./pg")), "MODULE_NOT_FOUND"); }); + it("treats a backslash-relative id as relative where the host separator is one", () => { + const winFiles: Record = { + "C:\\app\\node_modules\\p\\plugin.js": "module.exports = 'plugin';", + }; + const win: RuntimeRequireHost = { + ...host, + isFile: (path) => path in winFiles, + realpathSync: (path) => path, + readFileSync: (path) => winFiles[path]!, + isAbsolute: (path) => /^[A-Z]:\\/.test(path), + relative: (from, to) => + to.startsWith(from + "\\") ? to.slice(from.length + 1) : to === from ? "" : "..\\x", + join: (...paths) => { + const parts: string[] = []; + for (const part of paths.join("\\").split(/[\\/]/)) { + if (part === "..") parts.pop(); + else if (part !== "." && part !== "") parts.push(part); + } + return parts.join("\\"); + }, + dirname: (path) => path.slice(0, path.lastIndexOf("\\")), + sep: "\\", + }; + const { requireFrom: from } = createRuntimeRequire(win, "C:\\app", [], [ + "C:\\app\\node_modules", + ]); + const own = from("C:\\app\\node_modules\\p\\lib"); + assertEquals(own("..\\plugin"), "plugin"); + assertEquals(own.resolve("..\\plugin.js"), "C:\\app\\node_modules\\p\\plugin.js"); + // Nor can a backslash smuggle a path into a package name. + assertEquals(errorCode(() => own("x\\..\\..\\plugin")), "MODULE_NOT_FOUND"); + }); + it("rejects a native addon and an ES module with Node's codes", () => { assertEquals(errorCode(() => require("/app/node_modules/p/addon.node")), "ERR_DLOPEN_FAILED"); assertEquals(errorCode(() => require("/app/node_modules/p/esm.mjs")), "ERR_REQUIRE_ESM"); @@ -1061,8 +1126,13 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); describe("prependRuntimeRequire()", () => { - it("binds require to the runtime require with the host built-ins", () => { - const code = prependRuntimeRequire("export {};\n", "/app", ["/app/node_modules"]); + // A name clash makes the bundler parse the bundle; its service must stop. + afterEach(async () => { + await stopBundler(); + }); + + it("binds require to the runtime require with the host built-ins", async () => { + const code = await prependRuntimeRequire("export {};\n", "/app", ["/app/node_modules"]); assert(code.includes(createRuntimeRequire.toString()), code); assert(code.includes('import __vf_runtime_process from "node:process";'), code); assert( @@ -1073,22 +1143,47 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { ); assert( code.includes( - "const { require, register: __vf_register_bundled_module, requireFrom: __vf_require_from } = (", + "const { require, register: __vf_runtime_register, requireFrom: __vf_runtime_require_from } = (", ), code, ); assert(code.endsWith("\nexport {};\n"), code); }); - it("keeps a hashbang on the first line", () => { - const code = prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", "/app", []); + it("keeps a hashbang on the first line", async () => { + const code = await prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", "/app", []); assert(code.startsWith("#!/usr/bin/env node\nimport __vf_runtime_process from "), code); }); - it("picks helper names the bundle does not already use", () => { - const code = prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app", []); + it("picks helper names the bundle does not already use", async () => { + const code = await prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app", []); assert(code.includes("import * as __vf_runtime___fs from"), code); }); + + it("renames the helper calls dependency preparation wrote", async () => { + const bundled = [ + registerBundledModule("module.exports = 1;", "/app/node_modules/p/a.js"), + 'var r = __vf_require_from("/app/node_modules/p")("./x");', + ].join("\n"); + const code = await prependRuntimeRequire(bundled, "/app", []); + const body = code.slice(code.indexOf(bundled.slice(0, 20))); + assert(!/\b__vf_(?:register_bundled_module|require_from)\b/.test(body), body); + assert(body.includes('__vf_runtime_require_from("/app/node_modules/p")'), body); + assert(body.includes('__vf_runtime_register("/app/node_modules/p/a.js", module)'), body); + }); + + it("keeps a route's own declaration of a helper name and still binds the helpers", async () => { + const bundled = [ + "const __vf_require_from = (id) => 'route:' + id;", + "function __vf_register_bundled_module() { return 'route'; }", + "export const own = [__vf_require_from('a'), __vf_register_bundled_module()];", + ].join("\n"); + const code = await prependRuntimeRequire(bundled, "/app", []); + // It must parse and evaluate: the prologue's bindings do not clash. + const url = "data:text/javascript;base64," + btoa(code); + const mod = await import(url); + assertEquals(mod.own, ["route:a", "route"]); + }); }); describe("parsing dependency files", () => { @@ -1197,6 +1292,34 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(code?.includes('exports.d = "/d/p";'), String(code)); }); + it("prepares a .ts file by its package type and syntax, as Node 24 loads it", async () => { + const commonJS = await prepareCommonJSFile( + "const n: number = 1;\nexports.d = __dirname + n;", + "/d/p/index.ts", + undefined, + resolveRelative, + ); + assert(commonJS?.includes('exports.d = "/d/p" + n;'), String(commonJS)); + assertEquals( + await prepareCommonJSFile( + "exports.d = __dirname;", + "/d/p/index.ts", + "module", + resolveRelative, + ), + null, + ); + assertEquals( + await prepareCommonJSFile( + "export const d: string = __dirname;", + "/d/p/esm.ts", + undefined, + resolveRelative, + ), + null, + ); + }); + it("prepares a .cts file as CommonJS TypeScript", async () => { const code = await prepareCommonJSFile( "const d: string = __dirname;\nexports.d = d;", diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 74f7a55cc4..65aaad0d48 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -25,7 +25,7 @@ * @module routing/api/module-loader/compiled-cjs-bundling */ import { builtinModules } from "node:module"; -import { build as bundle } from "veryfront/extensions/bundler"; +import { build as bundle, transform } from "veryfront/extensions/bundler"; import type { Plugin, PluginBuild } from "veryfront/extensions/bundler"; import * as pathHelper from "#veryfront/compat/path"; import { isWithinDirectory } from "#veryfront/security/path-validation.ts"; @@ -310,7 +310,10 @@ export function createImportExternalsPlugin( resolveDir: args.resolveDir, pluginData: { [RESOLVING_EXTERNAL]: true }, }); - if (result.errors?.length || !result.path || result.external) return external; + // The nearest copy refusing the path is Node's answer, even if the + // project's copy would serve it. + if (result.errors?.length) return { errors: result.errors }; + if (!result.path || result.external) return external; const name = packageName(args.path); const real = (await options.realPath(result.path)) ?? result.path; const copy = (await topLevelDirectories(name)).find((directory) => @@ -556,7 +559,8 @@ export async function prepareCommonJSFile( ): Promise { const cjsExtension = /\.c[jt]s$/i.test(filePath); // Node's require() loads `.jsx` through its `.js` loader too. - const jsExtension = /\.jsx?$/i.test(filePath); + // Node 24 loads `.ts` by the same rules, with its types stripped. + const jsExtension = /\.(?:jsx?|ts)$/i.test(filePath); // Node's require() loads a file without an extension as CommonJS, whatever // its package declares; a package `main` may name one. const extensionless = pathHelper.extname(filePath) === ""; @@ -812,7 +816,7 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise // A file Node may load as CommonJS registers its module for the runtime // `require`; an ES module has no `module` to register. const mayBeCommonJS = /\.c[jt]s$/i.test(logicalPath) || - (/(?:\.jsx?|[\\/][^.\\/]+)$/i.test(logicalPath) && input.packageType !== "module"); + (/(?:\.(?:jsx?|ts)|[\\/][^.\\/]+)$/i.test(logicalPath) && input.packageType !== "module"); const register = (code: string) => mayBeCommonJS ? registerBundledModule(code, resolvedPath) : code; // Most package files read none of these names; parsing them would change @@ -1016,8 +1020,10 @@ export function createRuntimeRequire( if (relative === "" || up || host.isAbsolute(relative)) return null; return { path, real, relative }; }; + // A path relative to the requiring file; on Windows a backslash separates too. const relativeId = (id: string) => - id === "." || id === ".." || id.startsWith("./") || id.startsWith("../"); + id === "." || id === ".." || id.startsWith("./") || id.startsWith("../") || + id.startsWith("." + host.sep) || id.startsWith(".." + host.sep); const first = (candidates: string[]) => { for (const candidate of candidates) { const found = projectFile(candidate); @@ -1033,7 +1039,10 @@ export function createRuntimeRequire( const findFile = (base: string) => { const file = asFile(base); if (file !== null) return file; - const main = readManifest(base)?.main; + const manifest = readManifest(base); + // Node refuses a directory whose package.json is malformed. + if (manifest === null) return null; + const main = manifest?.main; if (typeof main === "string") { const target = host.join(base, main); const found = asFile(target) ?? asIndex(target); @@ -1045,21 +1054,24 @@ export function createRuntimeRequire( // `node`, `require` and `default` conditions, in the order it lists them. // null blocks the path, even when a later condition would match; undefined // means no condition matched. - const conditionalTarget = (value: unknown): string | null | undefined => { + const conditionalTarget = ( + value: unknown, + valid: (target: string) => boolean = () => true, + ): string | null | undefined => { if (typeof value === "string") return value; if (Array.isArray(value)) { - // As in Node, the first item that resolves to a target wins; a null or - // unmatched item falls through to the next. + // As in Node, the first item that resolves to a valid target wins; a + // null, unmatched or invalid item falls through to the next. for (const item of value) { - const target = conditionalTarget(item); - if (typeof target === "string") return target; + const target = conditionalTarget(item, valid); + if (typeof target === "string" && valid(target)) return target; } return null; } if (value === null || typeof value !== "object") return null; for (const [condition, item] of Object.entries(value)) { if (condition !== "node" && condition !== "require" && condition !== "default") continue; - const target = conditionalTarget(item); + const target = conditionalTarget(item, valid); if (target !== undefined) return target; } return undefined; @@ -1067,8 +1079,14 @@ export function createRuntimeRequire( // A key of a package's `exports` or `imports` map, as Node matches it: an // exact key, else the `*` pattern with the longest prefix, whose match // replaces each `*` in the target. - const mapTarget = (map: Record, key: string): string | null => { - if (!key.includes("*") && Object.hasOwn(map, key)) return conditionalTarget(map[key]) ?? null; + const mapTarget = ( + map: Record, + key: string, + valid: (target: string) => boolean, + ): string | null => { + if (!key.includes("*") && Object.hasOwn(map, key)) { + return conditionalTarget(map[key], valid) ?? null; + } let best: { pattern: string; match: string } | null = null; for (const pattern of Object.keys(map)) { const star = pattern.indexOf("*"); @@ -1086,7 +1104,7 @@ export function createRuntimeRequire( if (longer) best = { pattern, match: key.slice(prefix.length, key.length - suffix.length) }; } if (best === null) return null; - const target = conditionalTarget(map[best.pattern]); + const target = conditionalTarget(map[best.pattern], valid); return typeof target === "string" ? target.split("*").join(best.match) : null; }; const readManifest = (packageDir: string): Record | null | undefined => { @@ -1102,10 +1120,11 @@ export function createRuntimeRequire( // A `./` target of a package map, which Node rejects if any later segment // is empty, `.`, `..` or `node_modules`, so it cannot leave its package. // A target is a URL path, so its escapes name the file: `%20` is a space. - const packageTarget = (packageDir: string, target: string) => { + // Its query or fragment is not part of the file's path. + const targetSegments = (target: string) => { if (!target.startsWith("./")) return null; const decoded: string[] = []; - for (const segment of target.slice(2).split(/[\\/]/)) { + for (const segment of target.slice(2).split(/[?#]/)[0]!.split(/[\\/]/)) { let name: string; try { name = decodeURIComponent(segment); @@ -1121,8 +1140,18 @@ export function createRuntimeRequire( } decoded.push(name); } - return projectFile(host.join(packageDir, ...decoded)); + return decoded; }; + const packageTarget = (packageDir: string, target: string) => { + const segments = targetSegments(target); + return segments === null ? null : projectFile(host.join(packageDir, ...segments)); + }; + const validExport = (target: string) => targetSegments(target) !== null; + // An `imports` target may also name a package. + const validImport = (target: string) => + validExport(target) || + !(target.startsWith(".") || target.startsWith("/") || target.startsWith("#") || + /^[a-z][a-z0-9+.-]*:/i.test(target)); // The `type` of the package.json that scopes a file, as Node reads it: the // nearest one, never past a `node_modules` directory or the project. const packageType = (file: string) => { @@ -1150,12 +1179,16 @@ export function createRuntimeRequire( const exports = manifest?.exports; if (exports !== undefined && exports !== null) { const key = "." + subpath; - const keyed = typeof exports === "object" && !Array.isArray(exports) && - Object.keys(exports).some((name) => name.startsWith(".")); - const target = keyed - ? mapTarget(exports as Record, key) + const keys = typeof exports === "object" && !Array.isArray(exports) + ? Object.keys(exports) + : []; + const subpathKeys = keys.filter((name) => name.startsWith(".")).length; + // Node rejects an `exports` object that mixes subpaths and conditions. + if (subpathKeys > 0 && subpathKeys !== keys.length) return false; + const target = subpathKeys > 0 + ? mapTarget(exports as Record, key, validExport) : key === "." - ? conditionalTarget(exports) ?? null + ? conditionalTarget(exports, validExport) ?? null : null; return (target === null ? null : packageTarget(packageDir, target)) ?? false; } @@ -1168,7 +1201,11 @@ export function createRuntimeRequire( const nameLength = id.startsWith("@") ? 2 : 1; if ( parts.length < nameLength || - parts.some((part) => part === "" || part === "." || part === "..") + parts.some((part) => + part === "" || part === "." || part === ".." || + // A Windows separator would let the name walk out of node_modules. + (host.sep !== "/" && part.includes(host.sep)) + ) ) { return null; } @@ -1195,7 +1232,7 @@ export function createRuntimeRequire( if (manifest !== undefined) { const imports = manifest?.imports; if (imports === null || typeof imports !== "object" || Array.isArray(imports)) return null; - const target = mapTarget(imports as Record, id); + const target = mapTarget(imports as Record, id, validImport); if (target === null) return null; if (target.startsWith("./")) return packageTarget(current, target); return target.startsWith("#") || target.startsWith("/") || target.startsWith(".") @@ -1330,19 +1367,53 @@ export function createRuntimeRequire( }; } +// Whether every occurrence of `name` in `code` is a call that dependency +// preparation wrote: `name("...")`, or the `typeof name === "function"` guard. +function onlyHelperCalls(code: string, name: string): boolean { + const all = code.match(new RegExp(`\\b${name}\\b`, "g"))?.length ?? 0; + const calls = code.match( + new RegExp(`\\b${name}\\("|\\btypeof ${name} === "function"`, "g"), + )?.length ?? 0; + return all === calls; +} + /** * Binds the module-level `require` that esbuild's output calls for anything it * did not bundle, to {@link createRuntimeRequire}. esbuild's ESM output never * declares a top-level `require` (it renames a route's own binding of that * name), so the prologue cannot collide with route code. */ -export function prependRuntimeRequire( +export async function prependRuntimeRequire( code: string, projectRoot: string, dependencyDirectories: readonly string[], -): string { +): Promise { let prefix = "__vf_runtime"; while (code.includes(prefix)) prefix += "_"; + // Dependency code calls the helpers by fixed names, which route code may + // declare too. The prologue binds them under names the bundle does not use, + // and only the free references, which are dependency calls, are renamed. + const registry = `${prefix}_register`; + const requireFrom = `${prefix}_require_from`; + const helpers: Record = { + [BUNDLED_MODULE_REGISTRY]: registry, + [BUNDLED_REQUIRE_FROM]: requireFrom, + }; + if (Object.keys(helpers).every((name) => onlyHelperCalls(code, name))) { + for (const [name, local] of Object.entries(helpers)) { + code = code.replace(new RegExp(`\\b${name}\\b`, "g"), local); + } + } else { + // Some other use of a helper's name, such as a declaration in route code: + // the parser tells free references apart, as the bundler does. + code = (await transform(code, { + loader: "js", + format: "esm", + platform: "neutral", + logLevel: "silent", + define: helpers, + })).code; + } const host = [ `getBuiltinModule: (id) => ${prefix}_process.getBuiltinModule(id)`, `isFile: (path) => { try { return ${prefix}_fs.statSync(path).isFile(); } catch { return false; } }`, @@ -1360,7 +1431,7 @@ export function prependRuntimeRequire( `import ${prefix}_process from "node:process";`, `import * as ${prefix}_fs from "node:fs";`, `import * as ${prefix}_path from "node:path";`, - `const { require, register: ${BUNDLED_MODULE_REGISTRY}, requireFrom: ${BUNDLED_REQUIRE_FROM} } = (${createRuntimeRequire.toString()})({ ${ + `const { require, register: ${registry}, requireFrom: ${requireFrom} } = (${createRuntimeRequire.toString()})({ ${ host.join(", ") } }, ${JSON.stringify(projectRoot)}, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])}, ${ JSON.stringify(dependencyDirectories) diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 457f3f04ba..2239d75897 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -1328,6 +1328,26 @@ async function canonicalProjectRoot(projectDir: string): Promise { } /** Whether a path is a regular file on the host. */ +/** + * A project resolver's target that is an installed dependency file, as a + * file-namespace result: there the project boundary prepares it as a + * dependency, and it is the same module the dependency's own imports reach. + * Undefined for any other target, which the resolver's namespace loads. + */ +export async function dependencyTarget( + path: string, + suffix: string | undefined, + isDependencyFile: DependencyFilePredicate, +): Promise<{ path: string; suffix?: string } | undefined> { + if (!isDependencyFile(path)) return undefined; + for (const extension of [...FILE_EXTENSIONS, ".cjs", ".json"]) { + if (await hostIsFile(path + extension)) { + return suffix ? { path: path + extension, suffix } : { path: path + extension }; + } + } + return undefined; +} + /** A path with symlinks followed, or null when it does not exist. */ async function hostRealPath(path: string): Promise { try { @@ -1383,7 +1403,7 @@ function createImportMapPlugin( return { name: "import-map", setup(build) { - build.onResolve({ filter: /.*/ }, (args) => { + build.onResolve({ filter: /.*/ }, async (args) => { if (args.path.startsWith("node:")) return { path: args.path, external: true }; // A dependency resolves its own imports as Node does: the project's // import map does not apply inside node_modules. @@ -1470,6 +1490,12 @@ function createImportMapPlugin( `Import map resolved: ${args.path} -> ${projectModuleLabel(absolutePath, projectDir)}`, ); + const dependency = await dependencyTarget( + absolutePath, + moduleSuffixOfSpecifier(resolvedPath), + isDependencyFile, + ); + if (dependency) return dependency; return { path: absolutePath, namespace: "import-map", @@ -1796,7 +1822,7 @@ function createProjectAliasPlugin( return { name: "vf-project-alias", setup(build) { - build.onResolve({ filter: /^@\// }, (args) => { + build.onResolve({ filter: /^@\// }, async (args) => { // `@/` names a project file only in project code, never in a dependency. if (isDependencyFile(args.importer)) return undefined; const aliasedPath = args.path.slice(2); @@ -1810,6 +1836,12 @@ function createProjectAliasPlugin( return { errors: [{ text: `Project alias escapes project: ${args.path}` }] }; } + const dependency = await dependencyTarget( + absolutePath, + moduleSuffixOfSpecifier(aliasedPath), + isDependencyFile, + ); + if (dependency) return dependency; return { path: absolutePath, namespace: "vf-project-alias", @@ -1844,7 +1876,7 @@ function createAdapterResolvePlugin( return { name: "vf-adapter-resolve", setup(build) { - build.onResolve({ filter: /^\.\.?\// }, (args) => { + build.onResolve({ filter: /^\.\.?\// }, async (args) => { if (args.namespace === "http-url" || args.namespace === "import-map") return undefined; // Node resolves a dependency's relative imports, including directory // indexes and .json files; the project boundary still loads them. @@ -1872,6 +1904,12 @@ function createAdapterResolvePlugin( args.importer ? projectModuleLabel(args.importer, projectDir) : "stdin" }) -> ${projectModuleLabel(absolutePath, projectDir)}`, ); + const dependency = await dependencyTarget( + absolutePath, + moduleSuffixOfSpecifier(args.path), + isDependencyFile, + ); + if (dependency) return dependency; return { path: absolutePath, namespace: "vf-adapter", @@ -2352,7 +2390,9 @@ function buildTranspiledModuleSource( logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); const output = result.outputFiles?.[0]?.text ?? "export {}"; - const js = compiled ? prependRuntimeRequire(output, canonicalRoot, dependencyDirs) : output; + const js = compiled + ? await prependRuntimeRequire(output, canonicalRoot, dependencyDirs) + : output; logger.debug(`transpiled size ${js.length} bytes`); // Only dependencies left external are rewritten. The compiled bundle diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index af3c7ae40b..afecda6d9c 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -19,6 +19,7 @@ import { prepareDependencyJson, prependRuntimeRequire, } from "#veryfront/routing/api/module-loader/compiled-cjs-bundling.ts"; +import { dependencyTarget } from "#veryfront/routing/api/module-loader/loader.ts"; async function writeFiles(root: string, files: Record): Promise { for (const [relative, contents] of Object.entries(files)) { @@ -130,7 +131,7 @@ async function bundleAndImport( dependencyFilePlugin(projectDir, isDependencyFile), ], }); - const code = prependRuntimeRequire( + const code = await prependRuntimeRequire( result.outputFiles?.[0]?.text ?? "", projectDir, await dependencyDirectories(projectDir, hostFs), @@ -727,6 +728,42 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("hands a project resolver's target inside a dependency to the file namespace", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/pkg/lib/index.js": "module.exports = __dirname;", + "node_modules/pkg/data.json": "{}", + "src/own.ts": "export {};", + }); + const isDependencyFile = await createDependencyFilePredicate(projectDir, hostFs); + assertEquals( + await dependencyTarget(`${projectDir}/node_modules/pkg/lib/index`, "?v", isDependencyFile), + { path: `${projectDir}/node_modules/pkg/lib/index.js`, suffix: "?v" }, + ); + assertEquals( + await dependencyTarget( + `${projectDir}/node_modules/pkg/data.json`, + undefined, + isDependencyFile, + ), + { path: `${projectDir}/node_modules/pkg/data.json` }, + ); + // Project files and missing files stay with the resolver's own namespace. + assertEquals( + await dependencyTarget(`${projectDir}/src/own`, undefined, isDependencyFile), + undefined, + ); + assertEquals( + await dependencyTarget( + `${projectDir}/node_modules/pkg/missing`, + undefined, + isDependencyFile, + ), + undefined, + ); + }); + }); + it("rejects a native addon at build time", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { From e77ca40599ef560e2c7bf47ebec230518b770484 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 03:33:04 +0100 Subject: [PATCH 22/29] test(module-loader): parse the helper-collision prologue instead of importing it Bun cannot import a data URL that long. --- .../module-loader/compiled-cjs-bundling.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index abb72a18b2..1f20bc48eb 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,7 +1,7 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals, assertThrows } from "#veryfront/testing/assert.ts"; import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; -import { type PluginBuild, stop as stopBundler } from "veryfront/extensions/bundler"; +import { type PluginBuild, stop as stopBundler, transform } from "veryfront/extensions/bundler"; import { builtinModuleName, createCompiledBuiltinPlugin, @@ -1179,10 +1179,15 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "export const own = [__vf_require_from('a'), __vf_register_bundled_module()];", ].join("\n"); const code = await prependRuntimeRequire(bundled, "/app", []); - // It must parse and evaluate: the prologue's bindings do not clash. - const url = "data:text/javascript;base64," + btoa(code); - const mod = await import(url); - assertEquals(mod.own, ["route:a", "route"]); + // It must parse: a clashing prologue binding is a redeclaration error. + // (Bun cannot import a data URL this long, so the check parses it.) + await transform(code, { loader: "js", format: "esm", logLevel: "silent" }); + // The route keeps its own names; the prologue binds the helpers elsewhere. + const text = code.replaceAll('"', "'"); + assert(text.includes("const __vf_require_from = (id) => 'route:' + id;")); + assert(text.includes("[__vf_require_from('a'), __vf_register_bundled_module()]")); + assert(/register: (\w+)/.exec(text)?.[1] !== "__vf_register_bundled_module"); + assert(/requireFrom: (\w+)/.exec(text)?.[1] !== "__vf_require_from"); }); }); From 3d4664b5d0347f88fa31eaad47cabd7cb7145e08 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 04:29:52 +0100 Subject: [PATCH 23/29] fix(module-loader): share Node's module cache with bundled CommonJS files Bundled dependency files now run through the runtime require's cache: cached before the body runs, evicted on throw or on deletion. Top-level arguments carry Node's five wrapper values. Forced CommonJS files with ES module syntax throw a SyntaxError. The build reads module-sync. The runtime require strips TypeScript types, reads BOM-prefixed and rejects malformed package scopes, accepts pkg/ root ids, evicts unreadable files, and resolves import.meta.resolve packages with import conditions. Windows backslash require ids resolve at build time, and package target suffixes are kept. --- .../compiled-cjs-bundling.test.ts | 406 +++++++++++++---- .../module-loader/compiled-cjs-bundling.ts | 416 +++++++++++++----- src/routing/api/module-loader/loader.ts | 8 +- .../compiled-cjs-bundling.test.ts | 140 +++++- 4 files changed, 774 insertions(+), 196 deletions(-) diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index 1f20bc48eb..d83a8adc3b 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -1,5 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assert, assertEquals, assertThrows } from "#veryfront/testing/assert.ts"; +import { assert, assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; import { type PluginBuild, stop as stopBundler, transform } from "veryfront/extensions/bundler"; import { @@ -19,7 +19,6 @@ import { prepareDependencyJson, prepareModuleFile, prependRuntimeRequire, - registerBundledModule, resolveLiteralCalls, rewriteImportSpecifiers, type RuntimeRequireHost, @@ -123,6 +122,18 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { // Node checks the exact path before adding an extension. assertEquals(await nodePreferredFile("/d/entry", "./entry", "/d", isFile), "/d/entry"); }); + + it("reads a backslash-relative require() id as relative where the host does", async () => { + assertEquals( + await nodePreferredFile("/d/impl.ts", ".\\impl", "/d", isFile, "\\"), + "/d/impl.js", + ); + // Elsewhere a backslash is part of a name, as in Node. + assertEquals( + await nodePreferredFile("/d/impl.ts", ".\\impl", "/d", isFile, "/"), + "/d/impl.ts", + ); + }); }); describe("nodeRelativeFile()", () => { @@ -279,6 +290,40 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(await handler!(args("./impl", "/d/pages/api/route.ts")), undefined); }); + it("reads a backslash in a require() id as a separator where the host does", async () => { + const withPackage = new Set([...files, "/d/node_modules/p/dir/package.json"]); + const resolved: unknown[] = []; + const hooks: ResolveHandler[] = []; + const build = { + onResolve: (_options: unknown, handler: ResolveHandler) => hooks.push(handler), + onLoad: () => {}, + onDispose: () => {}, + resolve: (path: string, options: { kind: string }) => { + resolved.push([path, options.kind]); + return Promise.resolve({ path: "/d/node_modules/p/dir/main.js", errors: [] }); + }, + }; + createDependencyResolvePlugin( + isDependency, + (path) => Promise.resolve(withPackage.has(path)), + "\\", + ).setup(build as unknown as PluginBuild); + const handler = hooks[1]!; + assertEquals(await handler(args(".\\impl")), { path: "/d/node_modules/p/impl.js" }); + // A directory whose package.json decides goes to the bundler as a path. + assertEquals( + ((await handler(args(".\\dir"))) as { path: string }).path, + "/d/node_modules/p/dir/main.js", + ); + assertEquals(resolved, [["./dir", "require-call"]]); + // An import specifier is a URL: a backslash there is part of a name. + assertEquals(await handler({ ...args(".\\impl"), kind: "import-statement" }), undefined); + const [, posix] = await resolveHandlers( + createDependencyResolvePlugin(isDependency, isFile, "/"), + ); + assertEquals(await posix!(args(".\\impl")), undefined); + }); + it("leaves a native addon found by Node's extension order to the loader", async () => { const [, handler] = await resolveHandlers( createDependencyResolvePlugin(isDependency, isFile), @@ -398,13 +443,18 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { onLoad: () => {}, resolve: (path: string, options: { importer: string; pluginData: unknown }) => { seen.push(options.pluginData); - const target = path === "dual/feature" + const target = path === "dual/feature" || path === "dual/query" ? "/d/store/dual/dist/feature.js" : resolved[options.importer]; return Promise.resolve( path === "dual/missing" || target === undefined ? { errors: [{ text: "Could not resolve" }] } - : { path: target, errors: [], namespace: "file", suffix: "" }, + : { + path: target, + errors: [], + namespace: "file", + suffix: path === "dual/query" ? "?mode=x" : "", + }, ); }, }; @@ -435,6 +485,11 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { path: "file:///d/node_modules/dual/dist/feature.js", external: true, }); + // A target's query is part of the module's URL, as in Node. + assertEquals(await handler!(args("/d/node_modules/q/index.js", "dual/query")), { + path: "file:///d/node_modules/dual/dist/feature.js?mode=x", + external: true, + }); // Route code and the plugin's own lookup stay as before. assertEquals(await handler!(args("/d/src/route.ts")), external); // A copy that refuses the path fails the build, as Node would fail. @@ -445,7 +500,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { await handler!({ ...args("/d/node_modules/p/index.js"), pluginData: seen[0] }), undefined, ); - assertEquals(seen.length, 4); + assertEquals(seen.length, 5); }); it("registers nothing without packages", async () => { @@ -464,59 +519,24 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { isFile: () => Promise.resolve(false), }); - it("registers a file that reads no CommonJS names and no import.meta, unparsed", async () => { + it("leaves a file that reads no CommonJS names and no import.meta as it is", async () => { const contents = 'module.exports = require("./other");'; - assertEquals( - await prepareDependencyFile(input(contents, "/d/a.js")), - registerBundledModule(contents, "/d/a.js"), - ); - assertEquals( - await prepareDependencyFile(input(contents, "/d/bin/tool")), - registerBundledModule(contents, "/d/bin/tool"), - ); - assertEquals( - await prepareDependencyFile(input(contents, "/d/a.cjs", "module")), - registerBundledModule(contents, "/d/a.cjs"), - ); - assertEquals( - await prepareDependencyFile(input(contents, "/d/a.ts")), - registerBundledModule(contents, "/d/a.ts"), - ); - // An ES module has no `module` to register. + for ( + const [path, type] of [["/d/a.js"], ["/d/bin/tool"], ["/d/a.cjs", "module"], ["/d/a.ts"]] + ) { + assertEquals(await prepareDependencyFile(input(contents, path!, type)), contents); + } assertEquals(await prepareDependencyFile(input(contents, "/d/a.js", "module")), contents); assertEquals(await prepareDependencyFile(input(contents, "/d/a.mjs")), contents); }); - it("registers a module object by its path after the module's own code", () => { - const code = registerBundledModule("module.exports = 1;", '/d/"q".js'); - const registered: unknown[] = []; - new Function("module", "__vf_register_bundled_module", code)( - { exports: 2 }, - (path: string, module: unknown) => registered.push([path, module]), - ); - assertEquals(registered, [['/d/"q".js', { exports: 1 }]]); - // Outside a CommonJS module, or without the registry, it does nothing. - const bare = registerBundledModule("", "/d/esm.js"); - new Function("module", "__vf_register_bundled_module", bare)(undefined, () => { - throw new Error("registered"); - }); - new Function("module", bare)(null); - new Function("module", code)({ exports: 0 }); - }); - - it("turns dependency JSON into a module that registers itself and rejects bad JSON", () => { - const code = prepareDependencyJson('{"a":[1]}', "/d/data.json"); - const registered: Array<{ exports: unknown }> = []; + it("turns dependency JSON into a CommonJS module and rejects bad JSON", () => { const module = { exports: {} }; - new Function("module", "__vf_register_bundled_module", code)( - module, - (_path: string, value: { exports: unknown }) => registered.push(value), - ); + new Function("module", prepareDependencyJson('{"a":[1]}'))(module); assertEquals(module.exports, { a: [1] }); - assert(registered[0] === module); - assertThrows(() => prepareDependencyJson("{", "/d/bad.json"), SyntaxError); + assertThrows(() => prepareDependencyJson("{"), SyntaxError); const bom = { exports: {} }; - new Function("module", prepareDependencyJson('\uFEFF{"b":2}', "/d/bom.json"))(bom); + new Function("module", prepareDependencyJson('\uFEFF{"b":2}'))(bom); assertEquals(bom.exports, { b: 2 }); }); @@ -771,8 +791,33 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { "/app/node_modules/p/escape-imports.js": "require('#up');", "/app/node_modules/linked/index.js": "module.exports = 'linked';", "/app/node_modules/.store/linked/index.js": "module.exports = 'linked';", + "/app/node_modules/bommain/package.json": '\uFEFF{"main":"main.js"}', + "/app/node_modules/bommain/main.js": "module.exports = 'bom main';", + "/app/node_modules/badscope/package.json": "{", + "/app/node_modules/badscope/plugin.js": "module.exports = 'plugin';", + "/app/node_modules/badscope/p.cjs": "module.exports = 'cjs';", + "/app/node_modules/rootonly/package.json": JSON.stringify({ main: "main.js" }), + "/app/node_modules/rootonly/main.js": "module.exports = 'root';", + "/app/node_modules/p/flaky.js": "module.exports = 'read';", + "/app/node_modules/p/shared.js": "", + "/app/node_modules/p/fails.js": "", + "/app/node_modules/p/typed.cts": "const n: number = 2; module.exports = n;", + "/app/node_modules/p/typed.ts": "const t: number = 3; module.exports = t;", + "/app/node_modules/esmtype/typed.ts": "export default 1;", + "/app/node_modules/cond/package.json": JSON.stringify({ + exports: { + ".": { "module-sync": "./sync.js", import: "./esm.js", require: "./cjs.js" }, + "./q": "./cjs.js?mode=x", + }, + }), + "/app/node_modules/cond/sync.js": "", + "/app/node_modules/cond/esm.js": "", + "/app/node_modules/cond/cjs.js": "module.exports = 'cjs';", + "/app/node_modules/b/package.json": JSON.stringify({ main: "index.js" }), + "/app/node_modules/b/index.js": "", }; let reads = 0; + let unreadable = ""; const host: RuntimeRequireHost = { // Like Deno's, it knows bare prefix-only names that Node does not. getBuiltinModule: (id) => id === "test" ? { kind: "test" } : builtins[id], @@ -785,6 +830,7 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { : path, readFileSync: (path) => { reads++; + if (path === unreadable) throw new Error("EACCES"); return files[path]!; }, compile: (source) => @@ -811,13 +857,19 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }, dirname: (path) => path.slice(0, path.lastIndexOf("/")), sep: "/", + stripTypes: (source) => source.replace(/: number/g, ""), + toFileUrl: (path) => "file://" + path, }; - const { require, register, requireFrom } = createRuntimeRequire( + const { require, bundled, requireFrom } = createRuntimeRequire( host, "/app", ["test"], ["/app/node_modules"], + "/app", ); + const asImport = { [Symbol.for("veryfront.importConditions")]: true } as { + paths?: unknown; + }; it("serves any built-in, including a computed id", () => { assertEquals(require("fs"), { kind: "fs" }); @@ -937,17 +989,104 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assertEquals(require("/app/node_modules/p/counter.js"), (first as number) + 1); }); - it("returns a module the bundle registered, so both loads share it", () => { - const bundled = { exports: { bundled: true } }; - register("/app/node_modules/p/registered.js", bundled); - register("/app/node_modules/p/registered.js", { exports: { second: true } }); - // A path through a link is keyed by its real path. - const linked = { exports: { linked: true } }; - register("/app/link.json", linked); - assert(require.cache["/outside.json"] === linked); - delete require.cache["/outside.json"]; - files["/app/node_modules/p/registered.js"] = "module.exports = { evaluated: true };"; - assert(require("/app/node_modules/p/" + "registered.js") === bundled.exports); + it("runs a bundled file through the module cache computed requires share", () => { + let runs = 0; + const seen: unknown[] = []; + const body = (exports: unknown, module: { exports: unknown }) => { + runs++; + // In the cache before its body runs, as in Node: a cycle sees it. + seen.push(require.cache["/app/node_modules/p/shared.js"] === module); + seen.push(require("/app/node_modules/p/" + "shared.js") === exports); + module.exports = { runs }; + }; + const module = bundled("node_modules/p/shared.js", body)!; + assertEquals(module.exports, { runs: 1 }); + assertEquals(seen, [true, true]); + assert(bundled("node_modules/p/shared.js", body) === module); + assert(require("/app/node_modules/p/shared.js") === module.exports); + assertEquals(runs, 1); + // Deleted from the cache, it runs again on its next load. + delete require.cache["/app/node_modules/p/shared.js"]; + assertEquals(bundled("node_modules/p/shared.js", body)!.exports, { runs: 2 }); + // A body that throws leaves the cache and runs again next time. + let fail = true; + const flaky = () => { + if (fail) throw new Error("boom"); + }; + assertThrows(() => bundled("node_modules/p/fails.js", flaky), Error, "boom"); + assert(!("/app/node_modules/p/fails.js" in require.cache)); + fail = false; + assert(bundled("node_modules/p/fails.js", flaky) !== null); + // A file a computed require loaded first is the module the bundle gets. + const counter = require("/app/node_modules/p/counter.js"); + assert(bundled("node_modules/p/counter.js", body)!.exports === counter); + // A link is keyed by its real path; project code is the bundler's own. + assert( + bundled("node_modules/linked/index.js", () => {}) === + require.cache["/app/node_modules/.store/linked/index.js"], + ); + delete require.cache["/app/node_modules/.store/linked/index.js"]; + assertEquals(bundled("src/own.js", body), null); + assertEquals(bundled(7, body), null); + }); + + it("reads Node's package config: BOMs, malformed scopes and root ids", () => { + assertEquals(require("bommain"), "bom main"); + assertEquals( + errorCode(() => require("/app/node_modules/badscope/plugin.js")), + "ERR_INVALID_PACKAGE_CONFIG", + ); + // Node reads no package type for a .cjs file. + assertEquals(require("/app/node_modules/badscope/p.cjs"), "cjs"); + for (const id of ["rootonly/", "rootonly//", "rootonly/.", "rootonly/./"]) { + assertEquals(require(id), "root"); + } + assertEquals(errorCode(() => require("rootonly/../b")), "MODULE_NOT_FOUND"); + assertEquals(errorCode(() => require("./rootonly")), "MODULE_NOT_FOUND"); + }); + + it("drops a module whose source cannot be read, so the next load reads it again", () => { + unreadable = "/app/node_modules/p/flaky.js"; + assertThrows(() => require("/app/node_modules/p/flaky.js"), Error, "EACCES"); + unreadable = ""; + assertEquals(require("/app/node_modules/p/flaky.js"), "read"); + }); + + it("strips TypeScript's types, as Node 24 does, and refuses a TypeScript module", () => { + assertEquals(require("/app/node_modules/p/typed.cts"), 2); + assertEquals(require("/app/node_modules/p/typed.ts"), 3); + assertEquals( + errorCode(() => require("/app/node_modules/esmtype/typed.ts")), + "ERR_REQUIRE_ESM", + ); + const { require: noStrip } = createRuntimeRequire( + { ...host, stripTypes: undefined }, + "/app", + [], + ["/app/node_modules"], + ); + assertEquals( + errorCode(() => noStrip("/app/node_modules/p/typed.cts")), + "ERR_UNKNOWN_FILE_EXTENSION", + ); + }); + + it("resolves with the import conditions for import.meta.resolve", () => { + // require() cannot run the ES module that module-sync names. + assertEquals(require("cond"), "cjs"); + assertEquals(require.resolve("cond"), "/app/node_modules/cond/cjs.js"); + assertEquals(require.resolve("cond", asImport), "file:///app/node_modules/cond/sync.js"); + assertEquals(require.resolve("cond/q"), "/app/node_modules/cond/cjs.js"); + assertEquals( + require.resolve("cond/q", asImport), + "file:///app/node_modules/cond/cjs.js?mode=x", + ); + assertEquals( + requireFrom("/app/node_modules/p").resolve("#own", asImport), + "file:///app/node_modules/p/base.js", + ); + // The import conditions apply to that call only. + assertEquals(require.resolve("cond"), "/app/node_modules/cond/cjs.js"); }); it("takes Node's paths option in require.resolve", () => { @@ -1132,18 +1271,18 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); it("binds require to the runtime require with the host built-ins", async () => { - const code = await prependRuntimeRequire("export {};\n", "/app", ["/app/node_modules"]); + const code = await prependRuntimeRequire("export {};\n", "/app", ["/app/node_modules"], "/w"); assert(code.includes(createRuntimeRequire.toString()), code); assert(code.includes('import __vf_runtime_process from "node:process";'), code); assert( code.includes( - '}, "/app", ["test","test/reporters","sqlite","sea"], ["/app/node_modules"]);', + '}, "/app", ["test","test/reporters","sqlite","sea"], ["/app/node_modules"], "/w");', ), code, ); assert( code.includes( - "const { require, register: __vf_runtime_register, requireFrom: __vf_runtime_require_from } = (", + "const { require, bundled: __vf_runtime_bundled, requireFrom: __vf_runtime_require_from } = (", ), code, ); @@ -1151,44 +1290,81 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); it("keeps a hashbang on the first line", async () => { - const code = await prependRuntimeRequire("#!/usr/bin/env node\nexport {};\n", "/app", []); + const code = await prependRuntimeRequire( + "#!/usr/bin/env node\nexport {};\n", + "/app", + [], + "/", + ); assert(code.startsWith("#!/usr/bin/env node\nimport __vf_runtime_process from "), code); }); it("picks helper names the bundle does not already use", async () => { - const code = await prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app", []); + const code = await prependRuntimeRequire("const __vf_runtime_fs = 1;\n", "/app", [], "/"); assert(code.includes("import * as __vf_runtime___fs from"), code); }); it("renames the helper calls dependency preparation wrote", async () => { - const bundled = [ - registerBundledModule("module.exports = 1;", "/app/node_modules/p/a.js"), - 'var r = __vf_require_from("/app/node_modules/p")("./x");', - ].join("\n"); - const code = await prependRuntimeRequire(bundled, "/app", []); - const body = code.slice(code.indexOf(bundled.slice(0, 20))); - assert(!/\b__vf_(?:register_bundled_module|require_from)\b/.test(body), body); + const bundled = 'var r = __vf_require_from("/app/node_modules/p")("./x");'; + const code = await prependRuntimeRequire(bundled, "/app", [], "/"); + const body = code.slice(code.indexOf("var r = ")); + assert(!/\b__vf_require_from\b/.test(body), body); assert(body.includes('__vf_runtime_require_from("/app/node_modules/p")'), body); - assert(body.includes('__vf_runtime_register("/app/node_modules/p/a.js", module)'), body); }); - it("keeps a route's own declaration of a helper name and still binds the helpers", async () => { + it("keeps a route's own declaration of the helper name and still binds the helper", async () => { const bundled = [ "const __vf_require_from = (id) => 'route:' + id;", - "function __vf_register_bundled_module() { return 'route'; }", - "export const own = [__vf_require_from('a'), __vf_register_bundled_module()];", + "export const own = __vf_require_from('a');", ].join("\n"); - const code = await prependRuntimeRequire(bundled, "/app", []); + const code = await prependRuntimeRequire(bundled, "/app", [], "/"); // It must parse: a clashing prologue binding is a redeclaration error. // (Bun cannot import a data URL this long, so the check parses it.) await transform(code, { loader: "js", format: "esm", logLevel: "silent" }); - // The route keeps its own names; the prologue binds the helpers elsewhere. + // The route keeps its own name; the prologue binds the helper elsewhere. const text = code.replaceAll('"', "'"); assert(text.includes("const __vf_require_from = (id) => 'route:' + id;")); - assert(text.includes("[__vf_require_from('a'), __vf_register_bundled_module()]")); - assert(/register: (\w+)/.exec(text)?.[1] !== "__vf_register_bundled_module"); + assert(text.includes("own = __vf_require_from('a')"), text); assert(/requireFrom: (\w+)/.exec(text)?.[1] !== "__vf_require_from"); }); + + // esbuild 0.28 prints its CommonJS helper like this. + const helper = (inner: string, error = "e") => + [ + `var __commonJS = (cb, mod) => function ${inner}() {`, + " try {", + " return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;", + ` } catch (${error}) {`, + ` throw mod = 0, ${error};`, + " }", + "};", + ].join("\n"); + + it("runs bundled CommonJS files through the runtime module cache", async () => { + // A direct eval in the bundle makes the bundler rename the helper's locals. + for (const [inner, error] of [["__require", "e"], ["__require2", "e2"]]) { + const code = await prependRuntimeRequire(helper(inner!, error) + "\n", "/app", [], "/w"); + assert(code.includes(` } catch (${error}) {\n throw mod = 0, ${error};`), code); + assert(code.includes(`var __commonJS = (cb, mod) => function ${inner}() {`), code); + assert(code.includes("const managed = __vf_runtime_bundled(key, cb[key]);"), code); + // A file the runtime does not manage keeps the bundler's own cache. + assert(code.includes("return mod || (0, cb[key])((mod = { exports: {} })"), code); + } + }); + + it("fails the build on a CommonJS helper it does not know", async () => { + await assertRejects( + () => + prependRuntimeRequire( + "var __commonJS = (cb, mod) => () => mod || cb();\n", + "/app", + [], + "/", + ), + Error, + "unexpected CommonJS helper", + ); + }); }); describe("parsing dependency files", () => { @@ -1201,6 +1377,50 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { Promise.resolve(specifier.startsWith("./") ? "/d/p/" + specifier.slice(2) + ".js" : null); describe("prepareCommonJSFile()", () => { + it("gives top-level arguments Node's five wrapper arguments", async () => { + const code = await prepareCommonJSFile( + [ + '"use strict";', + "exports.req = arguments[1];", + "exports.rest = [arguments.length, arguments[3], arguments[4]];", + "exports.inner = (function () { return arguments[0]; })('own');", + "exports.arrow = (() => arguments[2] === module)();", + ].join("\n"), + "/d/p/index.js", + undefined, + () => Promise.resolve(null), + ); + assert(code !== null); + assert(code.startsWith('"use strict";\nvar __vf_arguments_'), code); + const module = { exports: {} as Record }; + const requireFrom = (directory: string) => ({ directory }); + new Function("exports", "module", "__vf_require_from", code)( + module.exports, + module, + requireFrom, + ); + assertEquals(module.exports, { + req: { directory: "/d/p" }, + rest: [5, "/d/p/index.js", "/d/p"], + inner: "own", + arrow: true, + }); + }); + + it("throws a SyntaxError when a file Node parses as CommonJS has ES module syntax", async () => { + const esm = "export const a = __dirname;"; + for (const [path, type] of [["/d/p/a.cjs"], ["/d/p/a.js", "commonjs"], ["/d/p/a.cts"]]) { + const code = await prepareCommonJSFile(esm, path!, type, () => Promise.resolve(null)); + assert(code !== null, path); + assertThrows(() => new Function(code)(), SyntaxError, "uses ES module syntax", path); + } + // Without a declared type, the syntax decides, as Node's detection does. + assertEquals( + await prepareCommonJSFile(esm, "/d/p/a.js", undefined, () => Promise.resolve(null)), + null, + ); + }); + it("binds __dirname and __filename and resolves require.resolve literals", async () => { const code = await prepareCommonJSFile( [ @@ -1400,8 +1620,12 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { }); describe("prepareDependencyFile()", () => { - const resolved: Record = { + const resolved: Record< + string, + { path?: string; external?: boolean; errors?: unknown[]; suffix?: string } + > = { "./a": { path: "/d/node_modules/p/a.js" }, + "pkg/q": { path: "/d/node_modules/pkg/q.js", suffix: "?mode=x" }, pkg: { path: "/d/node_modules/pkg/main.js" }, ext: { path: "ext", external: true }, project: { path: "/d/src/project.js" }, @@ -1472,18 +1696,28 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { assert(assigned.includes('require = require("esm")(module);'), assigned); assert(assigned.includes("module.exports = require(name);"), assigned); assert(!assigned.includes("__vf_require_from"), assigned); + // A file that reads only `arguments`, or a CommonJS file with only + // ES module syntax, is parsed too. + const args = await prepareDependencyFile( + input("module.exports = arguments.length;", "/d/node_modules/p/index.js"), + ); + assert(args.includes("return arguments;"), args); + const forced = await prepareDependencyFile( + input("export default 1;", "/d/node_modules/p/index.cjs"), + ); + assert(forced.includes("SyntaxError"), forced); // Literal calls alone leave nothing for runtime, so nothing is parsed. const literal = "module.exports = [require(\"./a\"), require( 'b' )];"; assertEquals( await prepareDependencyFile(input(literal, "/d/node_modules/p/index.js")), - registerBundledModule(literal, "/d/node_modules/p/index.js"), + literal, ); }); it("resolves import.meta.resolve literals in an ESM file", async () => { const code = await prepareDependencyFile( input( - ["fs", "./x.js", "https://e.test/m.js", "pkg", "ext"].map((id) => + ["fs", "./x.js", "https://e.test/m.js", "pkg", "ext", "pkg/q"].map((id) => `import.meta.resolve(${JSON.stringify(id)});` ).join("\n"), "/d/node_modules/p/index.mjs", @@ -1495,6 +1729,8 @@ describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { '"file:///d/node_modules/p/x.js";', '"https://e.test/m.js";', '"file:///d/node_modules/pkg/main.js";', + // A package target's query stays in the URL, as in Node. + '"file:///d/node_modules/pkg/q.js?mode=x";', "_resolve_", ] ) { diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.ts b/src/routing/api/module-loader/compiled-cjs-bundling.ts index 65aaad0d48..29f10660e6 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.ts @@ -99,13 +99,16 @@ export function createNativeAddonPlugin(): Plugin { /** * Resolution options of a compiled-binary route build. Bundled packages read - * the `node` condition and `main`, as Node does; Node never reads `module`. + * the `node` and `module-sync` conditions and `main`, as Node 24 does; Node + * never reads `module`. `node-addons` stays off, as under Node's `--no-addons`: + * the compiled binary cannot load a native addon, so a package's universal + * entry is the one it can run. * The extension order serves project code, and * {@link createDependencyResolvePlugin} restores Node's order inside packages. */ export const COMPILED_RESOLVE_OPTIONS = { mainFields: ["main"], - conditions: ["node"], + conditions: ["node", "module-sync"], resolveExtensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".json"], }; @@ -128,7 +131,9 @@ export async function nodePreferredFile( specifier: string, resolveDir: string, isFile: IsFile, + sep: string = pathHelper.sep, ): Promise { + specifier = requirePath(specifier, sep); const relative = specifier.startsWith("./") || specifier.startsWith("../") || specifier === "." || specifier === ".." || pathHelper.isAbsolute(specifier); const extension = pathHelper.extname(resolvedPath); @@ -146,6 +151,18 @@ export async function nodePreferredFile( } const RELATIVE_SPECIFIER = /^(?:\.\.?(?:\/|$)|\/)/; +// Also a Windows path relative to the importer; see requirePath(). +const RELATIVE_REQUIRE_FILTER = /^(?:\.\.?(?:[\\/]|$)|\/)/; + +/** + * A `require()` id with Windows separators written as `/` when the host + * separates paths with a backslash, where Node's require() reads `.\\x` as + * relative. Elsewhere, and for ES module specifiers, which are URLs, a + * backslash is part of a name. + */ +export function requirePath(specifier: string, sep: string = pathHelper.sep): string { + return sep === "\\" && /^\.\.?\\/.test(specifier) ? specifier.replaceAll("\\", "/") : specifier; +} // A package name: not a path and not a package's own `#import`, which // resolves from the importer's package. A URL is told apart in the handler // (the bundler's filters are Go regular expressions, without lookahead). @@ -192,6 +209,7 @@ export async function nodeRelativeFile( export function createDependencyResolvePlugin( isDependencyFile: (path: string) => boolean, isFile: IsFile, + sep: string = pathHelper.sep, ): Plugin { return { name: "vf-dependency-resolve", @@ -235,10 +253,32 @@ export function createDependencyResolvePlugin( pluginData: result.pluginData, }; }); - build.onResolve({ filter: RELATIVE_SPECIFIER }, async (args) => { + build.onResolve({ filter: RELATIVE_REQUIRE_FILTER }, async (args) => { if (!args.importer || !isDependencyFile(args.importer)) return undefined; - const file = await nodeRelativeFile(args.path, args.resolveDir, isFile); - return file === null ? undefined : { path: file }; + const required = args.kind === "require-call" || args.kind === "require-resolve"; + const path = required ? requirePath(args.path, sep) : args.path; + if (!RELATIVE_SPECIFIER.test(path)) return undefined; + const file = await nodeRelativeFile(path, args.resolveDir, isFile); + if (file !== null) return { path: file }; + if (path === args.path || !build.resolve) return undefined; + // The bundler reads `.\\x` as a package name; it resolves the path. + const data = args.pluginData as Record | undefined; + if (data?.[RESOLVING_WINDOWS_PATH] === true) return undefined; + const result = await build.resolve(path, { + kind: args.kind, + importer: args.importer, + resolveDir: args.resolveDir, + pluginData: { [RESOLVING_WINDOWS_PATH]: true }, + }); + if (result.errors?.length) return { errors: result.errors }; + return { + path: result.path, + external: result.external, + namespace: result.namespace, + suffix: result.suffix, + sideEffects: result.sideEffects, + pluginData: result.pluginData, + }; }); }, }; @@ -325,7 +365,8 @@ export function createImportExternalsPlugin( // `exports`, so the import names it. if (args.path === name) return external; const file = pathHelper.join(copy.logical, pathHelper.relative(copy.real, real)); - return { path: pathHelper.toFileUrl(file).href, external: true }; + // A target's query or fragment is part of the module's URL, as in Node. + return { path: pathHelper.toFileUrl(file).href + (result.suffix ?? ""), external: true }; } return { path: result.path, @@ -346,6 +387,7 @@ function packageName(specifier: string): string { } const RESOLVING_EXTERNAL = "vfResolvingExternal"; +const RESOLVING_WINDOWS_PATH = "vfResolvingWindowsPath"; /** Reads a package.json as text, or null when the directory has none. */ export type ReadManifest = (path: string) => Promise; @@ -386,7 +428,9 @@ export class PackageTypeLookup { const manifest = await this.readManifest(pathHelper.join(directory, "package.json")); if (manifest !== null) { try { - const type = (JSON.parse(manifest) as { type?: unknown }).type; + // Node drops a byte order mark before parsing a package.json. + const text = manifest.charCodeAt(0) === 0xfeff ? manifest.slice(1) : manifest; + const type = (JSON.parse(text) as { type?: unknown }).type; return typeof type === "string" ? type : undefined; } catch { // Node rejects a malformed package.json; esbuild reports it as well. @@ -545,7 +589,10 @@ export type ResolveRequireLiteral = (specifier: string) => Promise Promise globalThis.process?.getBuiltinModule?.(id))(specifier);`, + }, (id) => globalThis.process?.getBuiltinModule?.(id), (id) => typeof ${BUNDLED_REQUIRE_FROM} === "function" ? ${BUNDLED_REQUIRE_FROM}(${ + JSON.stringify(pathHelper.dirname(filePath)) + }).resolve(id, { [Symbol.for("veryfront.importConditions")]: true }) : null)(specifier);`, "}", `const ${marker}_meta = { url: ${JSON.stringify(url)}, filename: ${ JSON.stringify(filePath) @@ -727,8 +801,8 @@ export async function prepareModuleFile( /** * The runtime `import.meta.resolve` of a bundled dependency module at * `moduleUrl`: URLs and relative paths resolve against the module, built-ins - * to their `node:` ids, and a package name throws `ERR_MODULE_NOT_FOUND`, - * since the bundle cannot locate packages at runtime. + * to their `node:` ids, and a package name or `#import` to the URL + * `resolvePackage` finds. One it cannot find throws `ERR_MODULE_NOT_FOUND`. * * {@link prepareModuleFile} embeds this function's source in the module, so * it must not refer to anything outside its own body. @@ -737,6 +811,7 @@ export function createImportMetaResolve( moduleUrl: string, prefixOnlyNames: readonly string[], getBuiltinModule: (id: string) => unknown, + resolvePackage: (id: string) => string | null = () => null, ): (specifier: unknown) => string { return (specifier) => { const id = String(specifier); @@ -744,6 +819,13 @@ export function createImportMetaResolve( if (/^(?:\.\.?(?:\/|$)|\/)/.test(id)) return new URL(id, moduleUrl).href; // A bare prefix-only name such as `test` is a package, as in Node. if (!prefixOnlyNames.includes(id) && getBuiltinModule(id) !== undefined) return "node:" + id; + let found: string | null = null; + try { + found = resolvePackage(id); + } catch { + // Not found; reported below with ESM's code. + } + if (found !== null) return found; const error = new Error( "Cannot find package " + JSON.stringify(id) + " imported from a bundled dependency", ) as Error & { code?: string }; @@ -752,13 +834,6 @@ export function createImportMetaResolve( }; } -/** - * The name a bundled dependency module registers itself under, so that the - * runtime `require` shares one module cache with the bundle. The route - * module's prologue binds it; see {@link prependRuntimeRequire}. - */ -export const BUNDLED_MODULE_REGISTRY = "__vf_register_bundled_module"; - /** * The name a bundled CommonJS dependency calls for the `require` of its own * directory; see {@link RuntimeRequireBinding.requireFrom}. @@ -766,27 +841,16 @@ export const BUNDLED_MODULE_REGISTRY = "__vf_register_bundled_module"; export const BUNDLED_REQUIRE_FROM = "__vf_require_from"; /** - * `code` with a last statement that registers its CommonJS `module` by - * `path`. In an ES module `module` is unbound and nothing happens. A file - * that leaves early with a top-level `return` is not registered, and a - * computed `require()` of it runs it again. + * A dependency's JSON file as a CommonJS module, which the runtime `require` + * caches like any bundled dependency, so a computed `require()` of the same + * file returns the same object, as in Node. Malformed JSON fails the build, as + * the bundler's JSON loader would. */ -export function registerBundledModule(code: string, path: string): string { - const registry = BUNDLED_MODULE_REGISTRY; - return `${code}\n;typeof module === "object" && module !== null && ` + - `typeof ${registry} === "function" && ${registry}(${JSON.stringify(path)}, module);\n`; -} - -/** - * A dependency's JSON file as a CommonJS module that registers itself, so a - * computed `require()` of the same file returns the same object, as in Node. - * Malformed JSON fails the build, as the bundler's JSON loader would. - */ -export function prepareDependencyJson(contents: string, path: string): string { +export function prepareDependencyJson(contents: string): string { // Node drops a byte order mark before parsing JSON. const json = contents.charCodeAt(0) === 0xfeff ? contents.slice(1) : contents; JSON.parse(json); - return registerBundledModule(`module.exports = JSON.parse(${JSON.stringify(json)});`, path); + return `module.exports = JSON.parse(${JSON.stringify(json)});\n`; } /** Input for {@link prepareDependencyFile}. */ @@ -813,23 +877,23 @@ export interface DependencyFileInput { */ export async function prepareDependencyFile(input: DependencyFileInput): Promise { const { build, contents, logicalPath, resolvedPath, isDependencyFile } = input; - // A file Node may load as CommonJS registers its module for the runtime - // `require`; an ES module has no `module` to register. - const mayBeCommonJS = /\.c[jt]s$/i.test(logicalPath) || - (/(?:\.(?:jsx?|ts)|[\\/][^.\\/]+)$/i.test(logicalPath) && input.packageType !== "module"); - const register = (code: string) => - mayBeCommonJS ? registerBundledModule(code, resolvedPath) : code; + // A file Node parses as CommonJS whatever its syntax. + const forcedCommonJS = /\.c[jt]s$/i.test(logicalPath) || + (/\.(?:jsx?|ts)$/i.test(logicalPath) && input.packageType === "commonjs"); // Most package files read none of these names; parsing them would change // nothing. The check is textual, so it can only send extra files to the parser. const importMeta = /\bimport\.meta\b/.test(contents); const readsCommonJSNames = contents.includes("__dirname") || contents.includes("__filename") || + /\barguments\b/.test(contents) || // Any use of require but a call with a string literal, which the bundler // bundles: a computed id must resolve from this file's directory. /\brequire\b/.test(contents.replace(LITERAL_REQUIRE_CALL, "")) || // A direct eval may read any of them. - /\beval\s*\(/.test(contents); - if (!readsCommonJSNames && !importMeta) return register(contents); + /\beval\s*\(/.test(contents) || + // ES module syntax, which such a file must not run as. + (forcedCommonJS && /\b(?:import|export)\b/.test(contents)); + if (!readsCommonJSNames && !importMeta) return contents; const resolveDir = pathHelper.dirname(resolvedPath); const dependencyPath = async ( specifier: string, @@ -839,15 +903,21 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise const result = await build.resolve(specifier, { kind, importer: resolvedPath, resolveDir }); if (result.errors?.length || !result.path || result.external) return null; if (!isDependencyFile(result.path)) return null; - return await nodePreferredFile(result.path, specifier, resolveDir, input.isFile); + // Only require() reads a Windows backslash as a separator. + const sep = kind === "require-resolve" ? pathHelper.sep : "/"; + return { + path: await nodePreferredFile(result.path, specifier, resolveDir, input.isFile, sep), + // A package target's query or fragment: import.meta.resolve keeps it. + suffix: result.suffix ?? "", + }; }; const commonJS = await prepareCommonJSFile( contents, logicalPath, input.packageType, - (specifier) => dependencyPath(specifier, "require-resolve"), + async (specifier) => (await dependencyPath(specifier, "require-resolve"))?.path ?? null, ); - if (commonJS !== null) return registerBundledModule(commonJS, resolvedPath); + if (commonJS !== null) return commonJS; if (!importMeta) return contents; const moduleUrl = pathHelper.toFileUrl(logicalPath).href + (input.suffix ?? ""); return await prepareModuleFile(contents, logicalPath, async (specifier) => { @@ -857,8 +927,8 @@ export async function prepareDependencyFile(input: DependencyFileInput): Promise if (URL_SPECIFIER.test(specifier) || RELATIVE_SPECIFIER.test(specifier)) { return new URL(specifier, moduleUrl).href; } - const path = await dependencyPath(specifier, "import-statement"); - return path === null ? null : pathHelper.toFileUrl(path).href; + const found = await dependencyPath(specifier, "import-statement"); + return found === null ? null : pathHelper.toFileUrl(found.path).href + found.suffix; }, input.suffix ?? ""); } @@ -900,6 +970,10 @@ export interface RuntimeRequireHost { readFileSync(path: string, encoding: "utf8"): string; /** Compiles a CommonJS module body. */ compile(source: string, filename: string): CommonJSModuleBody; + /** Node's `stripTypeScriptTypes`, when the runtime has it. */ + stripTypes?: (source: string) => string; + /** A path's `file:` URL. */ + toFileUrl(path: string): string; isAbsolute(path: string): boolean; relative(from: string, to: string): string; join(...paths: string[]): string; @@ -931,10 +1005,17 @@ export interface RuntimeModule { export interface RuntimeRequireBinding { readonly require: RuntimeRequire; /** - * Records a module the bundler bundled, by real path, so that a computed - * `require()` of the same file returns it instead of running it again. + * Runs a CommonJS file the bundler bundled, named by `key` relative to the + * bundle's working directory, through the module cache that computed + * requires use, as Node's one cache: it runs once, is in `require.cache` + * before its body runs, leaves it if the body throws, and runs again once + * deleted from it. Null for a file that is not a dependency, which the + * bundler caches itself. */ - readonly register: (path: string, module: { exports: unknown }) => void; + readonly bundled: ( + key: unknown, + body: (exports: unknown, module: { exports: unknown }) => void, + ) => { exports: unknown } | null; /** * The `require` of a bundled dependency file in `directory`, for the calls * the bundler left to runtime, such as a computed relative id. @@ -964,15 +1045,16 @@ export interface RuntimeRequire { * `.json`, then its `index.js` or `index.json`. * - A package name is looked up in `node_modules` upwards, through the * package's `exports` (exact keys and `*` patterns; `node`, `require` and - * `default` conditions) or its `main`. A `#name` reads the `imports` of the + * `default` conditions, or `import` and `module-sync` for a bundled module's + * `import.meta.resolve`) or its `main`. A `#name` reads the `imports` of the * package that holds the module. * * Whatever it finds must have its real path inside the project, and nothing * looks above it. JSON loads from anywhere in the project. JavaScript loads * only from the project's dependencies, as CommonJS, with its own `require` - * and `module`. A native addon or an ES module throws. Each file loads once, - * and a module the bundle already ran is the same module; see - * {@link RuntimeRequireBinding.register}. `require.cache` holds them by real + * and `module`, its TypeScript types stripped as Node 24 strips them. A native + * addon or an ES module throws. Each file loads once, and bundled files share + * the same cache; see {@link RuntimeRequireBinding.bundled}. `require.cache` holds them by real * path, `require.resolve` returns the path it found and takes Node's `paths` * option, and `require.ensure`, which code written for webpack calls, runs * its callback. Anything else throws `MODULE_NOT_FOUND`, so optional @@ -987,12 +1069,16 @@ export function createRuntimeRequire( projectRoot: string, prefixOnlyNames: readonly string[], dependencyDirectories: readonly string[], + bundleRoot?: string, ): RuntimeRequireBinding { // Tables without a prototype: `in` reads only their own keys and does not // depend on built-ins that route code could replace later. const prefixOnly: Record = Object.create(null); for (const name of prefixOnlyNames) prefixOnly[name] = true; const modules: Record = Object.create(null); + // Set while resolving for `import.meta.resolve`, which reads a package map's + // `import` condition where require() reads `require`; see `resolve`. + let importing = false; const failure = (message: string, code: string) => { const error = new Error(message) as Error & { code?: string }; error.code = code; @@ -1070,7 +1156,13 @@ export function createRuntimeRequire( } if (value === null || typeof value !== "object") return null; for (const [condition, item] of Object.entries(value)) { - if (condition !== "node" && condition !== "require" && condition !== "default") continue; + // `module-sync` only for an import: this require() cannot run an ES + // module, which is when Node turns the condition off too. + const active = condition === "node" || condition === "default" || + (importing + ? condition === "import" || condition === "module-sync" + : condition === "require"); + if (!active) continue; const target = conditionalTarget(item, valid); if (target !== undefined) return target; } @@ -1111,7 +1203,9 @@ export function createRuntimeRequire( const manifestPath = host.join(packageDir, "package.json"); if (!host.isFile(manifestPath)) return undefined; try { - const manifest = JSON.parse(host.readFileSync(manifestPath, "utf8")); + const text = host.readFileSync(manifestPath, "utf8"); + // Node drops a byte order mark before parsing a package.json. + const manifest = JSON.parse(text.charCodeAt(0) === 0xfeff ? text.slice(1) : text); return manifest !== null && typeof manifest === "object" ? manifest : null; } catch { return null; // Node rejects a malformed package.json @@ -1144,7 +1238,10 @@ export function createRuntimeRequire( }; const packageTarget = (packageDir: string, target: string) => { const segments = targetSegments(target); - return segments === null ? null : projectFile(host.join(packageDir, ...segments)); + if (segments === null) return null; + const found = projectFile(host.join(packageDir, ...segments)); + // An import keeps the target's query or fragment in the module's URL. + return found === null ? null : { ...found, suffix: /[?#][\s\S]*$/.exec(target)?.[0] ?? "" }; }; const validExport = (target: string) => targetSegments(target) !== null; // An `imports` target may also name a package. @@ -1158,7 +1255,15 @@ export function createRuntimeRequire( let current = host.dirname(file); while (inProject(current) && !current.endsWith(host.sep + "node_modules")) { const manifest = readManifest(current); - if (manifest !== undefined) return manifest?.type; + // Node refuses to load a file whose package scope is malformed. + if (manifest === null) { + throw failure( + "Invalid package config " + + JSON.stringify(host.relative(projectRoot, host.join(current, "package.json"))), + "ERR_INVALID_PACKAGE_CONFIG", + ); + } + if (manifest !== undefined) return manifest.type; const parent = host.dirname(current); if (parent === current) return undefined; current = parent; @@ -1201,8 +1306,10 @@ export function createRuntimeRequire( const nameLength = id.startsWith("@") ? 2 : 1; if ( parts.length < nameLength || - parts.some((part) => - part === "" || part === "." || part === ".." || + parts.some((part, index) => + part === ".." || + // The name itself; Node reads `pkg/`, `pkg//` and `pkg/.` as the package. + (index < nameLength && (part === "" || part === ".")) || // A Windows separator would let the name walk out of node_modules. (host.sep !== "/" && part.includes(host.sep)) ) @@ -1270,8 +1377,19 @@ export function createRuntimeRequire( require, }; modules[real] = module; - const source = host.readFileSync(real, "utf8"); try { + // Reading may fail too; the module then leaves the cache, as in Node. + let source = host.readFileSync(real, "utf8"); + // Node 24 strips TypeScript's types and runs the rest as JavaScript. + if (/\.c?ts$/i.test(real)) { + if (host.stripTypes === undefined) { + throw failure( + "TypeScript is not supported here: " + JSON.stringify(host.relative(projectRoot, real)), + "ERR_UNKNOWN_FILE_EXTENSION", + ); + } + source = host.stripTypes(source); + } // A hashbang is not JavaScript; Node ignores it too. host.compile(source.startsWith("#!") ? "//" + source : source, real).call( module.exports, @@ -1282,7 +1400,7 @@ export function createRuntimeRequire( directory, ); } catch (error) { - delete modules[real]; + if (modules[real] === module) delete modules[real]; throw error; } module.loaded = true; @@ -1305,7 +1423,7 @@ export function createRuntimeRequire( if (/\.node$/i.test(real)) { throw failure("Native addons are not supported in API routes: " + name, "ERR_DLOPEN_FAILED"); } - if (/\.m[jt]s$/i.test(real) || (/\.js$/i.test(real) && packageType(real) === "module")) { + if (/\.m[jt]s$/i.test(real) || (/\.(?:js|ts)$/i.test(real) && packageType(real) === "module")) { throw failure("require() of an ES module is not supported here: " + name, "ERR_REQUIRE_ESM"); } return evaluate(real); @@ -1333,8 +1451,22 @@ export function createRuntimeRequire( }) as RuntimeRequire; require.resolve = (id: unknown, options?: { paths?: unknown }) => { if (builtin(id) !== undefined) return id as string; - const found = findFrom(id, directory, options); + // A bundled module's import.meta.resolve asks with this key for its + // package maps' import conditions. + const asImport = options !== null && typeof options === "object" && + (options as Record)[Symbol.for("veryfront.importConditions")] === true; + importing = asImport; + let found: ReturnType; + try { + found = findFrom(id, directory, options); + } finally { + importing = false; + } if (found === null) throw missing(label(id)); + // For an import, the module's URL, as import.meta.resolve returns it. + if (asImport) { + return host.toFileUrl(found.real) + ("suffix" in found ? found.suffix : ""); + } // Node returns the real path, which also keys require.cache. return found.real; }; @@ -1346,23 +1478,55 @@ export function createRuntimeRequire( return require; }; const byDirectory: Record = Object.create(null); + const requireFrom = (directory: unknown) => { + if (typeof directory !== "string") return create(undefined); + return byDirectory[directory] ??= create(directory); + }; + // The real path of each bundled file, by the bundler's key; null when it + // is not a dependency file. + const bundledPaths: Record = Object.create(null); + const bundledPath = (key: string) => { + if (key in bundledPaths) return bundledPaths[key]!; + let real: string | null = null; + try { + // The bundler names files relative to its working directory, with + // symlinks followed; this keys them the way computed requires do. + const path = host.realpathSync(host.join(bundleRoot!, key)); + if (dependencyPrefixes.some((prefix) => path.startsWith(prefix))) real = path; + } catch { + // Not a file path, such as a key in another namespace. + } + return bundledPaths[key] = real; + }; return { // Route code's computed ids resolve from the project, as Node's would. require: create(projectRoot), - requireFrom: (directory) => { - if (typeof directory !== "string") return create(undefined); - return byDirectory[directory] ??= create(directory); - }, - // The bundler names files by their real path; this keys the module the - // same way even where a symlink above the project was kept. - register: (path, module) => { - let real = path; + requireFrom, + bundled: (key, body) => { + if (bundleRoot === undefined || typeof key !== "string") return null; + const real = bundledPath(key); + if (real === null) return null; + const cached = modules[real]; + if (cached !== undefined) return cached; + const directory = host.dirname(real); + const module: RuntimeModule = { + exports: {}, + id: real, + filename: real, + path: directory, + loaded: false, + children: [], + require: requireFrom(directory), + }; + modules[real] = module; try { - real = host.realpathSync(path); - } catch { - // A file the bundler read is there; keep its own name if it is not. + body.call(module.exports, module.exports, module); + } catch (error) { + if (modules[real] === module) delete modules[real]; + throw error; } - if (!(real in modules)) modules[real] = module; + module.loaded = true; + return module; }, }; } @@ -1377,43 +1541,88 @@ function onlyHelperCalls(code: string, name: string): boolean { return all === calls; } +// esbuild's helper that runs a bundled CommonJS file once, as esbuild 0.28 +// prints it. Its names may carry a numeric suffix, and a direct eval in the +// bundle renames its locals. {@link prependRuntimeRequire} routes it through +// the runtime module cache. +const COMMONJS_HELPER = new RegExp( + [ + "var (__commonJS\\d*) = \\((\\w+), (\\w+)\\) => function (__require\\d*)\\(\\) \\{\\n", + " try \\{\\n", + " return \\3 \\|\\| \\(0, \\2\\[(__getOwnPropNames\\d*)\\(\\2\\)\\[0\\]\\]\\)", + "\\(\\(\\3 = \\{ exports: \\{\\} \\}\\)\\.exports, \\3\\), \\3\\.exports;\\n", + " \\} catch \\((\\w+)\\) \\{\\n", + " throw \\3 = 0, \\6;\\n", + " \\}\\n", + "\\};", + ].join(""), +); + /** * Binds the module-level `require` that esbuild's output calls for anything it * did not bundle, to {@link createRuntimeRequire}. esbuild's ESM output never * declares a top-level `require` (it renames a route's own binding of that - * name), so the prologue cannot collide with route code. + * name), so the prologue cannot collide with route code. Each bundled + * CommonJS dependency then runs through the runtime module cache; see + * {@link RuntimeRequireBinding.bundled}. `bundleRoot` is the bundle's working + * directory, which its file names are relative to. */ export async function prependRuntimeRequire( code: string, projectRoot: string, dependencyDirectories: readonly string[], + bundleRoot: string, ): Promise { let prefix = "__vf_runtime"; while (code.includes(prefix)) prefix += "_"; - // Dependency code calls the helpers by fixed names, which route code may - // declare too. The prologue binds them under names the bundle does not use, + // Dependency code calls the helper by a fixed name, which route code may + // declare too. The prologue binds it under a name the bundle does not use, // and only the free references, which are dependency calls, are renamed. - const registry = `${prefix}_register`; const requireFrom = `${prefix}_require_from`; - const helpers: Record = { - [BUNDLED_MODULE_REGISTRY]: registry, - [BUNDLED_REQUIRE_FROM]: requireFrom, - }; - if (Object.keys(helpers).every((name) => onlyHelperCalls(code, name))) { - for (const [name, local] of Object.entries(helpers)) { - code = code.replace(new RegExp(`\\b${name}\\b`, "g"), local); - } + if (onlyHelperCalls(code, BUNDLED_REQUIRE_FROM)) { + code = code.replace(new RegExp(`\\b${BUNDLED_REQUIRE_FROM}\\b`, "g"), requireFrom); } else { - // Some other use of a helper's name, such as a declaration in route code: - // the parser tells free references apart, as the bundler does. + // Some other use of the name, such as a declaration in route code: the + // parser tells free references apart, as the bundler does. code = (await transform(code, { loader: "js", format: "esm", platform: "neutral", logLevel: "silent", - define: helpers, + define: { [BUNDLED_REQUIRE_FROM]: requireFrom }, })).code; } + const bundled = `${prefix}_bundled`; + if (/\bvar __commonJS\d* = /.test(code)) { + if (!COMMONJS_HELPER.test(code)) { + // Another bundler version: fail closed rather than run modules twice. + throw new Error("[API] handler build failed: unexpected CommonJS helper in the bundle"); + } + code = code.replace( + COMMONJS_HELPER, + ( + _whole, + helper: string, + cb: string, + mod: string, + inner: string, + ownNames: string, + error: string, + ) => + [ + `var ${helper} = (${cb}, ${mod}) => function ${inner}() {`, + ` const key = ${ownNames}(${cb})[0];`, + ` const managed = ${bundled}(key, ${cb}[key]);`, + " if (managed !== null) return managed.exports;", + " try {", + ` return ${mod} || (0, ${cb}[key])((${mod} = { exports: {} }).exports, ${mod}), ${mod}.exports;`, + ` } catch (${error}) {`, + ` throw ${mod} = 0, ${error};`, + " }", + "};", + ].join("\n"), + ); + } const host = [ `getBuiltinModule: (id) => ${prefix}_process.getBuiltinModule(id)`, `isFile: (path) => { try { return ${prefix}_fs.statSync(path).isFile(); } catch { return false; } }`, @@ -1421,21 +1630,24 @@ export async function prependRuntimeRequire( `readFileSync: ${prefix}_fs.readFileSync`, // Captured now: route code runs after the prologue. `compile: ((F) => (source) => new F("exports", "require", "module", "__filename", "__dirname", source))(Function)`, + `stripTypes: ${prefix}_process.getBuiltinModule("node:module")?.stripTypeScriptTypes`, `isAbsolute: ${prefix}_path.isAbsolute`, `relative: ${prefix}_path.relative`, `join: ${prefix}_path.join`, `dirname: ${prefix}_path.dirname`, `sep: ${prefix}_path.sep`, + `toFileUrl: (path) => ${prefix}_url.pathToFileURL(path).href`, ]; const lines = [ `import ${prefix}_process from "node:process";`, `import * as ${prefix}_fs from "node:fs";`, `import * as ${prefix}_path from "node:path";`, - `const { require, register: ${registry}, requireFrom: ${requireFrom} } = (${createRuntimeRequire.toString()})({ ${ + `import * as ${prefix}_url from "node:url";`, + `const { require, bundled: ${bundled}, requireFrom: ${requireFrom} } = (${createRuntimeRequire.toString()})({ ${ host.join(", ") } }, ${JSON.stringify(projectRoot)}, ${JSON.stringify([...PREFIX_ONLY_BUILTINS])}, ${ JSON.stringify(dependencyDirectories) - });`, + }, ${JSON.stringify(bundleRoot)});`, ]; // A hashbang is only valid as the first line. const hashbangEnd = code.startsWith("#!") ? code.indexOf("\n") + 1 : 0; diff --git a/src/routing/api/module-loader/loader.ts b/src/routing/api/module-loader/loader.ts index 2239d75897..e235e94485 100644 --- a/src/routing/api/module-loader/loader.ts +++ b/src/routing/api/module-loader/loader.ts @@ -2007,11 +2007,11 @@ function createProjectBoundaryPlugin( importMap: workerImportMap, }); } - // A dependency's JSON registers itself, so that a computed require() - // of the same file returns the same object. + // A dependency's JSON is a CommonJS module, which the runtime module + // cache shares with a computed require() of the same file. if (!executableModule && dependency) { return { - contents: prepareDependencyJson(source.contents, args.path), + contents: prepareDependencyJson(source.contents), loader: "js", resolveDir: pathHelper.dirname(source.logicalPath), }; @@ -2391,7 +2391,7 @@ function buildTranspiledModuleSource( logger.debug(`built handler ${projectModuleLabel(resolvedPath, projectDir)}`); const output = result.outputFiles?.[0]?.text ?? "export {}"; const js = compiled - ? await prependRuntimeRequire(output, canonicalRoot, dependencyDirs) + ? await prependRuntimeRequire(output, canonicalRoot, dependencyDirs, workingDir) : output; logger.debug(`transpiled size ${js.length} bytes`); diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index afecda6d9c..c880b6f947 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -2,6 +2,7 @@ import "#veryfront/schemas/_test-setup.ts"; import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { mkdir, readdir, readFile, realpath, stat, symlink, writeFile } from "node:fs/promises"; +import process from "node:process"; import { pathToFileURL } from "node:url"; import { withTempDir } from "#veryfront/testing/deno-compat.ts"; import * as esbuild from "veryfront/extensions/bundler"; @@ -70,7 +71,7 @@ function dependencyFilePlugin( if (!isDependencyFile(args.path)) return undefined; const contents = await readFile(args.path, "utf8"); if (args.path.endsWith(".json")) { - return { contents: prepareDependencyJson(contents, args.path), loader: "js" }; + return { contents: prepareDependencyJson(contents), loader: "js" }; } return { contents: await prepareDependencyFile({ @@ -135,6 +136,8 @@ async function bundleAndImport( result.outputFiles?.[0]?.text ?? "", projectDir, await dependencyDirectories(projectDir, hostFs), + // The bundler names files relative to its working directory. + process.cwd(), ); const file = `${projectDir}/handler.mjs`; await writeFile(file, code); @@ -417,6 +420,117 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { }); }); + it("bundles a direct-eval dependency beside one that imports an external package", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/usesub/package.json": JSON.stringify({ name: "usesub", main: "index.js" }), + "node_modules/usesub/index.js": 'exports.load = () => import("esmsub/feature");', + "node_modules/esmsub/package.json": JSON.stringify({ + name: "esmsub", + type: "module", + exports: { ".": "./index.js", "./feature": "./dist/feature.js" }, + }), + "node_modules/esmsub/index.js": 'export default "root";', + "node_modules/esmsub/dist/feature.js": 'export default "feature";', + "node_modules/evaler/package.json": JSON.stringify({ name: "evaler", main: "index.js" }), + "node_modules/evaler/index.js": [ + '"use strict";', + 'exports.plugin = eval("require")("./plugin").name;', + ].join("\n"), + "node_modules/evaler/plugin.js": 'exports.name = "plugin";', + }); + const mod = await bundleAndImport( + projectDir, + [ + 'import u from "usesub";', + 'import e from "evaler";', + "export const evaler = e.plugin;", + "export const feature = (await u.load()).default;", + ].join("\n"), + ["esmsub"], + ); + assertEquals({ ...mod }, { evaler: "plugin", feature: "feature" }); + }); + }); + + it("shares one module cache with computed requires while modules load", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/cyc/package.json": JSON.stringify({ name: "cyc", main: "a.js" }), + "node_modules/cyc/counter.js": "module.exports = { a: 0, c: 0 };", + "node_modules/cyc/a.js": [ + 'require("./counter").a++;', + "exports.early = true;", + // b requires this file by a computed path while it is still loading. + 'exports.sawPartial = require("./b").sawPartial;', + 'exports.runs = require("./counter").a;', + "exports.reload = () => {", + ' delete require.cache[require.resolve("./c")];', + ' return require("./c");', + "};", + 'exports.first = require("./c");', + 'exports.early2 = require(["./early", "js"].join(".")) === require("./early");', + ].join("\n"), + "node_modules/cyc/b.js": [ + 'var path = require("path");', + 'var a = require(path.join(__dirname, "a" + ".js"));', + "exports.sawPartial = a.early === true && a.runs === undefined;", + ].join("\n"), + "node_modules/cyc/c.js": 'module.exports = { n: ++require("./counter").c };', + // A top-level return leaves early; the module is still cached. + "node_modules/cyc/early.js": "exports.x = 1;\nif (exports.x) return;\nexports.y = 2;", + }); + const mod = await bundleAndImport( + projectDir, + 'import cyc from "cyc";\nexport const result = cyc;\n', + ); + const cyc = mod.result as Record & { reload(): { n: number } }; + assertEquals(cyc.sawPartial, true); + assertEquals(cyc.runs, 1); + assertEquals(cyc.first, { n: 1 }); + assertEquals(cyc.early2, true); + // Deleted from require.cache, the bundled file runs again, as in Node. + assertEquals(cyc.reload(), { n: 2 }); + }); + }); + + it("runs dependency files the way Node 24 loads them", async () => { + await inProject(async (projectDir) => { + await writeFiles(projectDir, { + "node_modules/node24/package.json": JSON.stringify({ name: "node24", main: "index.js" }), + "node_modules/node24/index.js": [ + 'var path = require("path");', + "exports.wrapper = [arguments.length, typeof arguments[1], path.basename(arguments[3])];", + 'exports.sync = require("dual").kind;', + 'try { require("./forced.cjs"); } catch (error) { exports.forced = error.name; }', + 'exports.typed = require(["./typed", "cts"].join("."));', + 'exports.bom = require("./bom/x.js");', + ].join("\n"), + "node_modules/node24/forced.cjs": "export default 1;", + "node_modules/node24/typed.cts": "const n: number = 2;\nmodule.exports = n;", + // A byte order mark does not make a package.json malformed. + "node_modules/node24/bom/package.json": '\uFEFF{"type":"module"}', + "node_modules/node24/bom/x.js": "export const kind = 'esm';", + "node_modules/dual/package.json": JSON.stringify({ + name: "dual", + exports: { ".": { "module-sync": "./sync.mjs", require: "./legacy.cjs" } }, + }), + "node_modules/dual/sync.mjs": 'export const kind = "sync";', + "node_modules/dual/legacy.cjs": 'exports.kind = "legacy";', + }); + const mod = await bundleAndImport( + projectDir, + 'import node24 from "node24";\nexport const result = node24;\n', + ); + const result = mod.result as Record; + assertEquals(result.wrapper, [5, "function", "index.js"]); + assertEquals(result.sync, "sync"); + assertEquals(result.forced, "SyntaxError"); + assertEquals(result.typed, 2); + assertEquals((result.bom as { kind: string }).kind, "esm"); + }); + }); + it("shares one module between a bundled require and a computed one", async () => { await inProject(async (projectDir) => { await writeFiles(projectDir, { @@ -508,7 +622,11 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { name: "meta", type: "module", main: "index.js", + // import.meta.resolve reads the import condition. + imports: { "#worker": { import: "./worker.mjs", require: "./worker.cjs" } }, }), + "node_modules/meta/worker.mjs": "export {};", + "node_modules/meta/worker.cjs": "", "node_modules/meta/index.js": [ "const meta = import.meta;", "const resolve = import.meta.resolve;", @@ -519,11 +637,19 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { 'export const builtin = import.meta.resolve("fs");', 'export const computedBuiltin = resolve(["node", "path"].join(":"));', 'export const sibling = import.meta.resolve("other");', - "let bare;", - 'try { resolve(["oth", "er"].join("")); } catch (error) { bare = error.code; }', - "export const computedBare = bare;", + 'export const computedBare = resolve(["oth", "er"].join(""));', + 'export const computedImport = resolve(["#work", "er"].join(""));', + 'export const suffixed = import.meta.resolve("suffixed/feature");', + "let missing;", + 'try { resolve(["not", "installed"].join("-")); } catch (error) { missing = error.code; }', + "export const computedMissing = missing;", ].join("\n"), "node_modules/other/package.json": JSON.stringify({ name: "other", main: "main.js" }), + "node_modules/suffixed/package.json": JSON.stringify({ + name: "suffixed", + exports: { "./feature": "./feature.js?mode=x" }, + }), + "node_modules/suffixed/feature.js": "export {};", "node_modules/other/main.js": "module.exports = 1;", }); const mod = await bundleAndImport( @@ -538,7 +664,11 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { builtin: "node:fs", computedBuiltin: "node:path", sibling: `${base}/other/main.js`, - computedBare: "ERR_MODULE_NOT_FOUND", + computedBare: `${base}/other/main.js`, + computedImport: `${base}/meta/worker.mjs`, + // Node keeps a package target's query. + suffixed: `${base}/suffixed/feature.js?mode=x`, + computedMissing: "ERR_MODULE_NOT_FOUND", }); }); }); From 34e7bf1438627f301fa8676ec01ee6c3a4c3e294 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 05:11:02 +0100 Subject: [PATCH 24/29] test(module-loader): expect TypeScript stripping only where the runtime has it Bun has no stripTypeScriptTypes; there the runtime require refuses the file with ERR_UNKNOWN_FILE_EXTENSION instead of running it. --- .../api/module-loader/compiled-cjs-bundling.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index c880b6f947..5348a60986 100644 --- a/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/tests/integration/semantic-unit-boundary/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -503,7 +503,7 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { "exports.wrapper = [arguments.length, typeof arguments[1], path.basename(arguments[3])];", 'exports.sync = require("dual").kind;', 'try { require("./forced.cjs"); } catch (error) { exports.forced = error.name; }', - 'exports.typed = require(["./typed", "cts"].join("."));', + 'try { exports.typed = require(["./typed", "cts"].join(".")); } catch (error) { exports.typed = error.code; }', 'exports.bom = require("./bom/x.js");', ].join("\n"), "node_modules/node24/forced.cjs": "export default 1;", @@ -526,7 +526,10 @@ describe("compiled-binary route bundling of CommonJS dependencies", () => { assertEquals(result.wrapper, [5, "function", "index.js"]); assertEquals(result.sync, "sync"); assertEquals(result.forced, "SyntaxError"); - assertEquals(result.typed, 2); + // Deno, which the compiled binary runs on, strips types as Node 24 does; a + // runtime without the stripper refuses the file instead of running it. + const strips = typeof (await import("node:module")).stripTypeScriptTypes === "function"; + assertEquals(result.typed, strips ? 2 : "ERR_UNKNOWN_FILE_EXTENSION"); assertEquals((result.bom as { kind: string }).kind, "esm"); }); }); From 2706f75bec8f0cbb7df5d99c6ddaba7b2e070dc9 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Sat, 7 Nov 2026 05:24:45 +0100 Subject: [PATCH 25/29] fix(module-loader): keep Node's CommonJS scoping rules in bundled dependencies - A dependency that assigns require, plainly or compoundly, gets its own binding. - Files with extensions Node has no loader for are prepared as CommonJS. - #imports stop at node_modules; a package resolves its own name first. - import.meta.resolve returns package-map targets that do not exist yet. - require.resolve keeps Node's paths API, require.resolve.paths and module.paths. - An ES module JSON import with type json stays apart from require()'s module. - ES module dependencies see no require binding. - Generated code writes paths through codeString. --- .../ext-bundler-esbuild/src/plugin-adapter.ts | 1 + src/extensions/bundler/bundler.ts | 2 + .../compiled-cjs-bundling.test.ts | 89 +++++++- .../module-loader/compiled-cjs-bundling.ts | 193 ++++++++++++++---- src/routing/api/module-loader/loader.ts | 8 +- .../compiled-cjs-bundling.test.ts | 103 +++++++++- 6 files changed, 348 insertions(+), 48 deletions(-) diff --git a/extensions/ext-bundler-esbuild/src/plugin-adapter.ts b/extensions/ext-bundler-esbuild/src/plugin-adapter.ts index 2d909ef0b2..4544481feb 100644 --- a/extensions/ext-bundler-esbuild/src/plugin-adapter.ts +++ b/extensions/ext-bundler-esbuild/src/plugin-adapter.ts @@ -63,6 +63,7 @@ export function toEsbuildPlugin( namespace: args.namespace, suffix: args.suffix, pluginData: args.pluginData, + with: args.with, }; const result = await callback(loadArgs); if (result == null) return result ?? null; diff --git a/src/extensions/bundler/bundler.ts b/src/extensions/bundler/bundler.ts index 2567aed188..5a4f5ac715 100644 --- a/src/extensions/bundler/bundler.ts +++ b/src/extensions/bundler/bundler.ts @@ -234,6 +234,8 @@ export interface OnLoadArgs { namespace: string; suffix?: string; pluginData?: unknown; + /** Import attributes of the import that loads the file. */ + with?: Record; } /** Result returned from an `onLoad` callback. */ diff --git a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts index d83a8adc3b..d1d2ef77e6 100644 --- a/src/routing/api/module-loader/compiled-cjs-bundling.test.ts +++ b/src/routing/api/module-loader/compiled-cjs-bundling.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; import { type PluginBuild, stop as stopBundler, transform } from "veryfront/extensions/bundler"; import { builtinModuleName, + codeString, createCompiledBuiltinPlugin, createDependencyFilePredicate, createDependencyResolvePlugin, @@ -53,6 +54,15 @@ function errorCode(fn: () => unknown): unknown { } describe("routing/api/module-loader/compiled-cjs-bundling (inbox#4177)", () => { + describe("codeString()", () => { + it("writes a string literal that cannot end a script or a line", () => { + const value = "