diff --git a/config.json b/config.json index 18dfd74cd..1d601b3c2 100644 --- a/config.json +++ b/config.json @@ -109,6 +109,57 @@ "install" ] } + }, + ">=12.0.0": { + "url": "https://registry.npmjs.org/pnpm/-/pnpm-{}.tgz", + "bin": { + "pnpm": "pnpm", + "pnpx": "pnpx" + }, + "nativePackages": { + "win32-x64": { + "package": "@pnpm/exe.win32-x64", + "bin": "pnpm.exe" + }, + "win32-arm64": { + "package": "@pnpm/exe.win32-arm64", + "bin": "pnpm.exe" + }, + "darwin-x64": { + "package": "@pnpm/exe.darwin-x64", + "bin": "pnpm" + }, + "darwin-arm64": { + "package": "@pnpm/exe.darwin-arm64", + "bin": "pnpm" + }, + "linux-x64": { + "package": "@pnpm/exe.linux-x64", + "bin": "pnpm" + }, + "linux-arm64": { + "package": "@pnpm/exe.linux-arm64", + "bin": "pnpm" + }, + "linux-x64-musl": { + "package": "@pnpm/exe.linux-x64-musl", + "bin": "pnpm" + }, + "linux-arm64-musl": { + "package": "@pnpm/exe.linux-arm64-musl", + "bin": "pnpm" + } + }, + "registry": { + "type": "npm", + "package": "pnpm" + }, + "commands": { + "use": [ + "pnpm", + "install" + ] + } } } }, diff --git a/sources/corepackUtils.ts b/sources/corepackUtils.ts index 6c386bcec..75cfdb8bf 100644 --- a/sources/corepackUtils.ts +++ b/sources/corepackUtils.ts @@ -1,3 +1,5 @@ +import {spawn} from 'child_process'; +import {UsageError} from 'clipanion'; import {createHash} from 'crypto'; import {once} from 'events'; import fs from 'fs'; @@ -205,6 +207,129 @@ async function download(installTarget: string, url: string, algo: string, binPat }; } +interface DiagnosticReport { + header?: { + glibcVersionRuntime?: string; + }; +} + +// `excludeNetwork` is missing from the `@types/node` release we depend on. +type ProcessReport = NodeJS.ProcessReport & {excludeNetwork: boolean}; + +function detectLinuxLibcFamily(): `glibc` | `musl` | null { + if (process.platform !== `linux`) + return null; + + const processReport = process.report as ProcessReport | undefined; + if (processReport == null) + return null; + + // Gathering the network interfaces is the slowest part of generating a report, + // and we only care about the header: https://github.com/lovell/detect-libc/pull/21 + const {excludeNetwork} = processReport; + processReport.excludeNetwork = true; + + try { + // glibc builds expose `glibcVersionRuntime`; musl builds leave it unset. + const report = processReport.getReport() as DiagnosticReport; + return report.header?.glibcVersionRuntime ? `glibc` : `musl`; + } catch { + return null; + } finally { + processReport.excludeNetwork = excludeNetwork; + } +} + +function getBinNames(bin: BinSpec | BinList): Array { + return Array.isArray(bin) ? bin : Object.keys(bin); +} + +/** + * Whether all the binaries recorded for an install are present on disk. + */ +async function isNativeInstallIntact(installFolder: string, bin: BinSpec | BinList): Promise { + if (!isValidBinSpec(bin)) + return false; + + try { + await Promise.all(Object.values(bin).map(target => fs.promises.access(path.join(installFolder, target)))); + return true; + } catch { + return false; + } +} + +/** + * Puts the native executable in place of the placeholders shipped in `tmpFolder`, + * like the install lifecycle script of the package manager would have done. + */ +async function installNativeBinaries(installTarget: string, tmpFolder: string, locator: Locator, version: string, spec: PackageManagerSpec): Promise { + let platformKey = `${process.platform}-${process.arch}`; + if (detectLinuxLibcFamily() === `musl`) + platformKey += `-musl`; + + const nativePackage = spec.nativePackages![platformKey]; + if (nativePackage == null) + throw new UsageError(`${locator.name}@${version} does not ship a prebuilt executable for ${platformKey}`); + + // The main package pins the exact version of its companion packages in its + // `optionalDependencies`; if we can't read it, assume they share its version. + let nativeVersion = version; + try { + const manifest = JSON.parse(await fs.promises.readFile(path.join(tmpFolder, `package.json`), `utf8`)); + nativeVersion = manifest?.optionalDependencies?.[nativePackage.package] ?? version; + } catch {} + + const {tarball, signatures, integrity} = await npmRegistryUtils.fetchTarballURLAndSignature(nativePackage.package, nativeVersion); + + let url = tarball; + if (process.env.COREPACK_NPM_REGISTRY) { + url = url.replace( + npmRegistryUtils.DEFAULT_NPM_REGISTRY_URL, + () => process.env.COREPACK_NPM_REGISTRY!, + ); + } + + debugUtils.log(`Downloading native executable package ${nativePackage.package}@${nativeVersion} from ${url}`); + const {tmpFolder: nativeTmpFolder, hash: actualHash} = await download(installTarget, url, `sha512`); + + try { + if (!shouldSkipIntegrityCheck()) { + npmRegistryUtils.verifySignature({signatures, integrity, packageName: nativePackage.package, version: nativeVersion}); + + const expectedHash = Buffer.from(integrity.slice(`sha512-`.length), `base64`).toString(`hex`); + if (actualHash !== expectedHash) { + throw new Error(`Mismatch hashes. Expected ${expectedHash}, got ${actualHash}`); + } + } + + const nativeBinPath = path.join(nativeTmpFolder, nativePackage.bin); + const ext = process.platform === `win32` ? `.exe` : ``; + + const bin: BinSpec = {}; + for (const binName of getBinNames(spec.bin)) { + const target = `${binName}${ext}`; + const destPath = path.join(tmpFolder, target); + + // The executable adapts to the name it was invoked under, so the same file + // can replace the placeholder of each bin (e.g. `pnpx` = `pnpm dlx`). + await fs.promises.rm(destPath, {force: true}); + try { + await fs.promises.link(nativeBinPath, destPath); + } catch { + await fs.promises.copyFile(nativeBinPath, destPath); + } + await fs.promises.chmod(destPath, 0o755); + + bin[binName] = target; + } + + return bin; + } finally { + await fs.promises.rm(nativeTmpFolder, {recursive: true, force: true}); + } +} + export async function installVersion(installTarget: string, locator: Locator, {spec}: {spec: PackageManagerSpec}): Promise { const locatorIsASupportedPackageManager = isSupportedPackageManagerLocator(locator); const locatorReference = locatorIsASupportedPackageManager ? semverParse(locator.reference)! : parseURLReference(locator); @@ -218,13 +343,20 @@ export async function installVersion(installTarget: string, locator: Locator, {s const corepackData = JSON.parse(corepackContent); - debugUtils.log(`Reusing ${locator.name}@${locator.reference} found in ${installFolder}`); + if (locatorIsASupportedPackageManager && spec.nativePackages != null && !await isNativeInstallIntact(installFolder, corepackData.bin)) { + // Older Corepack releases didn't fetch the native executable, and recorded + // bins that don't exist; such installs have to be done anew. + debugUtils.log(`Discarding incomplete install of ${locator.name}@${locator.reference} found in ${installFolder}`); + await fs.promises.rm(installFolder, {recursive: true, force: true}); + } else { + debugUtils.log(`Reusing ${locator.name}@${locator.reference} found in ${installFolder}`); - return { - hash: corepackData.hash as string, - location: installFolder, - bin: corepackData.bin, - }; + return { + hash: corepackData.hash as string, + location: installFolder, + bin: corepackData.bin, + }; + } } catch (err) { if (nodeUtils.isNodeError(err) && err.code !== `ENOENT`) { throw err; @@ -308,6 +440,9 @@ export async function installVersion(installTarget: string, locator: Locator, {s if (build[1] && actualHash !== build[1]) throw new Error(`Mismatch hashes. Expected ${build[1]}, got ${actualHash}`); + if (locatorIsASupportedPackageManager && spec.nativePackages != null) + bin = await installNativeBinaries(installTarget, tmpFolder, locator, version, spec); + const serializedHash = `${algo}.${actualHash}`; await fs.promises.writeFile(path.join(tmpFolder, `.corepack`), JSON.stringify({ @@ -409,6 +544,13 @@ export async function runVersion(locator: Locator, installSpec: InstallSpec & {s if (!binPath) throw new Error(`Assertion failed: Unable to locate path for bin '${binName}'`); + process.env.COREPACK_ROOT = path.dirname(require.resolve(`corepack/package.json`)); + + if (installSpec.spec.nativePackages != null) { + await runNativeVersion(binPath, args); + return; + } + if (!Module.enableCompileCache) { // Node.js segfaults when using npm@>=9.7.0 and v8-compile-cache // $ docker run -it node:20.3.0-slim corepack npm@9.7.1 --version @@ -426,8 +568,6 @@ export async function runVersion(locator: Locator, installSpec: InstallSpec & {s // - Yarn uses process.argv[1] to determine its own path: https://github.com/yarnpkg/berry/blob/0da258120fc266b06f42aed67e4227e81a2a900f/packages/yarnpkg-cli/sources/main.ts#L80 // - pnpm uses `require.main == null` to determine its own version: https://github.com/pnpm/pnpm/blob/e2866dee92991e979b2b0e960ddf5a74f6845d90/packages/cli-meta/src/index.ts#L14 - process.env.COREPACK_ROOT = path.dirname(require.resolve(`corepack/package.json`)); - process.argv = [ process.execPath, binPath, @@ -447,6 +587,34 @@ export async function runVersion(locator: Locator, installSpec: InstallSpec & {s } } +/** + * Spawns a native executable, which cannot be loaded into the current process. + */ +async function runNativeVersion(binPath: string, args: Array): Promise { + const child = spawn(binPath, args, {stdio: `inherit`}); + + // Ctrl+C is delivered to the whole foreground process group, so the child gets + // it on its own; we only have to stay alive until it's done handling it. Other + // signals sent to Corepack are forwarded. + const onSigint = () => {}; + const forwardSignal = (signal: NodeJS.Signals) => { + child.kill(signal); + }; + + process.on(`SIGINT`, onSigint); + process.on(`SIGTERM`, forwardSignal); + + const [exitCode, signal] = await once(child, `exit`) as [number | null, NodeJS.Signals | null]; + + process.off(`SIGINT`, onSigint); + process.off(`SIGTERM`, forwardSignal); + + if (signal != null) + process.kill(process.pid, signal); + + process.exitCode = exitCode ?? 1; +} + export function shouldSkipIntegrityCheck() { return process.env.COREPACK_INTEGRITY_KEYS === `` || process.env.COREPACK_INTEGRITY_KEYS === `0`; diff --git a/sources/types.ts b/sources/types.ts index 9fca1dc40..fae40abff 100644 --- a/sources/types.ts +++ b/sources/types.ts @@ -36,6 +36,11 @@ export type RegistrySpec = | NpmRegistrySpec | UrlRegistrySpec; +export interface NativePackageSpec { + package: string; + bin: string; +} + /** * Defines how the package manager is meant to be downloaded and accessed. */ @@ -44,6 +49,13 @@ export interface PackageManagerSpec { bin: BinSpec | BinList; registry: RegistrySpec; npmRegistry?: NpmRegistrySpec; + /** + * Set when the package manager is distributed as a native executable, which + * `url` doesn't contain: it must be fetched from a companion package instead. + * Keys are `${process.platform}-${process.arch}`, with a `-musl` suffix on + * Linux systems using musl libc. + */ + nativePackages?: {[platformKey: string]: NativePackageSpec}; commands?: { use?: Array; }; diff --git a/tests/_registryServer.mjs b/tests/_registryServer.mjs index cb4553d04..cbd61a543 100644 --- a/tests/_registryServer.mjs +++ b/tests/_registryServer.mjs @@ -61,19 +61,69 @@ function createSimpleTarArchive(fileName, fileContent, mode = 0o644) { ]); } -const mockPackageTarGz = gzipSync(Buffer.concat([ - createSimpleTarArchive(`package/bin/customPkgManager.js`, `#!/usr/bin/env node\nconsole.log("customPkgManager: Hello from custom registry");\n`, 0o755), - createSimpleTarArchive(`package/bin/pnpm.js`, `#!/usr/bin/env node\nconsole.log("pnpm: Hello from custom registry");\n`, 0o755), - createSimpleTarArchive(`package/bin/yarn.js`, `#!/usr/bin/env node\nconsole.log("yarn: Hello from custom registry");\n`, 0o755), - createSimpleTarArchive(`package/package.json`, JSON.stringify({bin: {yarn: `bin/yarn.js`, pnpm: `bin/pnpm.js`, customPkgManager: `bin/customPkgManager.js`}})), - Buffer.alloc(1024), -])); -const shasum = createHash(`sha1`).update(mockPackageTarGz).digest(`hex`); -const integrity = `sha512-${createHash(`sha512`).update( - process.env.TEST_INTEGRITY === `invalid_integrity` ? - mockPackageTarGz.subarray(1) : - mockPackageTarGz, -).digest(`base64`)}`; +function createPackageArchive(entries) { + const tarGz = gzipSync(Buffer.concat([ + ...entries.map(([fileName, fileContent, mode]) => createSimpleTarArchive(fileName, fileContent, mode)), + Buffer.alloc(1024), + ])); + return { + tarGz, + shasum: createHash(`sha1`).update(tarGz).digest(`hex`), + integrity: `sha512-${createHash(`sha512`).update( + process.env.TEST_INTEGRITY === `invalid_integrity` ? + tarGz.subarray(1) : + tarGz, + ).digest(`base64`)}`, + }; +} + +const defaultPackageArchive = createPackageArchive([ + [`package/bin/customPkgManager.js`, `#!/usr/bin/env node\nconsole.log("customPkgManager: Hello from custom registry");\n`, 0o755], + [`package/bin/pnpm.js`, `#!/usr/bin/env node\nconsole.log("pnpm: Hello from custom registry");\n`, 0o755], + [`package/bin/yarn.js`, `#!/usr/bin/env node\nconsole.log("yarn: Hello from custom registry");\n`, 0o755], + [`package/package.json`, JSON.stringify({bin: {yarn: `bin/yarn.js`, pnpm: `bin/pnpm.js`, customPkgManager: `bin/customPkgManager.js`}})], +]); + +// pnpm v12 ships placeholders, and its real executable lives in a +// platform-specific package pinned in its `optionalDependencies`. +let nativePlatformKey = `${process.platform}-${process.arch}`; +if (process.platform === `linux`) { + try { + const report = process.report?.getReport(); + if (report != null && !report.header?.glibcVersionRuntime) { + nativePlatformKey += `-musl`; + } + } catch {} +} +const PNPM_V12_VERSION = `12.9998.9999`; +const pnpmExePackageName = `@pnpm/exe.${nativePlatformKey}`; +const pnpmExeBinName = process.platform === `win32` ? `pnpm.exe` : `pnpm`; + +const pnpmV12Archive = createPackageArchive([ + [`package/pnpm`, `This is a placeholder replaced by the native executable at install time.\n`], + [`package/pnpx`, `#!/bin/sh\nexec pnpm dlx "$@"\n`, 0o755], + [`package/package.json`, JSON.stringify({ + name: `pnpm`, + version: PNPM_V12_VERSION, + bin: {pnpm: `pnpm`, pnpx: `pnpx`}, + optionalDependencies: {[pnpmExePackageName]: PNPM_V12_VERSION}, + })], +]); +// Stands in for the native executable, printing the name it was invoked under. +const pnpmExeArchive = createPackageArchive([ + [`package/${pnpmExeBinName}`, `#!/bin/sh\necho "pnpm v12 native: $(basename "$0") $@"\n`, 0o755], + [`package/package.json`, JSON.stringify({name: pnpmExePackageName, version: PNPM_V12_VERSION})], +]); + +const packageArchives = { + __proto__: null, + [`pnpm@${PNPM_V12_VERSION}`]: pnpmV12Archive, + [`${pnpmExePackageName}@${PNPM_V12_VERSION}`]: pnpmExeArchive, +}; + +function getPackageArchive(packageName, version) { + return packageArchives[`${packageName}@${version}`] ?? defaultPackageArchive; +} const registry = { __proto__: null, @@ -84,8 +134,15 @@ const registry = { customPkgManager: [`1.0.0`], }; +if (process.env.TEST_PNPM_V12 === `1`) { + // `latest` is the last item of each list, so the v12 pre-release must come first. + registry.pnpm.unshift(PNPM_V12_VERSION); + registry[pnpmExePackageName] = [PNPM_V12_VERSION]; +} + function generateSignature(packageName, version) { if (privateKey == null) return undefined; + const {integrity} = getPackageArchive(packageName, version); const sign = createSign(`SHA256`).end(`${packageName}@${version}:${integrity}`); return {integrity, signatures: [{ keyid, @@ -93,6 +150,7 @@ function generateSignature(packageName, version) { }]}; } function generateVersionMetadata(packageName, version) { + const archive = getPackageArchive(packageName, version); return { name: packageName, version, @@ -100,8 +158,8 @@ function generateVersionMetadata(packageName, version) { [packageName]: `./bin/${packageName}.js`, }, dist: { - shasum, - size: mockPackageTarGz.length, + shasum: archive.shasum, + size: archive.tarGz.length, tarball: `https://registry.npmjs.org/${packageName}/-/${packageName}-${version}.tgz`, ...generateSignature(packageName, version), }, @@ -152,7 +210,7 @@ const server = createServer((req, res) => { if (registry[packageName].includes(version)) { res.end( isDownloadingRequest ? - mockPackageTarGz : + getPackageArchive(packageName, version).tarGz : JSON.stringify(generateVersionMetadata(packageName, version)), ); } else { diff --git a/tests/main.test.ts b/tests/main.test.ts index fac93914d..d5c4db253 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -1322,6 +1322,41 @@ it(`should download latest pnpm from custom registry`, async () => { }); }); +it(`should install the native executable of pnpm v12 from its platform-specific package`, async t => { + // The fake native executable is a shell script, which Windows cannot spawn. + if (process.platform === `win32`) t.skip(); + + await xfs.mktempPromise(async cwd => { + process.env.AUTH_TYPE = `COREPACK_NPM_TOKEN`; // See `_registryServer.mjs` + process.env.TEST_INTEGRITY = `valid`; // See `_registryServer.mjs` + process.env.TEST_PNPM_V12 = `1`; // See `_registryServer.mjs` + + await xfs.writeJsonPromise(ppath.join(cwd, `package.json` as Filename), { + packageManager: `pnpm@12.9998.9999`, + }); + + await expect(runCli(cwd, [`pnpm`, `install`], true)).resolves.toMatchObject({ + exitCode: 0, + stdout: `pnpm v12 native: pnpm install\n`, + stderr: ``, + }); + + // The aliases are hardlinked onto the very same executable. + await expect(runCli(cwd, [`pnpx`, `create-foo`], true)).resolves.toMatchObject({ + exitCode: 0, + stdout: `pnpm v12 native: pnpx create-foo\n`, + stderr: ``, + }); + + // Should keep working with cache + await expect(runCli(cwd, [`pnpm`, `run`, `build`])).resolves.toMatchObject({ + exitCode: 0, + stdout: `pnpm v12 native: pnpm run build\n`, + stderr: ``, + }); + }); +}); + describe(`should pick up COREPACK_INTEGRITY_KEYS from env`, () => { beforeEach(() => { process.env.AUTH_TYPE = `COREPACK_NPM_TOKEN`; // See `_registryServer.mjs`