From 7798c129228dc77f0ca663d0a2d495da881ccc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 11 Sep 2026 11:05:10 +0200 Subject: [PATCH 1/4] perf(bun): compress embedded assets without changing their runtime bytes --- .../0-bun-embedded-asset-compression.md | 9 + crates/perry-runtime/src/embedded.rs | 5 + .../perry-runtime/src/embedded/compressed.rs | 214 ++++++++++++++++++ crates/perry/src/commands/compile/embed.rs | 33 ++- .../src/commands/compile/embed/compression.rs | 130 +++++++++++ .../src/commands/compile/run_pipeline.rs | 1 + .../perry/tests/bun_embedded_compression.rs | 21 ++ scripts/test-bun-embedded-compression.mjs | 108 +++++++++ 8 files changed, 516 insertions(+), 5 deletions(-) create mode 100644 changelog.d/0-bun-embedded-asset-compression.md create mode 100644 crates/perry-runtime/src/embedded/compressed.rs create mode 100644 crates/perry/src/commands/compile/embed/compression.rs create mode 100644 crates/perry/tests/bun_embedded_compression.rs create mode 100644 scripts/test-bun-embedded-compression.mjs diff --git a/changelog.d/0-bun-embedded-asset-compression.md b/changelog.d/0-bun-embedded-asset-compression.md new file mode 100644 index 0000000000..6e46de3178 --- /dev/null +++ b/changelog.d/0-bun-embedded-asset-compression.md @@ -0,0 +1,9 @@ +Compress worthwhile embedded payloads in Bun-platform executables with checksummed zstd, +using the existing Bun CLI runtime feature. Keep small/incompressible payloads +raw and require at least 256 KiB of aggregate payload savings before enabling +compressed registration. Generated constructors validate and decode one exact +frame into immortal native storage before registration, preserving paths, +lengths, empty/binary bytes and explicit text-loader metadata. Corrupt data or +length/loader mismatches fail before publishing an asset. Existing raw APIs and +non-Bun embedding remain unchanged. Independent decoder, packaging, and native +filesystem/Bun/text-loader tests cover the compressed path and raw controls. diff --git a/crates/perry-runtime/src/embedded.rs b/crates/perry-runtime/src/embedded.rs index a2975e17c7..0093200524 100644 --- a/crates/perry-runtime/src/embedded.rs +++ b/crates/perry-runtime/src/embedded.rs @@ -20,6 +20,11 @@ //! The global never frees (matching Perry's "embedded data lives for the life of //! the process" model), mirroring the `crate::shared_sab` registry pattern. +#[cfg(feature = "bun-cli-utils")] +mod compressed; +#[cfg(feature = "bun-cli-utils")] +pub use compressed::js_register_embedded_zstd_asset; + use std::collections::BTreeMap; use std::sync::{Mutex, OnceLock}; diff --git a/crates/perry-runtime/src/embedded/compressed.rs b/crates/perry-runtime/src/embedded/compressed.rs new file mode 100644 index 0000000000..ad776dfc9c --- /dev/null +++ b/crates/perry-runtime/src/embedded/compressed.rs @@ -0,0 +1,214 @@ +//! Byte-exact compressed payload registration before JavaScript/GC startup. +//! Only the Bun utility pack links zstd; ordinary embedding remains unchanged. + +use super::register_asset; + +fn decode(packed: &[u8], original_len: usize) -> Option> { + if original_len > isize::MAX as usize + || zstd::zstd_safe::get_frame_content_size(packed).ok()?? != original_len as u64 + || zstd::zstd_safe::find_frame_compressed_size(packed).ok()? != packed.len() + { + return None; + } + // Validate the frame's own length before reserving. Never let an unchecked + // caller length drive an allocation, or accept trailing/concatenated data. + let mut bytes = Vec::new(); + bytes.try_reserve_exact(original_len).ok()?; + let written = zstd::bulk::Decompressor::new() + .ok()? + .decompress_to_buffer(packed, &mut bytes) + .ok()?; + if written != original_len || bytes.len() != original_len { + return None; + } + Some(bytes.into_boxed_slice()) +} + +/// Register one compiler-emitted zstd frame, returning 1 on success and 0 on +/// invalid metadata, corrupt/truncated input, or a failed allocation/decode. +/// No entry is published on failure. The generated constructor treats 0 as +/// fatal rather than exposing compressed bytes as a file or silently omitting it. +/// +/// Names are copied by the existing registry; decoded native storage is kept +/// for the process lifetime, exactly like the raw API's immortal byte range. +/// This does not allocate JavaScript values or access an initialized GC. +/// +/// # Safety +/// Non-null pointers must name readable ranges of the supplied lengths for +/// this call. Compressed input need not outlive the call. `text_module` must be +/// 0 (ordinary file) or 1 (explicit text loader); other values are rejected. +#[no_mangle] +pub unsafe extern "C" fn js_register_embedded_zstd_asset( + name_ptr: *const u8, + name_len: usize, + packed_ptr: *const u8, + packed_len: usize, + original_len: usize, + text_module: u32, +) -> i32 { + if name_ptr.is_null() + || packed_ptr.is_null() + || name_len > isize::MAX as usize + || packed_len > isize::MAX as usize + || text_module > 1 + { + return 0; + } + let packed = std::slice::from_raw_parts(packed_ptr, packed_len); + let Some(decoded) = decode(packed, original_len) else { + return 0; + }; + let bytes = Box::leak(decoded); + register_asset( + name_ptr, + name_len, + bytes.as_ptr(), + bytes.len(), + text_module == 1, + ); + 1 +} + +#[cfg(feature = "keepalive-anchors")] +#[used] +static KEEP_REGISTER_ZSTD: unsafe extern "C" fn( + *const u8, + usize, + *const u8, + usize, + usize, + u32, +) -> i32 = js_register_embedded_zstd_asset; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_bytes_and_lengths_including_empty_and_non_utf8() { + for bytes in [ + &b""[..], + &b"plain\0text\xff\xc3\xa9"[..], + &vec![b'x'; 32768][..], + ] { + let packed = zstd::bulk::compress(bytes, 3).unwrap(); + assert_eq!(decode(&packed, bytes.len()).unwrap().as_ref(), bytes); + assert!(decode(&packed, bytes.len() + 1).is_none()); + if !bytes.is_empty() { + assert!(decode(&packed, bytes.len() - 1).is_none()); + } + } + } + + #[test] + fn malformed_truncated_trailing_and_concatenated_frames_are_rejected() { + let packed = zstd::bulk::compress(&vec![b'x'; 32768], 3).unwrap(); + for end in 0..packed.len() { + assert!(decode(&packed[..end], 32768).is_none(), "prefix {end}"); + } + let mut invalid = packed.clone(); + invalid[0] ^= 0xff; + assert!(decode(&invalid, 32768).is_none()); + let mut trailing = packed.clone(); + trailing.push(0); + assert!(decode(&trailing, 32768).is_none()); + let mut concatenated = packed.clone(); + concatenated.extend(zstd::bulk::compress(b"", 3).unwrap()); + assert!(decode(&concatenated, 32768).is_none()); + assert!(decode(&packed, usize::MAX).is_none()); + } + + #[test] + fn payload_checksum_damage_is_rejected() { + let mut compressor = zstd::bulk::Compressor::new(3).unwrap(); + compressor.include_checksum(true).unwrap(); + let bytes = vec![b'x'; 32768]; + let mut packed = compressor.compress(&bytes).unwrap(); + assert_eq!(decode(&packed, bytes.len()).unwrap().as_ref(), bytes); + *packed.last_mut().unwrap() ^= 1; + assert!(decode(&packed, bytes.len()).is_none()); + } + + #[test] + fn registry_preserves_loader_path_size_and_ownership() { + let bytes = b"retained\0bytes\xff"; + let mut packed = zstd::bulk::compress(bytes, 3).unwrap(); + unsafe { + for (name, text) in [ + ("compressed-test/file.bin", 0), + ("compressed-test/text.md", 1), + ] { + assert_eq!( + js_register_embedded_zstd_asset( + name.as_ptr(), + name.len(), + packed.as_ptr(), + packed.len(), + bytes.len(), + text + ), + 1 + ); + } + } + packed.fill(0); // Registry must not retain any borrow of the input. + drop(packed); + assert_eq!( + super::super::lookup("$perryfs/compressed-test/file.bin"), + Some(bytes.as_slice()) + ); + assert!(super::super::lookup_text_module("compressed-test/file.bin").is_none()); + assert_eq!( + super::super::lookup_text_module("compressed-test/text.md"), + Some(bytes.as_slice()) + ); + assert_eq!( + super::super::metadata("compressed-test/file.bin") + .unwrap() + .size, + bytes.len() + ); + } + + #[test] + fn invalid_registration_does_not_publish_a_partial_entry() { + let name = b"compressed-test/invalid"; + let packed = zstd::bulk::compress(b"value", 3).unwrap(); + unsafe { + assert_eq!( + js_register_embedded_zstd_asset( + name.as_ptr(), + name.len(), + packed.as_ptr(), + packed.len(), + 5, + 2 + ), + 0 + ); + assert_eq!( + js_register_embedded_zstd_asset( + name.as_ptr(), + name.len(), + packed.as_ptr(), + packed.len() - 1, + 5, + 0 + ), + 0 + ); + assert_eq!( + js_register_embedded_zstd_asset( + name.as_ptr(), + name.len(), + std::ptr::null(), + 0, + 5, + 0 + ), + 0 + ); + } + assert!(super::super::lookup("compressed-test/invalid").is_none()); + } +} diff --git a/crates/perry/src/commands/compile/embed.rs b/crates/perry/src/commands/compile/embed.rs index deb3d424ac..d1cf0f2632 100644 --- a/crates/perry/src/commands/compile/embed.rs +++ b/crates/perry/src/commands/compile/embed.rs @@ -32,6 +32,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::OnceLock; +mod compression; + /// Read loader metadata preserved by unbun. Extensions are not authoritative: /// Bun can embed `notes.md` using its text loader or its file/Markdown loaders. /// Validate before graph compilation, so a malformed sidecar fails promptly. @@ -363,6 +365,7 @@ pub(super) fn generate_embedded_asset_object( assets: &[(String, PathBuf)], output_dir: &Path, text_modules: &std::collections::HashSet, + bun_platform: bool, ) -> Result> { if assets.is_empty() { return Ok(None); @@ -376,6 +379,10 @@ pub(super) fn generate_embedded_asset_object( } else { "__perry_embedded_assets.o" }); + let payloads = compression::prepare(assets, output_dir, bun_platform)?; + let compressed = payloads + .iter() + .any(|payload| payload.original_len.is_some()); // Mach-O prefixes C symbols with `_` and names its read-only-const section // `__TEXT,__const`; ELF uses the bare symbol and `.rodata`. Perry runs on @@ -401,6 +408,10 @@ pub(super) fn generate_embedded_asset_object( if !text_modules.is_empty() { c.push_str("extern void js_register_embedded_text_asset(const char *name, size_t name_len, const char *bytes, size_t bytes_len);\n\n"); } + if compressed { + c.push_str("#include \n#include \n#include \n"); + c.push_str("extern int32_t js_register_embedded_zstd_asset(const char *, size_t, const char *, size_t, size_t, uint32_t);\n"); + } for (idx, (name, path)) in assets.iter().enumerate() { // Names are tiny — keep them as ASCII-clean C string literals. @@ -421,11 +432,18 @@ pub(super) fn generate_embedded_asset_object( // and end label so the C side recovers the length as a link-time // constant (end − start). `.incbin` needs an unambiguous path, so feed // it the canonical absolute path. - let abs = path + let abs = payloads[idx] + .path .canonicalize() .map_err(|e| anyhow!("failed to resolve embed asset {}: {}", path.display(), e))?; - let start = format!("{sym_prefix}PERRY_ASSET_DATA_{idx}"); - let end = format!("{sym_prefix}PERRY_ASSET_END_{idx}"); + // Keep compressed bytes out of the old raw-data symbol namespace. + let kind = if payloads[idx].original_len.is_some() { + "ZSTD_" + } else { + "" + }; + let start = format!("{sym_prefix}PERRY_ASSET_{kind}DATA_{idx}"); + let end = format!("{sym_prefix}PERRY_ASSET_{kind}END_{idx}"); // Assembler-level escape for the path inside `.incbin "..."`; `asm_line` // adds the C-string-literal escaping on top. let asm_path = abs @@ -440,8 +458,8 @@ pub(super) fn generate_embedded_asset_object( c.push_str(&asm_line(&format!(".globl {end}"))); c.push_str(&asm_line(&format!("{end}:"))); c.push_str(");\n"); - writeln!(c, "extern const char PERRY_ASSET_DATA_{idx}[];").ok(); - writeln!(c, "extern const char PERRY_ASSET_END_{idx}[];").ok(); + writeln!(c, "extern const char PERRY_ASSET_{kind}DATA_{idx}[];").ok(); + writeln!(c, "extern const char PERRY_ASSET_{kind}END_{idx}[];").ok(); } // Register before `main`'s `js_runtime_init`. Unix hosts use a priority @@ -458,6 +476,11 @@ pub(super) fn generate_embedded_asset_object( c.push_str("static void perry_register_embedded_assets(void) {\n"); } for idx in 0..assets.len() { + if let Some(original_len) = payloads[idx].original_len { + let text = u32::from(text_modules.contains(&assets[idx].0)); + writeln!(c, " if (!js_register_embedded_zstd_asset(PERRY_ASSET_NAME_{idx}, PERRY_ASSET_NAME_LEN_{idx}, PERRY_ASSET_ZSTD_DATA_{idx}, (size_t)(PERRY_ASSET_ZSTD_END_{idx} - PERRY_ASSET_ZSTD_DATA_{idx}), {original_len}, {text})) {{ fputs(\"Corrupt compressed embedded asset\\n\", stderr); _Exit(74); }}").ok(); + continue; + } let register = if text_modules.contains(&assets[idx].0) { "js_register_embedded_text_asset" } else { diff --git a/crates/perry/src/commands/compile/embed/compression.rs b/crates/perry/src/commands/compile/embed/compression.rs new file mode 100644 index 0000000000..de5f6c1d9f --- /dev/null +++ b/crates/perry/src/commands/compile/embed/compression.rs @@ -0,0 +1,130 @@ +//! Lossless Bun asset storage, using its already-selected zstd runtime pack. +use anyhow::{Context, Result}; +use std::fs; +use std::path::{Path, PathBuf}; + +// Avoid pulling a decoder into a tiny asset-only program for a handful of +// saved bytes. This is a storage threshold, not a total-executable size claim. +const MIN_TOTAL_SAVING: u64 = 256 * 1024; +const MIN_ASSET_BYTES: u64 = 4096; + +pub(super) struct Payload { + pub(super) path: PathBuf, + pub(super) original_len: Option, +} + +pub(super) fn prepare( + assets: &[(String, PathBuf)], + output_dir: &Path, + enabled: bool, +) -> Result> { + let mut payloads = Vec::with_capacity(assets.len()); + let mut saving = 0u64; + for (idx, (_, source)) in assets.iter().enumerate() { + let mut payload = Payload { + path: source.clone(), + original_len: None, + }; + if enabled && fs::metadata(source)?.len() >= MIN_ASSET_BYTES { + let bytes = + fs::read(source).with_context(|| format!("read asset {}", source.display()))?; + let mut compressor = zstd::bulk::Compressor::new(3)?; + compressor.include_checksum(true)?; + let packed = compressor + .compress(&bytes) + .context("compress embedded asset")?; + if packed.len().saturating_add(64) < bytes.len() { + let path = output_dir.join(format!("__perry_asset_{idx}.zst")); + fs::write(&path, &packed)?; + saving = saving.saturating_add((bytes.len() - packed.len()) as u64); + payload = Payload { + path, + original_len: Some(bytes.len()), + }; + } + } + payloads.push(payload); + } + if saving < MIN_TOTAL_SAVING { + // Keep every original raw symbol/path on the small-payload arm. + return Ok(assets + .iter() + .map(|(_, path)| Payload { + path: path.clone(), + original_len: None, + }) + .collect()); + } + Ok(payloads) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compresses_only_with_bun_and_a_material_aggregate_saving() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("large.bin"); + let bytes: Vec = (0..524288).map(|i| (i % 256) as u8).collect(); + fs::write(&source, &bytes).unwrap(); + let assets = vec![("large.bin".into(), source.clone())]; + let disabled = prepare(&assets, dir.path(), false).unwrap(); + assert!(disabled[0].original_len.is_none()); + assert_eq!(disabled[0].path, source); + let enabled = prepare(&assets, dir.path(), true).unwrap(); + assert_eq!(enabled[0].original_len, Some(bytes.len())); + let packed = fs::read(&enabled[0].path).unwrap(); + assert!(packed.len() < bytes.len() / 2); + assert_eq!(zstd::bulk::decompress(&packed, bytes.len()).unwrap(), bytes); + let mut damaged = packed; + *damaged.last_mut().unwrap() ^= 1; + assert!(zstd::bulk::decompress(&damaged, bytes.len()).is_err()); + assert_eq!(fs::read(&source).unwrap(), bytes); + } + + #[test] + fn small_assets_and_small_total_savings_keep_raw_bytes() { + let dir = tempfile::tempdir().unwrap(); + let tiny = dir.path().join("tiny.txt"); + let modest = dir.path().join("modest.txt"); + fs::write(&tiny, b"hello").unwrap(); + fs::write(&modest, vec![b'x'; 16384]).unwrap(); + let assets = vec![ + ("tiny.txt".into(), tiny.clone()), + ("modest.txt".into(), modest.clone()), + ]; + let result = prepare(&assets, dir.path(), true).unwrap(); + assert!(result.iter().all(|p| p.original_len.is_none())); + assert_eq!(result[0].path, tiny); + assert_eq!(result[1].path, modest); + } + + #[test] + fn incompressible_assets_remain_raw_beside_compressed_ones() { + let dir = tempfile::tempdir().unwrap(); + let compressible = dir.path().join("compressible.bin"); + let raw = dir.path().join("random.bin"); + fs::write(&compressible, vec![0u8; 524288]).unwrap(); + let mut state = 0x1234_5678u32; + let random: Vec = (0..524288) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + fs::write(&raw, &random).unwrap(); + let result = prepare( + &[("a".into(), compressible), ("b".into(), raw.clone())], + dir.path(), + true, + ) + .unwrap(); + assert!(result[0].original_len.is_some()); + assert!(result[1].original_len.is_none()); + assert_eq!(result[1].path, raw); + assert_eq!(fs::read(raw).unwrap(), random); + } +} diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 4622b1ba71..3a1900c400 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -6737,6 +6737,7 @@ pub fn run_with_parse_cache( &embedded_assets, &object_output_dir, &bunfs_text_modules, + ctx.bun_platform, )? { obj_cleanup_paths.push(obj.clone()); obj_paths.push(obj); diff --git a/crates/perry/tests/bun_embedded_compression.rs b/crates/perry/tests/bun_embedded_compression.rs new file mode 100644 index 0000000000..6bd5350928 --- /dev/null +++ b/crates/perry/tests/bun_embedded_compression.rs @@ -0,0 +1,21 @@ +//! Native coverage for compressed/raw assets and explicit Bun text loaders. +use std::{path::Path, process::Command}; + +#[test] +fn standalone_compressed_asset_regression() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let output = Command::new("node") + .arg(root.join("scripts/test-bun-embedded-compression.mjs")) + .env("PERRY_BIN", env!("CARGO_BIN_EXE_perry")) + .env("PERRY_WORKSPACE_ROOT", &root) + .env("PERRY_TEST_BUILD_RUNTIME", "1") + .current_dir(&root) + .output() + .expect("run bounded compression regression"); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/scripts/test-bun-embedded-compression.mjs b/scripts/test-bun-embedded-compression.mjs new file mode 100644 index 0000000000..c7424d3de4 --- /dev/null +++ b/scripts/test-bun-embedded-compression.mjs @@ -0,0 +1,108 @@ +// Independent native byte/loader regression. No extracted app or Bun install. +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { prepareRequireRuntime } from './test-require-runtime.mjs'; +const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +assert.equal(process.versions.node, fs.readFileSync(path.join(repo, '.node-version'), 'utf8').trim().replace(/^v/, '')); +prepareRequireRuntime(repo); +const compiler = process.env.PERRY_BIN ?? path.join(repo, 'target/perry-dev/perry'); +const work = fs.mkdtempSync(path.join(os.tmpdir(), 'perry-bun-embedded-compression-')); +const source = path.join(work, 'source'); +fs.mkdirSync(source); +const text = 'héllo\0世界\n'.repeat(32768); +const binary = Buffer.alloc(524288); +for (let i = 0; i < binary.length; i++) binary[i] = i % 256; +fs.writeFileSync(path.join(source, 'message.md'), text); +fs.writeFileSync(path.join(source, 'file-only.md'), text); +fs.writeFileSync(path.join(source, 'data.bin'), binary); +fs.writeFileSync(path.join(source, 'empty.txt'), ''); +fs.writeFileSync(path.join(source, 'tiny.txt'), 'tiny'); +fs.writeFileSync(path.join(source, 'package.json'), '{"type":"module","private":true}'); +fs.writeFileSync(path.join(source, 'unbun-manifest.json'), JSON.stringify({ + version: 1, runtime: 'bun', modules: [ + { path: '/$bunfs/root/message.md', extracted_path: 'message.md', loader: 13 }, + { path: '/$bunfs/root/file-only.md', extracted_path: 'file-only.md', loader: 5 }, + { path: '/$bunfs/root/empty.txt', extracted_path: 'empty.txt', loader: 13 }, + ], +})); +fs.writeFileSync(path.join(source, 'entry.js'), ` +import { readFileSync, statSync } from 'node:fs'; +import { createRequire } from 'node:module'; +const native = process.env.PERRY_EMBED_TEST === '1'; +const textPath = native ? '/$bunfs/root/message.md' : './message.md'; +const dataPath = native ? '/$bunfs/root/data.bin' : './data.bin'; +const tinyPath = native ? '/$bunfs/root/tiny.txt' : './tiny.txt'; +const emptyPath = native ? '/$bunfs/root/empty.txt' : './empty.txt'; +const expected = 'héllo\\0世界\\n'.repeat(32768); +const contents = readFileSync(textPath, 'utf8'); +if (contents !== expected) throw new Error('text changed'); +if (readFileSync(tinyPath, 'utf8') !== 'tiny') throw new Error('raw tiny asset changed'); +if (readFileSync(emptyPath, 'utf8') !== '') throw new Error('empty asset changed'); +const bytes = readFileSync(dataPath); +if (bytes.length !== 524288 || statSync(dataPath).size !== 524288) throw new Error('binary size changed'); +for (let i = 0; i < bytes.length; i++) if (bytes[i] !== i % 256) throw new Error('binary byte changed: ' + i); +if (native) { + const load = createRequire(import.meta.url); + if (load(textPath) !== expected || load(emptyPath) !== '') throw new Error('text loader changed'); + let rejected = false; + try { load('/$bunfs/root/file-only.md'); } catch (error) { rejected = error.code === 'MODULE_NOT_FOUND'; } + if (!rejected) throw new Error('file-only loader changed'); + if (await Bun.file(textPath).text() !== expected) throw new Error('Bun.file bytes changed'); + if (Bun.file(dataPath).size !== 524288) throw new Error('Bun.file size changed'); + globalThis.gc(); + if (load(textPath) !== expected || contents !== expected) throw new Error('retained text changed after GC'); +} +console.log('PASS: exact compressed/raw/empty bytes, sizes, text/file loaders and retained values'); +`); +function run(label, executable, args, cwd, env, timeout) { + const result = spawnSync(executable, args, { cwd, env, encoding: 'utf8', timeout, + maxBuffer: 8 * 1024 * 1024 }); + fs.writeFileSync(path.join(work, `${label}.stdout`), result.stdout ?? ''); + fs.writeFileSync(path.join(work, `${label}.stderr`), result.stderr ?? ''); + assert(!result.error, `${label}: ${result.error}`); + assert.equal(result.status, 0, `${label}\n${result.stdout}\n${result.stderr}`); + return result.stdout; +} +function tables(directory) { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => { + const filename = path.join(directory, entry.name); + return entry.isDirectory() ? tables(filename) : entry.name === '__perry_embedded_assets.c' ? [filename] : []; + }); +} +const env = { ...process.env }; +delete env.PERRY_EMBED_TEST; +const oracle = run('node', process.execPath, [path.join(source, 'entry.js')], source, env, 15000); +const results = []; +console.log('Evidence: ' + work); +for (const mode of ['default', 'compact']) { + const staging = path.join(work, mode); + fs.mkdirSync(staging); + const settings = { ...env, TMPDIR: staging, PERRY_LL_OPT_LEVEL: 'z' }; + for (const key of ['PERRY_RS4GC', 'PERRY_SHADOW_STACK', 'PERRY_INLINE_SHADOW_SLOT', 'PERRY_FULL_OUTLINE_IC']) delete settings[key]; + if (mode === 'compact') Object.assign(settings, { PERRY_RS4GC: '0', PERRY_SHADOW_STACK: '1', + PERRY_INLINE_SHADOW_SLOT: '0', PERRY_FULL_OUTLINE_IC: '1' }); + const output = path.join(work, mode + '-app'); + run(mode + '-compile', compiler, ['compile', path.join(source, 'entry.js'), + '-o', output, '--platform', 'bun', '--bunfs-root', source, '--keep-intermediates', + '--cache-dir', path.join(staging, 'cache'), '--no-auto-optimize', '--no-color', + ...(process.env.PERRY_TEST_WASM === '1' ? ['--enable-wasm-runtime'] : [])], source, settings, 120000); + const emitted = tables(staging); + assert.equal(emitted.length, 1, 'the actual asset emitter must be retained'); + const c = fs.readFileSync(emitted[0], 'utf8'); + assert.equal((c.match(/if \(!js_register_embedded_zstd_asset\(/g) ?? []).length, 3, + 'three large payloads must actually use compression'); + assert(c.includes('PERRY_ASSET_DATA_'), 'raw tiny/empty payloads must coexist'); + const moved = path.join(work, 'source-away'); + fs.renameSync(source, moved); + try { + const actual = run(mode + '-native', output, [], work, { ...settings, PERRY_EMBED_TEST: '1' }, 15000); + assert.equal(actual, oracle); + } finally { fs.renameSync(moved, source); } + results.push({ mode, passed: true, bytes: fs.statSync(output).size, compressedAssets: 3 }); + console.log('PASS: ' + mode); +} +fs.writeFileSync(path.join(work, 'result.json'), JSON.stringify({ results }, null, 2)); From 38640070d859291a9bc0d146e478fa5a5096f075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 11 Sep 2026 11:06:11 +0200 Subject: [PATCH 2/4] docs: key embedded compression changeset to PR 10053 --- ...-compression.md => 10053-bun-embedded-asset-compression.md} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename changelog.d/{0-bun-embedded-asset-compression.md => 10053-bun-embedded-asset-compression.md} (96%) diff --git a/changelog.d/0-bun-embedded-asset-compression.md b/changelog.d/10053-bun-embedded-asset-compression.md similarity index 96% rename from changelog.d/0-bun-embedded-asset-compression.md rename to changelog.d/10053-bun-embedded-asset-compression.md index 6e46de3178..f6974a084c 100644 --- a/changelog.d/0-bun-embedded-asset-compression.md +++ b/changelog.d/10053-bun-embedded-asset-compression.md @@ -1,4 +1,5 @@ -Compress worthwhile embedded payloads in Bun-platform executables with checksummed zstd, +Compress worthwhile embedded payloads in Bun-platform executables with +checksummed zstd, using the existing Bun CLI runtime feature. Keep small/incompressible payloads raw and require at least 256 KiB of aggregate payload savings before enabling compressed registration. Generated constructors validate and decode one exact From d8c817ef1f69bf7121bd9fb880eb7908ecb57743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 11 Sep 2026 11:08:39 +0200 Subject: [PATCH 3/4] test(bun): prebuild the native compression regression provider graph --- .github/workflows/test.yml | 2 +- crates/perry-runtime/src/embedded.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80e63da524..ca55f2c63c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1480,7 +1480,7 @@ jobs: # RFC-2945 abort guards that a JS throw trips — the opposite of the # shipped semantics. See the longer note in `cargo-test`. if printf '%s\n' "$SUITES" | grep -qE '^(perry|perry-stdlib) '; then - if printf '%s\n' "$SUITES" | grep -qE '^perry (bun_text_modules|import_meta_require_value) '; then + if printf '%s\n' "$SUITES" | grep -qE '^perry (bun_text_modules|bun_embedded_compression|import_meta_require_value) '; then # Prepare all require providers in one graph, outside the fixture's # timeout. A second perry-dev graph inside cargo test exceeded its # ten-minute bound on fresh runners (#9989/#9990). diff --git a/crates/perry-runtime/src/embedded.rs b/crates/perry-runtime/src/embedded.rs index 0093200524..e21f992eb5 100644 --- a/crates/perry-runtime/src/embedded.rs +++ b/crates/perry-runtime/src/embedded.rs @@ -4,9 +4,9 @@ //! `[compile] embed` in perry.toml) bakes the matched files' bytes into the //! binary. The compiler emits a generated C object whose `constructor` calls //! [`js_register_embedded_asset`] once per file before `main` runs, populating -//! a process-global registry. The bytes themselves live in the binary's -//! read-only data (static C literals), so the registry only stores -//! `&'static [u8]` slices into them — no copy, no per-asset heap allocation. +//! a process-global registry. Raw payloads live in the binary's read-only data +//! without copying. Compressed Bun payloads are decoded at startup into immortal +//! native storage. Both are exposed as the same byte-exact `&'static [u8]`. //! //! Three consumers read the registry at runtime: //! * `fs.readFileSync` / `fs.readFile` — a `$perryfs/...` virtual path (or a @@ -41,8 +41,8 @@ pub const VIRTUAL_PREFIX: &str = "$perryfs/"; /// code can keep passing the original string to `node:fs` and `Bun.file()`. pub const BUNFS_ROOT_PREFIX: &str = "/$bunfs/root/"; -/// One embedded file. `bytes` points into the binary's read-only data and is -/// valid for the life of the process. +/// One embedded file. `bytes` points into read-only data or decoded native +/// storage and is valid for the life of the process. struct EmbeddedAsset { /// Registry key — the embed-relative path, e.g. `dist/index.html`. name: String, @@ -153,7 +153,7 @@ unsafe fn register_asset( } /// Look up an embedded asset's bytes by virtual path (`$perryfs/...`) or by its -/// embed-relative key. Returns the `'static` slice into the binary. This is the +/// embed-relative key. Returns the immortal decoded/raw byte slice. This is the /// authoritative presence test — a path is "embedded" iff this returns `Some`. pub fn lookup(path: &str) -> Option<&'static [u8]> { let key = normalize_key(path); From 7793a66e3e999e3404f01377978b29bdff35288c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 11 Sep 2026 11:48:24 +0200 Subject: [PATCH 4/4] ci: pin scoped native tests to the repository Node oracle --- .github/workflows/test.yml | 16 ++++++++++++++++ .../10053-bun-embedded-asset-compression.md | 2 ++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca55f2c63c..ab323a52cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1425,6 +1425,22 @@ jobs: } >> "$GITHUB_OUTPUT" fi + - name: Setup pinned Node.js for scoped native oracles + if: steps.scope.outputs.rust_work == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version-file: .node-version + + - name: Verify scoped Node oracle before native builds + if: steps.scope.outputs.rust_work == 'true' + run: | + node --input-type=module -e ' + import assert from "node:assert/strict"; + import fs from "node:fs"; + const expected = fs.readFileSync(".node-version", "utf8").trim().replace(/^v/, ""); + assert.equal(process.versions.node, expected, "Scoped native tests require the pinned Node oracle"); + ' + - name: Install Rust toolchain if: steps.scope.outputs.rust_work == 'true' run: rustup toolchain install nightly-2026-08-20 --profile minimal diff --git a/changelog.d/10053-bun-embedded-asset-compression.md b/changelog.d/10053-bun-embedded-asset-compression.md index f6974a084c..1a6d20eaec 100644 --- a/changelog.d/10053-bun-embedded-asset-compression.md +++ b/changelog.d/10053-bun-embedded-asset-compression.md @@ -8,3 +8,5 @@ lengths, empty/binary bytes and explicit text-loader metadata. Corrupt data or length/loader mismatches fail before publishing an asset. Existing raw APIs and non-Bun embedding remain unchanged. Independent decoder, packaging, and native filesystem/Bun/text-loader tests cover the compressed path and raw controls. +Install and verify the pinned Node oracle in scoped native CI before any +expensive build, instead of inheriting the runner's ambient Node version.