Share exact retained function and class source ranges

Cargo.toml

@@ -394,6 +394,7 @@ libc = "0.2"
 lazy_static = "1.5"
 chrono = "0.4"
 regex = "1.12"
+aho-corasick = "1.1"
 regress = "0.11.1"
 hex = "0.4"
 tempfile = "3"

changelog.d/10065-retained-source-range-pool.md

@@ -0,0 +1,2 @@
+- Share exact nested function, closure, class and method source bytes within each compiled module. Reflection retains the original byte lengths, strictness metadata, registration order and copying/static ownership APIs; bounded matching falls back to independent constants.
+- Add deterministic range/budget tests, emitted-IR ownership/registration coverage and an application-independent reflection/GC gap fixture with a bounded default/compact native driver. Uses the already-locked aho-corasick dependency; no runtime ABI change or application-specific source rewriting.

crates/perry-codegen/Cargo.toml

@@ -46,6 +46,7 @@ perry-dispatch.workspace = true
 perry-api-manifest.workspace = true
 
 anyhow.workspace = true
+aho-corasick.workspace = true
 thiserror.workspace = true
 log.workspace = true
 serde.workspace = true

crates/perry-codegen/src/codegen/emission_order_tests.rs

@@ -492,3 +492,121 @@ fn registration_spelling_follows_output_kind() {
         );
     }
 }
+
+/// Exercise the real string-pool emitter, including class/method source,
+/// nonzero UTF-8 byte offsets, explicit lengths, registration order and both
+/// lifetime contracts. The padding crosses the production matcher's threshold.
+#[test]
+fn retained_source_ranges_preserve_registrations_and_ownership() {
+    let inner = "function inner() { return 1; }";
+    let method = "m() { return 1; }";
+    let class = format!("class C {{ {method} }}");
+    let outer = format!(
+        "function outer() {{ /*__retained_source_parent__{}\0*/ {inner} {class} }}",
+        "世界".repeat(700)
+    );
+    let make_module = || {
+        let mut module = empty_module("retained_source_ranges.ts");
+        module.functions.push(method_fn(100, "outer"));
+        module.functions.push(method_fn(101, "inner"));
+        module
+            .classes
+            .push(plain_class(3, "C", method_fn(200, "m")));
+        for (id, source, flag) in [
+            (100, outer.as_str(), true),
+            (101, inner, false),
+            (200, method, false),
+        ] {
+            module.closure_source_text.insert(
+                id,
+                perry_hir::FunctionSourceMetadata {
+                    text: source.to_owned(),
+                    is_non_strict_ordinary: flag,
+                },
+            );
+        }
+        module.class_source_text.insert(3, class.clone());
+        module
+    };
+    for output_type in ["executable", "dylib", "staticlib"] {
+        let emitted = ir_for_output_type(&make_module(), output_type);
+        assert_eq!(
+            emitted,
+            ir_for_output_type(&make_module(), output_type),
+            "deterministic source globals and registrations"
+        );
+        let source_globals: Vec<_> = emitted
+            .lines()
+            .filter(|line| {
+                line.starts_with('@') && line.contains("constant") && line.contains(inner)
+            })
+            .collect();
+        assert_eq!(
+            source_globals.len(),
+            1,
+            "nested sources must share one physical parent constant"
+        );
+        let base = source_globals[0].split_whitespace().next().unwrap();
+        let pointer = |source: &str| {
+            let offset = outer.find(source).unwrap();
+            if offset == 0 {
+                return base.to_owned();
+            }
+            let gep = emitted
+                .lines()
+                .find(|line| {
+                    line.contains("getelementptr")
+                        && line.contains(&format!("ptr {base}, i64 {offset}"))
+                })
+                .expect("nonzero source offset must actually be emitted");
+            gep.trim().split(" = ").next().unwrap().to_owned()
+        };
+        let register = if output_type == "executable" {
+            "js_register_function_source_static"
+        } else {
+            "js_register_function_source"
+        };
+        let calls: Vec<_> = emitted
+            .lines()
+            .filter(|line| line.contains("call void @js_register_function_source"))
+            .collect();
+        assert_eq!(
+            calls.len(),
+            3,
+            "two functions and one materialized method must be registered"
+        );
+        for (call, (symbol, source, flag)) in calls.iter().zip([
+            ("__outer", outer.as_str(), 1),
+            ("__inner", inner, 0),
+            ("__C__m", method, 0),
+        ]) {
+            assert!(call.contains(&format!("@{register}(")), "{call}");
+            assert!(
+                call.contains(&format!("{symbol},")),
+                "registration order/symbol changed: {call}"
+            );
+            assert!(
+                call.contains(&format!(
+                    "ptr {}, i32 {}, i32 {flag})",
+                    pointer(source),
+                    source.len()
+                )),
+                "source range/length/strictness changed: {call}"
+            );
+        }
+        let class_calls: Vec<_> = emitted
+            .lines()
+            .filter(|line| line.contains("call void @js_register_class_source("))
+            .collect();
+        assert_eq!(class_calls.len(), 1);
+        assert!(
+            class_calls[0].contains(&format!(
+                "(i32 3, ptr {}, i32 {})",
+                pointer(&class),
+                class.len()
+            )),
+            "class source must retain its copying API: {}",
+            class_calls[0]
+        );
+    }
+}

crates/perry-codegen/src/codegen/mod.rs

@@ -220,6 +220,7 @@ mod opts;
 #[cfg(test)]
 mod ordinary_param_guard_tests;
 mod param_guard;
+mod retained_source_pool;
 mod spec_abi;
 #[cfg(test)]
 mod spec_preserve_none_tests;

crates/perry-codegen/src/codegen/retained_source_pool.rs

@@ -0,0 +1,240 @@
+//! Share exact nested function/class source bytes, without changing runtime
+//! registration, strictness metadata or copying vs process-lifetime ownership.
+use std::collections::HashMap;
+
+use aho_corasick::{AhoCorasickBuilder, AhoCorasickKind};
+
+use crate::{
+    block::LlBlock,
+    module::LlModule,
+    types::{I64, I8},
+};
+
+const MIN_PATTERN_BYTES: usize = 4096;
+const MAX_PATTERN_BYTES: usize = 8 * 1024 * 1024;
+const MAX_MATCHES: usize = 1_000_000;
+
+#[derive(Clone)]
+pub(super) struct SourceRange {
+    global: String,
+    offset: usize,
+    pub(super) byte_len: usize,
+}
+
+impl SourceRange {
+    pub(super) fn pointer(&self, block: &mut LlBlock) -> String {
+        let base = format!("@{}", self.global);
+        if self.offset == 0 {
+            base
+        } else {
+            block.gep(I8, &base, &[(I64, &self.offset.to_string())])
+        }
+    }
+}
+
+pub(super) struct SourcePool<'a>(HashMap<&'a str, SourceRange>);
+
+impl<'a> SourcePool<'a> {
+    pub(super) fn emit(llmod: &mut LlModule, sources: impl Iterator<Item = &'a str>) -> Self {
+        // Intern before building the matcher. Hash iteration never determines
+        // emission order: first occurrence supplies every stable index.
+        let mut unique = Vec::new();
+        let mut seen = std::collections::HashSet::new();
+        for source in sources {
+            if seen.insert(source) {
+                unique.push(source);
+            }
+        }
+        let bytes: Vec<&[u8]> = unique.iter().map(|source| source.as_bytes()).collect();
+        let plan = plan(&bytes, MIN_PATTERN_BYTES, MAX_PATTERN_BYTES, MAX_MATCHES);
+        let mut globals = HashMap::new();
+        // Keep physical constants in original first-use order as well.
+        for (idx, &(parent, _)) in plan.iter().enumerate() {
+            if idx == parent {
+                globals.insert(idx, llmod.add_string_constant(unique[idx]).0);
+            }
+        }
+        Self(
+            unique
+                .into_iter()
+                .enumerate()
+                .map(|(idx, source)| {
+                    let (parent, offset) = plan[idx];
+                    (
+                        source,
+                        SourceRange {
+                            global: globals[&parent].clone(),
+                            offset,
+                            byte_len: source.len(),
+                        },
+                    )
+                })
+                .collect(),
+        )
+    }
+
+    pub(super) fn get(&self, source: &str) -> SourceRange {
+        self.0
+            .get(source)
+            .expect("retained source was prepared")
+            .clone()
+    }
+}
+
+/// Each result names an input parent and an exact byte offset. Budget/build
+/// failure keeps independent byte ranges, never a guessed match or missing text.
+fn plan(
+    input: &[&[u8]],
+    minimum_bytes: usize,
+    maximum_bytes: usize,
+    maximum_matches: usize,
+) -> Vec<(usize, usize)> {
+    let raw = || (0..input.len()).map(|idx| (idx, 0)).collect();
+    let total = input
+        .iter()
+        .fold(0usize, |sum, bytes| sum.saturating_add(bytes.len()));
+    if input.len() < 2
+        || total < minimum_bytes
+        || total > maximum_bytes
+        || input.iter().any(|bytes| bytes.is_empty())
+        || maximum_matches == 0
+    {
+        return raw();
+    }
+    // A contiguous NFA avoids the potentially much larger dense DFA. Pattern
+    // bytes and reported matches are bounded independently of source syntax.
+    let Ok(automaton) = AhoCorasickBuilder::new()
+        .kind(Some(AhoCorasickKind::ContiguousNFA))
+        .build(input)
+    else {
+        return raw();
+    };
+    let mut order: Vec<usize> = (0..input.len()).collect();
+    order.sort_by_key(|&idx| (std::cmp::Reverse(input[idx].len()), idx));
+    let mut locations = vec![None; input.len()];
+    let mut matches = 0;
+    for parent in order {
+        if locations[parent].is_some() {
+            continue;
+        }
+        locations[parent] = Some((parent, 0));
+        if matches == maximum_matches {
+            continue;
+        }
+        for found in automaton.find_overlapping_iter(input[parent]) {
+            let child = found.pattern().as_usize();
+            // The matcher reports byte offsets. No UTF-8 normalization,
+            // NUL termination or original-module coordinate inference occurs.
+            locations[child].get_or_insert((parent, found.start()));
+            matches += 1;
+            if matches == maximum_matches {
+                break;
+            }
+        }
+    }
+    locations
+        .into_iter()
+        .map(|location| location.expect("every source has a parent"))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn verify(
+        input: &[&[u8]],
+        minimum: usize,
+        maximum: usize,
+        matches: usize,
+    ) -> Vec<(usize, usize)> {
+        let result = plan(input, minimum, maximum, matches);
+        for (&expected, &(parent, offset)) in input.iter().zip(&result) {
+            assert_eq!(expected, &input[parent][offset..offset + expected.len()]);
+            assert_eq!(
+                result[parent],
+                (parent, 0),
+                "parents must not form alias chains"
+            );
+        }
+        result
+    }
+
+    #[test]
+    fn nested_unicode_nul_binary_and_deterministic_ties() {
+        let input: &[&[u8]] = &[
+            b"inner",
+            b"outer(inner)\0\xff",
+            b"(inner)",
+            "世界".as_bytes(),
+            "function 世界() { return '世界'; }".as_bytes(),
+            b"[inner]",
+        ];
+        let first = verify(input, 0, 1024, 100);
+        assert_eq!(first[0], (1, 6));
+        assert_eq!(first[2], (1, 5));
+        assert_eq!(first[3], (4, 9));
+        assert_eq!(first[5], (5, 0));
+        for _ in 0..16 {
+            assert_eq!(verify(input, 0, 1024, 100), first);
+        }
+    }
+
+    #[test]
+    fn budgets_and_empty_sources_keep_exact_fallbacks() {
+        let input: &[&[u8]] = &[b"a", b"aa", b"aaa", b"aaaa", b"b"];
+        for bound in [0, 1, 2, 3, 8, 100] {
+            verify(input, 0, 1024, bound);
+        }
+        let raw: Vec<_> = (0..input.len()).map(|idx| (idx, 0)).collect();
+        assert_eq!(verify(input, 1024, 2048, 100), raw);
+        assert_eq!(verify(input, 0, 1, 100), raw);
+        verify(&[b"", b"hello"], 0, 1024, 100);
+        verify(&[], 0, 1024, 100);
+    }
+
+    #[test]
+    fn emitted_pool_deduplicates_small_sources_without_a_matcher() {
+        let mut module = LlModule::new("aarch64-apple-darwin");
+        let pool = SourcePool::emit(&mut module, ["small", "other", "small", ""].into_iter());
+        assert_eq!(pool.get("small").global, ".str.0");
+        assert_eq!(pool.get("other").global, ".str.1");
+        assert_eq!(pool.get("").global, ".str.2");
+        assert_eq!(pool.0.len(), 3);
+        assert_eq!(pool.get("").byte_len, 0);
+        assert_eq!(
+            module
+                .to_ir()
+                .lines()
+                .filter(|line| line.contains("private unnamed_addr constant"))
+                .count(),
+            3
+        );
+    }
+
+    #[test]
+    fn emitted_pool_shares_nonzero_utf8_ranges_and_only_emits_parents() {
+        let child = "function 世界() { return '\\0'; }";
+        let parent = format!("/*{}\0*/{child}/*tail*/", "é".repeat(2200));
+        let mut module = LlModule::new("aarch64-apple-darwin");
+        let pool = SourcePool::emit(&mut module, [child, &parent, child].into_iter());
+        let range = pool.get(child);
+        assert_eq!(range.global, pool.get(&parent).global);
+        assert_eq!(range.offset, parent.find(child).unwrap());
+        assert_eq!(range.byte_len, child.len());
+        assert_eq!(
+            module
+                .to_ir()
+                .lines()
+                .filter(|line| line.contains("private unnamed_addr constant"))
+                .count(),
+            1
+        );
+    }
+
+    #[test]
+    fn equal_parents_choose_first_occurrence() {
+        let result = verify(&[b"middle", b"[middle]", b"(middle)"], 0, 1024, 100);
+        assert_eq!(result, [(1, 1), (1, 0), (2, 0)]);
+    }
+}