From 3aec3e2a2de27ec1cbd413ed91131b7679ed5e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 07:38:39 +0200 Subject: [PATCH 01/53] experiment(gc): prototype stack maps and statepoints --- crates/perry-codegen/src/codegen/helpers.rs | 41 + crates/perry-codegen/src/expr/shadow_slot.rs | 13 + crates/perry-codegen/src/function.rs | 732 +++++++++++++++++- crates/perry-codegen/src/module.rs | 51 ++ crates/perry-runtime/src/gc/mod.rs | 4 + crates/perry-runtime/src/gc/roots.rs | 3 + .../perry-runtime/src/gc/roots/stack_maps.rs | 528 +++++++++++++ .../perry/src/commands/compile/build_cache.rs | 2 + .../src/commands/compile/object_cache.rs | 15 +- .../object_cache/object_cache_tests.rs | 2 + docs/stack-map-gc-experiment.md | 234 ++++++ docs/statepoint-bridge-probe.ll | 36 + docs/statepoint-gc-experiment.md | 250 ++++++ 13 files changed, 1908 insertions(+), 3 deletions(-) create mode 100644 crates/perry-runtime/src/gc/roots/stack_maps.rs create mode 100644 docs/stack-map-gc-experiment.md create mode 100644 docs/statepoint-bridge-probe.ll create mode 100644 docs/statepoint-gc-experiment.md diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index f13844aee3..a9ae533752 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -74,6 +74,47 @@ pub(super) fn shadow_stack_enabled() -> bool { }) } +/// Research-only precise-root backend using LLVM's +/// `llvm.experimental.stackmap` intrinsic. +/// +/// `PERRY_STACK_MAPS=1` keeps the existing pointer-local/liveness analysis but +/// changes the storage and discovery mechanism: roots remain in their native +/// frame allocas and LLVM records those writable locations at call sites. The +/// runtime can then unwind the native stack and visit the exact slots without +/// a parallel heap-backed shadow stack. +/// +/// This is intentionally opt-in while the experiment establishes correctness, +/// target coverage, and performance. It is independent of +/// `PERRY_SHADOW_STACK=0`: the latter still disables precise-root analysis +/// entirely, while this selects the backend used when that analysis is on. +pub(crate) fn stack_maps_enabled() -> bool { + matches!( + std::env::var("PERRY_STACK_MAPS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) +} + +/// Research-only moving-GC backend using LLVM's explicit statepoint +/// relocation sequence. +/// +/// This is separate from `PERRY_STACK_MAPS` so the two native-stack +/// prototypes can be measured independently. Statepoint mode still consumes +/// LLVM's stack-map section at runtime, but supported calls are represented +/// by `gc.statepoint` / `gc.result` / `gc.relocate` instead of a standalone +/// metadata marker plus compiler memory barriers. +pub(crate) fn statepoints_enabled() -> bool { + matches!( + std::env::var("PERRY_STATEPOINTS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) +} + +/// Whether precise roots should use a native-stack metadata backend rather +/// than Perry's heap-backed shadow frame. +pub(crate) fn native_stack_roots_enabled() -> bool { + stack_maps_enabled() || statepoints_enabled() +} + /// Inline shadow-slot store gate (#7088). Default ON. /// /// When enabled, a store to a GC-rooted local is emitted as an address diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 4fca697ffd..fa2627a42c 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -197,6 +197,19 @@ pub(crate) fn emit_shadow_slot_bind_for_local(ctx: &mut FnCtx<'_>, local_id: u32 return; }; ctx.shadow_slots_bound.insert(slot_idx); + if crate::codegen::helpers::native_stack_roots_enabled() { + // Kept temporarily as a textual marker: LlFunction's final stack-map + // lowering records `slot_idx -> local_slot` and removes this call. + // The incremental root barrier remains real because the native slot + // can be updated after an in-flight cycle scanned this frame. + ctx.block().call_void( + "js_shadow_slot_bind", + &[(I32, &slot_idx.to_string()), (PTR, &local_slot)], + ); + let value_bits = ctx.block().load(I64, &local_slot); + emit_persistent_shadow_root_barrier(ctx, &value_bits); + return; + } // #7088: the hot per-store root write. Emitted inline against this // activation's cached `ShadowStackState` pointer when it has one; falls // through to the call otherwise. diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 5f55a76c45..81250e3e8b 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -127,6 +127,14 @@ pub struct LlFunction { shadow_frame_push: Option, /// Slot count currently baked into that push line. shadow_frame_slot_count: u32, + /// Research backend: preserve the existing precise-root slot numbering, + /// but encode the slots in LLVM stack maps instead of allocating a + /// parallel runtime shadow frame. + stack_map_requested: bool, + /// Logical root slots reserved by the existing liveness analysis. The + /// final IR pass resolves these indices to the native allocas named by + /// `js_shadow_slot_bind` calls, removes the calls, and emits stack maps. + stack_map_slot_count: u32, /// Runtime hooks emitted immediately before each non-pointer `ret`. /// Entry/module-init functions use this for process-level diagnostics /// that must run regardless of which block reaches the normal epilogue. @@ -227,6 +235,8 @@ impl LlFunction { shadow_frame_post_init_region: false, shadow_frame_push: None, shadow_frame_slot_count: 0, + stack_map_requested: false, + stack_map_slot_count: 0, pre_return_void_calls: Vec::new(), } } @@ -267,6 +277,13 @@ impl LlFunction { } fn enable_shadow_frame_inner(&mut self, slot_count: u32, post_init: bool) { + if crate::codegen::helpers::native_stack_roots_enabled() { + self.shadow_frame_requested = true; + self.shadow_frame_post_init_region = post_init; + self.stack_map_requested = slot_count != 0; + self.stack_map_slot_count = slot_count; + return; + } if self.shadow_frame_slot.is_some() { return; } @@ -354,6 +371,12 @@ impl LlFunction { if !self.shadow_frame_requested { return None; } + if crate::codegen::helpers::native_stack_roots_enabled() { + let idx = self.stack_map_slot_count; + self.stack_map_slot_count += 1; + self.stack_map_requested = true; + return Some(idx); + } if self.shadow_frame_push.is_none() { let post_init = self.shadow_frame_post_init_region; self.emit_shadow_frame_push(0, post_init); @@ -612,9 +635,17 @@ impl LlFunction { } else { "" }; + let gc_strategy = if self.stack_map_requested + && crate::codegen::helpers::statepoints_enabled() + && !self.has_try + { + " gc \"statepoint-example\"" + } else { + "" + }; let mut ir = format!( - "define {}{} @{}({}){} {{\n", - linkage, self.return_type, self.name, param_str, attrs + "define {}{} @{}({}){}{} {{\n", + linkage, self.return_type, self.name, param_str, attrs, gc_strategy ); for (i, blk) in self.blocks.iter().enumerate() { @@ -714,6 +745,20 @@ impl LlFunction { ir }; + // Research backend: turn the existing shadow-slot binding IR into + // native-frame stack maps only after lowering is complete, when every + // lazily-reserved scalar root and every call site is visible. + let ir = if self.stack_map_requested { + let backend = if crate::codegen::helpers::statepoints_enabled() && !self.has_try { + PreciseRootBackend::Statepoint + } else { + PreciseRootBackend::StackMap + }; + lower_precise_roots_to_native_stack(&ir, self.stack_map_slot_count, backend) + } else { + ir + }; + // setjmp volatile promotion (#6385). // // Runs LAST so it sees every instruction, including the ones the @@ -733,3 +778,686 @@ impl LlFunction { ir } } + +fn parse_shadow_bind(line: &str) -> Option<(usize, String)> { + let rest = line + .trim() + .strip_prefix("call void @js_shadow_slot_bind(i32 ")?; + let (idx, ptr) = rest.split_once(", ptr ")?; + let ptr = ptr.strip_suffix(')')?.trim(); + Some((idx.parse().ok()?, ptr.to_string())) +} + +fn parse_shadow_set(line: &str) -> Option<(usize, String)> { + let rest = line + .trim() + .strip_prefix("call void @js_shadow_slot_set(i32 ")?; + let (idx, value) = rest.split_once(", i64 ")?; + let value = value.strip_suffix(')')?.trim(); + Some((idx.parse().ok()?, value.to_string())) +} + +/// Compute a conservative set of active logical shadow slots before each IR +/// line. Joins use union ("active on any incoming path"), so a stale local can +/// be retained but a live root cannot be omitted. +fn stack_map_active_slots( + lines: &[&str], + slot_count: u32, +) -> Vec>> { + use std::collections::{HashMap, HashSet, VecDeque}; + + #[derive(Debug)] + struct Block { + first_line: usize, + end_line: usize, + successors: Vec, + } + + fn label_name(line: &str) -> Option<&str> { + if line.starts_with(char::is_whitespace) { + return None; + } + line.strip_suffix(':') + .filter(|name| !name.is_empty() && !name.starts_with(';')) + } + + fn referenced_labels(line: &str) -> Vec<&str> { + let mut labels = Vec::new(); + let mut rest = line; + while let Some(pos) = rest.find("label %") { + let after = &rest[pos + "label %".len()..]; + let len = after + .bytes() + .take_while(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b'$') + }) + .count(); + if len == 0 { + break; + } + labels.push(&after[..len]); + rest = &after[len..]; + } + labels + } + + let labels: Vec<(usize, &str)> = lines + .iter() + .enumerate() + .filter_map(|(idx, line)| label_name(line).map(|name| (idx, name))) + .collect(); + let mut states = vec![None; lines.len()]; + if labels.is_empty() { + return states; + } + + let label_to_block: HashMap<&str, usize> = labels + .iter() + .enumerate() + .map(|(block, (_, name))| (*name, block)) + .collect(); + let mut blocks: Vec = labels + .iter() + .enumerate() + .map(|(block, (label_line, _))| Block { + first_line: label_line + 1, + end_line: labels + .get(block + 1) + .map_or(lines.len(), |(next_line, _)| *next_line), + successors: Vec::new(), + }) + .collect(); + for block in &mut blocks { + let mut seen = HashSet::new(); + for line in &lines[block.first_line..block.end_line] { + for label in referenced_labels(line) { + if let Some(&successor) = label_to_block.get(label) { + if seen.insert(successor) { + block.successors.push(successor); + } + } + } + } + } + + fn apply_root_op(state: &mut HashSet, line: &str, slot_count: u32) { + if let Some((idx, _)) = parse_shadow_bind(line) { + if idx < slot_count as usize { + state.insert(idx); + } + } else if let Some((idx, value)) = parse_shadow_set(line) { + if idx < slot_count as usize { + if value == "0" { + state.remove(&idx); + } else { + state.insert(idx); + } + } + } + } + + let mut entries: Vec>> = vec![None; blocks.len()]; + entries[0] = Some(HashSet::new()); + let mut work = VecDeque::from([0usize]); + while let Some(block_idx) = work.pop_front() { + let Some(mut state) = entries[block_idx].clone() else { + continue; + }; + let block = &blocks[block_idx]; + for line in &lines[block.first_line..block.end_line] { + apply_root_op(&mut state, line, slot_count); + } + for &successor in &block.successors { + let changed = match &mut entries[successor] { + Some(existing) => { + let old_len = existing.len(); + existing.extend(state.iter().copied()); + existing.len() != old_len + } + entry @ None => { + *entry = Some(state.clone()); + true + } + }; + if changed { + work.push_back(successor); + } + } + } + + for (block_idx, block) in blocks.iter().enumerate() { + let Some(mut state) = entries[block_idx].clone() else { + continue; + }; + for (line_idx, line) in lines + .iter() + .enumerate() + .take(block.end_line) + .skip(block.first_line) + { + states[line_idx] = Some(state.clone()); + apply_root_op(&mut state, line, slot_count); + } + } + states +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PreciseRootBackend { + StackMap, + Statepoint, +} + +#[derive(Debug, Eq, PartialEq)] +struct DirectCall<'a> { + result: Option<&'a str>, + return_type: &'a str, + callee: &'a str, + args: Vec<&'a str>, + arg_types: Vec<&'a str>, +} + +fn split_call_args(args: &str) -> Option> { + if args.trim().is_empty() { + return Some(Vec::new()); + } + let mut out = Vec::new(); + let mut depth = 0i32; + let mut start = 0usize; + for (idx, ch) in args.char_indices() { + match ch { + '(' | '[' | '{' | '<' => depth += 1, + ')' | ']' | '}' | '>' => { + depth -= 1; + if depth < 0 { + return None; + } + } + ',' if depth == 0 => { + out.push(args[start..idx].trim()); + start = idx + 1; + } + _ => {} + } + } + if depth != 0 { + return None; + } + out.push(args[start..].trim()); + Some(out) +} + +fn statepoint_scalar_type(arg: &str) -> Option<&str> { + let ty = arg.split_ascii_whitespace().next()?; + matches!( + ty, + "i1" | "i8" | "i16" | "i32" | "i64" | "i128" | "float" | "double" | "ptr" + ) + .then_some(ty) +} + +/// Parse the deliberately small direct-call subset emitted by `LlBlock`. +/// +/// Calls with tail markers, operand attributes, aggregate types, inline asm, +/// indirect targets, or call-site suffixes stay on the plain stack-map +/// fallback. That keeps the research mode correct while making its explicit +/// statepoint coverage measurable and easy to expand. +fn parse_direct_statepoint_call(line: &str) -> Option> { + let trimmed = line.trim(); + let (result, call) = if let Some(call) = trimmed.strip_prefix("call ") { + (None, call) + } else { + let (result, call) = trimmed.split_once(" = call ")?; + (Some(result.trim()), call) + }; + let (return_type, target_and_args) = call.split_once(' ')?; + if !matches!( + return_type, + "void" | "i1" | "i8" | "i16" | "i32" | "i64" | "i128" | "float" | "double" | "ptr" + ) { + return None; + } + if return_type != "void" && result.is_none() { + return None; + } + let open = target_and_args.find('(')?; + let close = target_and_args.rfind(')')?; + if close + 1 != target_and_args.len() { + return None; + } + let callee = target_and_args[..open].trim(); + if !callee.starts_with('@') + || callee.starts_with("@llvm.") + || matches!(callee, "@setjmp" | "@_setjmp" | "@longjmp" | "@_longjmp") + { + return None; + } + let args = split_call_args(&target_and_args[open + 1..close])?; + let arg_types = args + .iter() + .map(|arg| statepoint_scalar_type(arg)) + .collect::>>()?; + Some(DirectCall { + result, + return_type, + callee, + args, + arg_types, + }) +} + +fn gc_result_suffix(ty: &str) -> Option<&'static str> { + match ty { + "i1" => Some("i1"), + "i8" => Some("i8"), + "i16" => Some("i16"), + "i32" => Some("i32"), + "i64" => Some("i64"), + "i128" => Some("i128"), + "float" => Some("f32"), + "double" => Some("f64"), + "ptr" => Some("p0"), + _ => None, + } +} + +fn emit_plain_stack_map(out: &mut String, line: &str, live: &[&String], map_id: u64) { + let operands = live + .iter() + .map(|ptr| format!(", ptr {ptr}")) + .collect::(); + out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); + out.push_str(&format!( + " call void (i64, i32, ...) @llvm.experimental.stackmap(i64 {map_id}, i32 0{operands})\n" + )); + out.push_str(line); + out.push('\n'); + out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); +} + +/// Emit one explicit statepoint relocation sequence. +/// +/// Perry roots remain ordinary NaN-boxed `i64` values everywhere else. At +/// this boundary we load each live word, carry its exact bits through a +/// temporary addrspace(1) pointer, and convert the `gc.relocate` result back +/// into the existing slot. LLVM therefore owns the spill/reload and the +/// post-safepoint SSA transition without requiring a whole-program +/// representation change for this prototype. +fn emit_statepoint(out: &mut String, call: &DirectCall<'_>, live: &[&String], statepoint_id: u64) { + for (root_idx, ptr) in live.iter().enumerate() { + out.push_str(&format!( + " %perry_sp_bits_{statepoint_id}_{root_idx} = load i64, ptr {ptr}\n" + )); + out.push_str(&format!( + " %perry_sp_root_{statepoint_id}_{root_idx} = inttoptr i64 \ + %perry_sp_bits_{statepoint_id}_{root_idx} to ptr addrspace(1)\n" + )); + } + + let function_type = format!("{} ({})", call.return_type, call.arg_types.join(", ")); + let call_args = call + .args + .iter() + .map(|arg| format!(", {arg}")) + .collect::(); + let gc_live = live + .iter() + .enumerate() + .map(|(root_idx, _)| format!("ptr addrspace(1) %perry_sp_root_{statepoint_id}_{root_idx}")) + .collect::>() + .join(", "); + out.push_str(&format!( + " %perry_sp_token_{statepoint_id} = call token (i64, i32, ptr, i32, i32, ...) \ + @llvm.experimental.gc.statepoint.p0(i64 {statepoint_id}, i32 0, \ + ptr elementtype({function_type}) {}, i32 {}, i32 0{call_args}, i32 0, i32 0) \ + [\"gc-live\"({gc_live})]\n", + call.callee, + call.args.len() + )); + + if let Some(result) = call.result { + let suffix = gc_result_suffix(call.return_type) + .expect("non-void statepoint return type was validated by the parser"); + out.push_str(&format!( + " {result} = call {} @llvm.experimental.gc.result.{suffix}(token \ + %perry_sp_token_{statepoint_id})\n", + call.return_type + )); + } + + for (root_idx, ptr) in live.iter().enumerate() { + out.push_str(&format!( + " %perry_sp_relocated_{statepoint_id}_{root_idx} = call ptr addrspace(1) \ + @llvm.experimental.gc.relocate.p1(token %perry_sp_token_{statepoint_id}, \ + i32 {root_idx}, i32 {root_idx})\n" + )); + out.push_str(&format!( + " %perry_sp_relocated_bits_{statepoint_id}_{root_idx} = ptrtoint \ + ptr addrspace(1) %perry_sp_relocated_{statepoint_id}_{root_idx} to i64\n" + )); + out.push_str(&format!( + " store i64 %perry_sp_relocated_bits_{statepoint_id}_{root_idx}, ptr {ptr}\n" + )); + } +} + +/// Lower Perry's existing precise-root operations to native-stack metadata. +/// +/// The old binding calls already name exactly the mutable native alloca that a +/// moving collection must rewrite. We use them as compile-time markers: +/// +/// * collect `logical slot -> native alloca`; +/// * remove the runtime bind calls and shadow-frame traffic; +/// * compute conservative per-call liveness from bind/clear markers without +/// mutating the native slot; +/// * either place a plain stack map before a call, or replace a supported call +/// with a statepoint/result/relocate sequence. +/// +/// Statepoint mode deliberately retains a plain-stack-map fallback for call +/// forms outside the narrow parser above. The fallback preserves correctness +/// while the report records how much of real Perry code reaches the explicit +/// relocation path. +fn lower_precise_roots_to_native_stack( + ir: &str, + slot_count: u32, + backend: PreciseRootBackend, +) -> String { + let lines: Vec<&str> = ir.lines().collect(); + let active_slots = stack_map_active_slots(&lines, slot_count); + let mut roots: Vec> = vec![None; slot_count as usize]; + for line in &lines { + if let Some((idx, ptr)) = parse_shadow_bind(line) { + if let Some(root) = roots.get_mut(idx) { + match root { + Some(existing) => { + debug_assert_eq!( + existing, &ptr, + "one precise-root slot must not bind two native allocas" + ); + } + None => *root = Some(ptr), + } + } + } + } + + let slot_roots = roots; + let root_ptrs: Vec = slot_roots.iter().flatten().cloned().collect(); + if root_ptrs.is_empty() { + return ir + .lines() + .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) + .map(|line| format!("{line}\n")) + .collect(); + } + + let mut out = String::with_capacity(ir.len() + root_ptrs.len() * 128); + let mut available = std::collections::HashSet::::new(); + let mut initialized = std::collections::HashSet::::new(); + let mut map_id = 0u64; + + for (line_idx, line) in lines.iter().enumerate() { + if parse_shadow_bind(line).is_some() { + // Compile-time marker only. The real slot is already populated by + // the local store immediately preceding this old bind. + continue; + } + if parse_shadow_set(line).is_some() { + // This marker changes stack-map liveness, not the program local. + // Shadow-stack clears only flipped SLOT_ACTIVE for the same + // reason: a value can be semantically read after its final + // GC-capable call. + continue; + } + + // A stack-map operand must dominate the intrinsic. Root allocas are + // normally entry-hoisted, but tracking definitions here also handles + // the few block-local scalar-replacement slots without emitting + // invalid SSA. + for ptr in &root_ptrs { + if line.trim_start().starts_with(&format!("{ptr} = ")) { + available.insert(ptr.clone()); + } + } + + out.push_str(line); + out.push('\n'); + + // Slots can be named by a stack map before their source-level `let` + // executes. Zero them directly after the alloca so an earlier + // safepoint never exposes uninitialized stack bytes as roots. + for ptr in &root_ptrs { + if available.contains(ptr) + && !initialized.contains(ptr) + && line.trim_start().starts_with(&format!("{ptr} = alloca ")) + { + out.push_str(&format!(" store i64 0, ptr {ptr}\n")); + initialized.insert(ptr.clone()); + } + } + + // Insert before calls, not after. Rebuild the tail when the line just + // appended is a call so the intrinsic's instruction offset is the + // actual call-site offset in the final machine function. + let trimmed = line.trim_start(); + let is_call = trimmed.starts_with("call ") + || trimmed.contains(" = call ") + || trimmed.starts_with("tail call ") + || trimmed.contains(" = tail call "); + if !is_call || trimmed.contains("@llvm.experimental.stackmap") { + continue; + } + let active = active_slots.get(line_idx).and_then(Option::as_ref); + let live: Vec<&String> = slot_roots + .iter() + .enumerate() + .filter(|(idx, _)| active.is_some_and(|slots| slots.contains(idx))) + .filter_map(|(_, ptr)| ptr.as_ref()) + .filter(|ptr| available.contains(*ptr) && initialized.contains(*ptr)) + .collect(); + if live.is_empty() { + continue; + } + if backend == PreciseRootBackend::Statepoint + && (trimmed.contains("@llvm.") || trimmed.contains("call void asm ")) + { + // LLVM intrinsics and zero-instruction compiler barriers cannot + // enter Perry's allocator, so they are not safepoints. The plain + // stack-map prototype instrumented every textual `call`; the + // statepoint path can make this distinction without losing roots. + continue; + } + + // Move the call line behind the intrinsic. + let call_len = line.len() + 1; + out.truncate(out.len() - call_len); + if backend == PreciseRootBackend::Statepoint { + if let Some(call) = parse_direct_statepoint_call(line) { + emit_statepoint(&mut out, &call, &live, map_id); + map_id += 1; + continue; + } + } + emit_plain_stack_map(&mut out, line, &live, map_id); + map_id += 1; + } + out +} + +#[cfg(test)] +mod stack_map_tests { + use super::{ + lower_precise_roots_to_native_stack, parse_direct_statepoint_call, PreciseRootBackend, + }; + + fn lower_stack_maps(input: &str, slots: u32) -> String { + lower_precise_roots_to_native_stack(input, slots, PreciseRootBackend::StackMap) + } + + fn lower_statepoints(input: &str, slots: u32) -> String { + lower_precise_roots_to_native_stack(input, slots, PreciseRootBackend::Statepoint) + } + + #[test] + fn lowers_bind_and_liveness_clear_to_native_frame_maps() { + let input = r#"define i64 @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 @may_collect() + call void @js_shadow_slot_set(i32 0, i64 0) + call void @may_collect_again() + ret i64 %r1 +} +"#; + let output = lower_stack_maps(input, 1); + assert!(!output.contains("@js_shadow_slot_bind")); + assert!(!output.contains("@js_shadow_slot_set")); + assert!(output.contains("%r0 = alloca i64\n store i64 0, ptr %r0")); + assert!(output.contains( + "@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)\n %r1 = call i64 \ + @may_collect()\n call void asm sideeffect \"\", \"~{memory}\"()" + )); + assert_eq!(output.matches("store i64 0, ptr %r0").count(), 1); + assert_eq!(output.matches("@llvm.experimental.stackmap").count(), 1); + assert!(output.contains("call void @may_collect_again()")); + } + + #[test] + fn does_not_reference_a_root_before_its_alloca_dominates() { + let input = r#"define void @probe() { +entry.0: + call void @early_call() + %r0 = alloca i64 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @late_call() + ret void +} +"#; + let output = lower_stack_maps(input, 1); + let early = output.find("call void @early_call()").unwrap(); + let first_map = output.find("@llvm.experimental.stackmap").unwrap(); + assert!(early < first_map); + assert!(output.contains( + "@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)\n call void @late_call()" + )); + } + + #[test] + fn unions_root_liveness_at_control_flow_joins() { + let input = r#"define void @probe(i1 %cond) { +entry.0: + %r0 = alloca i64 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + br i1 %cond, label %live.1, label %dead.2 +live.1: + call void @live_call() + br label %merge.3 +dead.2: + call void @js_shadow_slot_set(i32 0, i64 0) + call void @dead_call() + br label %merge.3 +merge.3: + call void @merge_call() + ret void +} +"#; + let output = lower_stack_maps(input, 1); + assert!(output.contains( + "@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)\n call void @live_call()" + )); + assert!(!output.contains( + "@llvm.experimental.stackmap(i64 1, i32 0, ptr %r0)\n call void @dead_call()" + )); + assert!(output.contains( + "@llvm.experimental.stackmap(i64 1, i32 0, ptr %r0)\n call void @merge_call()" + )); + } + + #[test] + fn parses_the_scalar_direct_call_subset() { + assert_eq!( + parse_direct_statepoint_call(" %r7 = call double @foo(i64 %r1, ptr %r2)"), + Some(super::DirectCall { + result: Some("%r7"), + return_type: "double", + callee: "@foo", + args: vec!["i64 %r1", "ptr %r2"], + arg_types: vec!["i64", "ptr"], + }) + ); + assert!(parse_direct_statepoint_call( + " %r7 = call double (i64, ptr)* %fn(i64 %r1, ptr %r2)" + ) + .is_none()); + assert!(parse_direct_statepoint_call(" call void @llvm.assume(i1 %ok)").is_none()); + assert!(parse_direct_statepoint_call(" %r7 = tail call i64 @foo()").is_none()); + } + + #[test] + fn lowers_direct_calls_to_explicit_statepoint_relocations() { + let input = r#"define i64 @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 @may_collect(i64 %arg) + ret i64 %r1 +} +"#; + let output = lower_statepoints(input, 1); + assert!(!output.contains("call i64 @may_collect")); + assert!(!output.contains("asm sideeffect")); + assert!(output + .contains("%perry_sp_root_0_0 = inttoptr i64 %perry_sp_bits_0_0 to ptr addrspace(1)")); + assert!(output.contains( + "ptr elementtype(i64 (i64)) @may_collect, i32 1, i32 0, i64 %arg, i32 0, i32 0" + )); + assert!(output + .contains("%r1 = call i64 @llvm.experimental.gc.result.i64(token %perry_sp_token_0)")); + assert!(output + .contains("@llvm.experimental.gc.relocate.p1(token %perry_sp_token_0, i32 0, i32 0)")); + assert!(output.contains("store i64 %perry_sp_relocated_bits_0_0, ptr %r0")); + } + + #[test] + fn statepoint_mode_falls_back_for_indirect_calls() { + let input = r#"define i64 @probe(i64 %arg, ptr %fn) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 ()* %fn() + ret i64 %r1 +} +"#; + let output = lower_statepoints(input, 1); + assert!(output.contains("@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)")); + assert!(output.contains("%r1 = call i64 ()* %fn()")); + assert!(!output.contains("@llvm.experimental.gc.statepoint")); + } + + #[test] + fn statepoint_mode_does_not_map_non_allocating_llvm_intrinsics() { + let input = r#"define void @probe(i64 %arg, i1 %condition) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @llvm.assume(i1 %condition) + call void @may_collect() + ret void +} +"#; + let output = lower_statepoints(input, 1); + assert!(output.contains("call void @llvm.assume(i1 %condition)")); + assert_eq!( + output + .matches("@llvm.experimental.gc.statepoint.p0") + .count(), + 1 + ); + assert!(!output.contains("@llvm.experimental.stackmap")); + } +} diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 297d1642f7..24f7f88abc 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -185,6 +185,30 @@ fn render_fn_external(f: &LlFunction) -> String { ir } +fn push_statepoint_declarations(ir: &mut String) { + ir.push_str( + "declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, \ + i32 immarg, i32 immarg, ...)\n\ + declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token, i32 immarg, \ + i32 immarg)\n", + ); + for (suffix, ty) in [ + ("i1", "i1"), + ("i8", "i8"), + ("i16", "i16"), + ("i32", "i32"), + ("i64", "i64"), + ("i128", "i128"), + ("f32", "float"), + ("f64", "double"), + ("p0", "ptr"), + ] { + ir.push_str(&format!( + "declare {ty} @llvm.experimental.gc.result.{suffix}(token)\n" + )); + } +} + pub struct LlModule { pub target_triple: String, declarations: Vec<(String, String)>, // (name, full "declare …" line) @@ -448,6 +472,16 @@ impl LlModule { let mut ir = String::new(); ir.push_str("; Generated by perry-codegen\n"); ir.push_str(&format!("target triple = \"{}\"\n\n", self.target_triple)); + if crate::codegen::helpers::native_stack_roots_enabled() + && self.target_triple.contains("apple") + { + // LLVM emits one local `__LLVM_StackMaps` atom per object. Perry's + // normal `-dead_strip` link otherwise discards those unreferenced + // atoms. This Mach-O directive marks each local atom live without + // globalizing the repeated symbol (which would collide across + // codegen units). + ir.push_str("module asm \".no_dead_strip __LLVM_StackMaps\"\n\n"); + } for sc in &self.string_constants { ir.push_str(sc); @@ -473,6 +507,12 @@ impl LlModule { ir.push_str(decl); ir.push('\n'); } + if crate::codegen::helpers::native_stack_roots_enabled() { + ir.push_str("declare void @llvm.experimental.stackmap(i64, i32, ...)\n"); + } + if crate::codegen::helpers::statepoints_enabled() { + push_statepoint_declarations(&mut ir); + } ir.push('\n'); for func in &funcs { @@ -634,6 +674,11 @@ impl LlModule { let mut ir = String::new(); ir.push_str("; Generated by perry-codegen (codegen unit)\n"); ir.push_str(&format!("target triple = \"{}\"\n\n", self.target_triple)); + if crate::codegen::helpers::native_stack_roots_enabled() + && self.target_triple.contains("apple") + { + ir.push_str("module asm \".no_dead_strip __LLVM_StackMaps\"\n\n"); + } for sc in &shared_strings { ir.push_str(sc); @@ -654,6 +699,12 @@ impl LlModule { ir.push_str(decl); ir.push('\n'); } + if crate::codegen::helpers::native_stack_roots_enabled() { + ir.push_str("declare void @llvm.experimental.stackmap(i64, i32, ...)\n"); + } + if crate::codegen::helpers::statepoints_enabled() { + push_statepoint_declarations(&mut ir); + } ir.push('\n'); for func in bucket { diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index dbea304905..a37055764e 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -663,6 +663,10 @@ pub fn gc_init() { #[no_mangle] pub extern "C" fn js_gc_init() { + // Parse LLVM stack-map metadata before the first collection. The parser + // allocates its immutable index once; root scans themselves must remain + // allocation-free while the collector owns the heap. + initialize_stack_maps(); // Windows: opt console stdout/stderr into VT/ANSI escape processing // once at program start so runtime-emitted escapes (console.clear, tty // cursor ops, color output keyed off isTTY) render instead of printing diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index 91390de30e..ffb6af3749 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -5,7 +5,9 @@ mod runtime_handles; mod scan_mode; mod scanner_shims; mod shadow_stack; +mod stack_maps; mod temp_roots; +pub(super) use stack_maps::initialize as initialize_stack_maps; pub(super) use runtime_handles::{ new_runtime_handle_root_scan_state, scan_runtime_handle_roots_mut, @@ -1371,6 +1373,7 @@ impl MutableRootSlot { /// mutable slot addresses so the same walk can support mark-only /// scanning and post-forwarding rewrites. pub(super) fn visit_shadow_stack_root_slots(mut visit: impl FnMut(MutableRootSlot)) { + stack_maps::visit_stack_map_root_slots(&mut visit); SHADOW.with(|cell| unsafe { let s = &mut *cell.get(); if s.len == 0 || s.ptr.is_null() { diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs new file mode 100644 index 0000000000..436dbb40b5 --- /dev/null +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -0,0 +1,528 @@ +//! Research precise-root backend for LLVM stack maps. +//! +//! The plain-map prototype places `llvm.experimental.stackmap` immediately +//! before mapped calls and records the address of each native root alloca. +//! The statepoint prototype instead records LLVM-owned spill slots for +//! `gc.relocate` values. Both are writable frame-register-relative locations +//! in the emitted stack-map section. +//! +//! This first implementation deliberately targets macOS, where the experiment +//! is being measured. It discovers the concatenated `__LLVM_STACKMAPS` section +//! in the main Mach-O image and uses the platform unwinder to recover the +//! frame-register value for each active generated frame. Unsupported targets +//! return no roots; neither native-stack experiment may be used for correctness +//! there. + +use super::{MutableRootSlot, MutableRootSlotKind}; +use std::ffi::c_void; +use std::sync::OnceLock; + +const STACK_MAP_VERSION: u8 = 3; +const LOCATION_DIRECT: u8 = 2; +const LOCATION_INDIRECT: u8 = 3; +const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct StackMapLocation { + dwarf_reg: u16, + offset: i32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct StackMapRecord { + pc: usize, + locations: Vec, +} + +static STACK_MAPS: OnceLock> = OnceLock::new(); + +pub(in crate::gc) fn initialize() { + let _ = stack_maps(); +} + +fn stack_maps() -> &'static [StackMapRecord] { + STACK_MAPS + .get_or_init(|| { + let Some(section) = loaded_stack_map_section() else { + return Vec::new(); + }; + let mut records = parse_concatenated_stack_maps(section).unwrap_or_default(); + records.sort_unstable_by_key(|record| record.pc); + records + }) + .as_slice() +} + +fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { + let insertion = maps.partition_point(|record| record.pc < ip); + let before = insertion + .checked_sub(1) + .and_then(|idx| maps.get(idx)) + .map(|record| record.pc); + let at_or_after = maps.get(insertion).map(|record| record.pc); + match (before, at_or_after) { + (Some(before), Some(after)) => Some(if ip.abs_diff(before) <= ip.abs_diff(after) { + before + } else { + after + }), + (Some(before), None) => Some(before), + (None, Some(after)) => Some(after), + (None, None) => None, + } +} + +pub(super) fn visit_stack_map_root_slots(visit: &mut impl FnMut(MutableRootSlot)) { + let maps = stack_maps(); + if maps.is_empty() { + return; + } + unwind::visit(maps, visit); +} + +fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { + let mut all = Vec::new(); + let mut base = 0usize; + while base < bytes.len() { + // Linkers preserve the input section's 8-byte alignment. Ignore a + // zero-filled tail, but do not search through malformed non-zero data. + if bytes[base..].iter().all(|byte| *byte == 0) { + break; + } + let (mut records, consumed) = parse_one_stack_map(&bytes[base..])?; + if consumed == 0 { + return None; + } + all.append(&mut records); + base = base.checked_add(consumed)?; + } + Some(all) +} + +fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { + if read_u8(bytes, 0)? != STACK_MAP_VERSION { + return None; + } + let function_count = read_u32(bytes, 4)? as usize; + let constant_count = read_u32(bytes, 8)? as usize; + let record_count = read_u32(bytes, 12)? as usize; + let mut offset = 16usize; + + let mut functions = Vec::with_capacity(function_count); + let mut expected_records = 0usize; + for _ in 0..function_count { + let address = read_u64(bytes, offset)? as usize; + let records = read_u64(bytes, offset + 16)? as usize; + functions.push((address, records)); + expected_records = expected_records.checked_add(records)?; + offset = offset.checked_add(24)?; + } + if expected_records != record_count { + return None; + } + offset = offset.checked_add(constant_count.checked_mul(8)?)?; + if offset > bytes.len() { + return None; + } + + let mut out = Vec::with_capacity(record_count); + for (function_address, function_record_count) in functions { + for _ in 0..function_record_count { + let instruction_offset = read_u32(bytes, offset + 8)? as usize; + let location_count = read_u16(bytes, offset + 14)? as usize; + offset = offset.checked_add(16)?; + + let mut locations = Vec::new(); + for _ in 0..location_count { + let kind = read_u8(bytes, offset)?; + let size = read_u16(bytes, offset + 2)?; + let dwarf_reg = read_u16(bytes, offset + 4)?; + let location_offset = read_i32(bytes, offset + 8)?; + if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { + let location = StackMapLocation { + dwarf_reg, + offset: location_offset, + }; + // A statepoint records a base/derived pair for every + // relocation. Perry currently uses the same value for + // both, so LLVM commonly emits the exact same spill slot + // twice. Visit that physical word once. + if !locations.contains(&location) { + locations.push(location); + } + } + offset = offset.checked_add(12)?; + } + + // LLVM aligns the live-out header independently from the whole + // record. This first padding is observable whenever the location + // count is odd (one Direct root is a common case). + offset = align_up(offset, 8)?; + // Two reserved bytes followed by the live-out count. + let live_out_count = read_u16(bytes, offset + 2)? as usize; + offset = offset + .checked_add(4)? + .checked_add(live_out_count.checked_mul(4)?)?; + offset = align_up(offset, 8)?; + if offset > bytes.len() { + return None; + } + + out.push(StackMapRecord { + pc: function_address.checked_add(instruction_offset)?, + locations, + }); + } + } + Some((out, offset)) +} + +fn align_up(value: usize, alignment: usize) -> Option { + value + .checked_add(alignment.checked_sub(1)?) + .map(|value| value & !(alignment - 1)) +} + +fn read_u8(bytes: &[u8], offset: usize) -> Option { + bytes.get(offset).copied() +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_le_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_le_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_i32(bytes: &[u8], offset: usize) -> Option { + Some(i32::from_le_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_le_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +#[cfg(target_os = "macos")] +fn loaded_stack_map_section() -> Option<&'static [u8]> { + use mach2::dyld::{_dyld_get_image_header, _dyld_get_image_vmaddr_slide}; + + const LC_SEGMENT_64: u32 = 0x19; + + #[repr(C)] + #[derive(Clone, Copy)] + struct MachHeader64 { + magic: u32, + cpu_type: i32, + cpu_subtype: i32, + file_type: u32, + command_count: u32, + commands_size: u32, + flags: u32, + reserved: u32, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct LoadCommand { + command: u32, + size: u32, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct SegmentCommand64 { + command: u32, + size: u32, + segment_name: [u8; 16], + vm_address: u64, + vm_size: u64, + file_offset: u64, + file_size: u64, + max_protection: i32, + initial_protection: i32, + section_count: u32, + flags: u32, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct Section64 { + section_name: [u8; 16], + segment_name: [u8; 16], + address: u64, + size: u64, + offset: u32, + alignment: u32, + relocation_offset: u32, + relocation_count: u32, + flags: u32, + reserved1: u32, + reserved2: u32, + reserved3: u32, + } + + fn fixed_name_matches(actual: &[u8; 16], expected: &[u8]) -> bool { + actual.get(..expected.len()) == Some(expected) + && actual.get(expected.len()).copied().unwrap_or(0) == 0 + } + + unsafe { + let raw_header = _dyld_get_image_header(0); + if raw_header.is_null() { + return None; + } + let header = &*(raw_header.cast::()); + let slide = _dyld_get_image_vmaddr_slide(0); + let mut command_ptr = raw_header + .cast::() + .add(std::mem::size_of::()); + for _ in 0..header.command_count { + let load = std::ptr::read_unaligned(command_ptr.cast::()); + if load.size < std::mem::size_of::() as u32 { + return None; + } + if load.command == LC_SEGMENT_64 { + let segment = std::ptr::read_unaligned(command_ptr.cast::()); + let mut section_ptr = command_ptr.add(std::mem::size_of::()); + for _ in 0..segment.section_count { + let section = std::ptr::read_unaligned(section_ptr.cast::()); + if fixed_name_matches(§ion.segment_name, b"__LLVM_STACKMAPS") + && fixed_name_matches(§ion.section_name, b"__llvm_stackmaps") + { + let address = (section.address as isize).checked_add(slide)? as usize; + let size = usize::try_from(section.size).ok()?; + if address == 0 || size == 0 { + return None; + } + return Some(std::slice::from_raw_parts(address as *const u8, size)); + } + section_ptr = section_ptr.add(std::mem::size_of::()); + } + } + command_ptr = command_ptr.add(load.size as usize); + } + } + None +} + +#[cfg(not(target_os = "macos"))] +fn loaded_stack_map_section() -> Option<&'static [u8]> { + None +} + +#[cfg(target_os = "macos")] +mod unwind { + use super::*; + + #[repr(C)] + struct UnwindContext { + _private: [u8; 0], + } + + unsafe extern "C" { + fn _Unwind_Backtrace( + trace: unsafe extern "C" fn(*mut UnwindContext, *mut c_void) -> i32, + argument: *mut c_void, + ) -> i32; + fn _Unwind_GetIP(context: *mut UnwindContext) -> usize; + fn _Unwind_GetGR(context: *mut UnwindContext, register: i32) -> usize; + } + + struct WalkState<'a, F> { + maps: &'a [StackMapRecord], + visit: &'a mut F, + } + + pub(super) fn visit(maps: &[StackMapRecord], visit: &mut F) { + let mut state = WalkState { maps, visit }; + unsafe { + _Unwind_Backtrace( + walk_frame::, + (&mut state as *mut WalkState<'_, _>).cast::(), + ); + } + } + + unsafe extern "C" fn walk_frame( + context: *mut UnwindContext, + argument: *mut c_void, + ) -> i32 { + let state = &mut *argument.cast::>(); + let ip = _Unwind_GetIP(context); + let Some(candidate_pc) = closest_record_pc(state.maps, ip) else { + return 0; + }; + let delta = ip.abs_diff(candidate_pc); + if delta > MAX_SAFEPOINT_RETURN_DELTA { + return 0; + } + + let first = state + .maps + .partition_point(|record| record.pc < candidate_pc); + let last = state + .maps + .partition_point(|record| record.pc <= candidate_pc); + for record in &state.maps[first..last] { + for location in &record.locations { + let base = _Unwind_GetGR(context, i32::from(location.dwarf_reg)); + let address = if location.offset < 0 { + base.checked_sub(location.offset.unsigned_abs() as usize) + } else { + base.checked_add(location.offset as usize) + }; + let Some(address) = address else { + continue; + }; + if address == 0 || address & (std::mem::align_of::() - 1) != 0 { + continue; + } + (state.visit)(MutableRootSlot { + // Reuse the compiled-frame telemetry bucket so the + // experiment compares root source counts directly. + kind: MutableRootSlotKind::ShadowStack, + ptr: address as *mut u64, + }); + } + } + 0 + } +} + +#[cfg(not(target_os = "macos"))] +mod unwind { + use super::*; + + pub(super) fn visit(_maps: &[StackMapRecord], _visit: &mut impl FnMut(MutableRootSlot)) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + fn one_map_with_locations( + function: u64, + id: u64, + offset: u32, + locations: &[(u8, i32)], + ) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[STACK_MAP_VERSION, 0, 0, 0]); + bytes.extend_from_slice(&1u32.to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&1u32.to_le_bytes()); + bytes.extend_from_slice(&function.to_le_bytes()); + bytes.extend_from_slice(&32u64.to_le_bytes()); + bytes.extend_from_slice(&1u64.to_le_bytes()); + bytes.extend_from_slice(&id.to_le_bytes()); + bytes.extend_from_slice(&offset.to_le_bytes()); + bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.extend_from_slice(&(locations.len() as u16).to_le_bytes()); + for (kind, frame_offset) in locations { + bytes.push(*kind); + bytes.push(0); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(&29u16.to_le_bytes()); + bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.extend_from_slice(&frame_offset.to_le_bytes()); + } + while bytes.len() % 8 != 0 { + bytes.push(0); + } + bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.extend_from_slice(&0u16.to_le_bytes()); + while bytes.len() % 8 != 0 { + bytes.push(0); + } + bytes + } + + fn one_map(function: u64, id: u64, offset: u32, frame_offset: i32) -> Vec { + one_map_with_locations(function, id, offset, &[(LOCATION_DIRECT, frame_offset)]) + } + + #[test] + fn parses_direct_mutable_frame_location() { + let bytes = one_map(0x1000, 42, 0x10, -8); + let (records, consumed) = parse_one_stack_map(&bytes).expect("valid stack map"); + assert_eq!(consumed, bytes.len()); + assert_eq!( + records, + vec![StackMapRecord { + pc: 0x1010, + locations: vec![StackMapLocation { + dwarf_reg: 29, + offset: -8, + }], + }] + ); + } + + #[test] + fn parses_linker_concatenated_input_sections() { + let mut bytes = one_map(0x1000, 42, 0x10, -8); + bytes.extend_from_slice(&one_map(0x2000, 43, 0x20, -16)); + let records = parse_concatenated_stack_maps(&bytes).expect("concatenated maps"); + assert_eq!(records.len(), 2); + assert_eq!(records[0].pc, 0x1010); + assert_eq!(records[1].pc, 0x2020); + } + + #[test] + fn parses_and_deduplicates_statepoint_spill_locations() { + let bytes = one_map_with_locations( + 0x1000, + 7, + 0x20, + &[(LOCATION_INDIRECT, -16), (LOCATION_INDIRECT, -16)], + ); + let (records, consumed) = parse_one_stack_map(&bytes).expect("valid statepoint map"); + assert_eq!(consumed, bytes.len()); + assert_eq!( + records, + vec![StackMapRecord { + pc: 0x1020, + locations: vec![StackMapLocation { + dwarf_reg: 29, + offset: -16, + }], + }] + ); + } + + #[test] + fn rejects_truncated_or_wrong_version_sections() { + assert!(parse_one_stack_map(&[]).is_none()); + let mut bytes = one_map(0x1000, 42, 0x10, -8); + bytes[0] = 2; + assert!(parse_one_stack_map(&bytes).is_none()); + bytes[0] = STACK_MAP_VERSION; + bytes.truncate(bytes.len() - 1); + assert!(parse_one_stack_map(&bytes).is_none()); + } + + #[test] + fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { + let maps = vec![ + StackMapRecord { + pc: 0x1000, + locations: Vec::new(), + }, + StackMapRecord { + pc: 0x1020, + locations: Vec::new(), + }, + ]; + assert_eq!(closest_record_pc(&maps, 0x1004), Some(0x1000)); + assert_eq!(closest_record_pc(&maps, 0x101c), Some(0x1020)); + assert_eq!(closest_record_pc(&maps, 0x1020), Some(0x1020)); + } +} diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index d31651a752..7c6b8f054c 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -35,6 +35,8 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_LLVM_CLANG", "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", + "PERRY_STACK_MAPS", + "PERRY_STATEPOINTS", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 2194ce4fbb..77facaaafa 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -229,7 +229,8 @@ fn stable_type_key(ty: &perry_hir::types::Type) -> String { /// We also mix in environment variables that `perry-codegen` reads /// at compile time but that aren't part of `CompileOptions`: /// `PERRY_DEBUG_INIT`, `PERRY_DEBUG_SYMBOLS`, `PERRY_LLVM_CLANG`, -/// `PERRY_WRITE_BARRIERS`, `PERRY_SHADOW_STACK`, +/// `PERRY_WRITE_BARRIERS`, `PERRY_SHADOW_STACK`, `PERRY_STACK_MAPS`, +/// `PERRY_STATEPOINTS`, /// `PERRY_DISABLE_BUFFER_FAST_PATH`, `PERRY_VERIFY_NATIVE_REGIONS`, /// `PERRY_UNBOXED_OBJECT_FIELDS`, and `PERRY_TARGET_CPU`. See the env-var /// block at the bottom of this function for the rationale. @@ -759,6 +760,10 @@ fn compute_object_cache_key_with_env( // calls at heap-store sites (codegen.rs / expr.rs). // - PERRY_SHADOW_STACK=0/off/false suppresses generated frame/slot // roots at function entry and pointer local stores. + // - PERRY_STACK_MAPS=1 lowers those precise roots to LLVM native-frame + // stack maps instead of the runtime shadow stack. + // - PERRY_STATEPOINTS=1 replaces supported calls with LLVM statepoint + // relocation sequences and uses native stack maps for the remainder. // - PERRY_DISABLE_BUFFER_FAST_PATH=1 overrides CompileOptions and // changes Buffer/Uint8Array lowering. // - PERRY_VERIFY_NATIVE_REGIONS=1 overrides CompileOptions and must @@ -794,6 +799,14 @@ fn compute_object_cache_key_with_env( "env_shadow_stack", env_var("PERRY_SHADOW_STACK").as_deref().unwrap_or(""), ); + h.field( + "env_stack_maps", + env_var("PERRY_STACK_MAPS").as_deref().unwrap_or(""), + ); + h.field( + "env_statepoints", + env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), + ); // #7088: flips the shadow-slot store between an inline sequence and the // `js_shadow_slot_*` calls. Two arms that shared a cached object would // silently measure the same code. diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index f8320f7af5..672bcee4a0 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -583,6 +583,8 @@ fn key_changes_with_codegen_env_vars() { "PERRY_LLVM_CLANG", "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", + "PERRY_STACK_MAPS", + "PERRY_STATEPOINTS", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", "PERRY_UNBOXED_OBJECT_FIELDS", diff --git a/docs/stack-map-gc-experiment.md b/docs/stack-map-gc-experiment.md new file mode 100644 index 0000000000..6b50f868fd --- /dev/null +++ b/docs/stack-map-gc-experiment.md @@ -0,0 +1,234 @@ +# Stack-map GC experiment + +Date: 2026-07-31 + +Branch: `exp/stackmap-viability` + +Base commit: `e2557c1a985cb983ed00aafd1a2c1b31f1570b98` + +## Decision + +LLVM stack maps are technically viable for Perry's moving collector on +macOS/arm64, but this prototype does **not** justify replacing the shadow stack +yet. + +- Correctness is promising: all eight GC-ratchet probes match Node, forced + evacuation verification passes, and retained heap is byte-for-byte identical + to the shadow-stack arm. +- Runtime performance is effectively flat on ordinary workloads. The GC suite + measured a noisy 1.5% geometric-mean improvement, while an interleaved + deep-stack run was 1.5% slower. The host was heavily loaded, so neither is a + defensible headline win. +- Compilation time is flat. Root-heavy executables gain an approximately + 16 KiB Mach-O segment even when the stack-map payload is only 2–9 KiB. +- Root enumeration is not simpler overall. The heap-backed frame stack and its + TLS traffic can disappear, but they are replaced by LLVM ABI parsing, + linker-retention rules, native unwinding, call-site matching, compiler memory + barriers, and control-flow liveness analysis. +- The direction is strategically useful, but a production follow-up should + investigate `gc.statepoint`/`gc.relocate`, not promote this plain + `llvm.experimental.stackmap` prototype. + +The prototype remains opt-in with `PERRY_STACK_MAPS=1`; the default +shadow-stack path is unchanged. + +## What was built + +The experiment reuses Perry's existing precise-root discovery and slot +numbering, then changes the backend: + +1. Shadow-slot bind/clear operations remain temporary IR markers. +2. A final per-function pass resolves logical slots to their native allocas. +3. A conservative CFG dataflow pass computes roots that may be live at each + call. A join uses union, so it may retain a stale value but cannot omit a + live root. +4. Each call with live roots gets an `llvm.experimental.stackmap` carrying the + addresses of those allocas. LLVM emits them as writable `Direct` + frame-register-relative locations. +5. Empty memory-clobbering inline assembly brackets mapped calls. It generates + no instructions, but makes the collector's otherwise invisible slot rewrite + observable to LLVM before and after the call. +6. The runtime parses LLVM stack-map v3 records from the main Mach-O image, + unwinds active frames with `_Unwind_Backtrace`, finds the record immediately + preceding each return PC, and feeds its mutable slots into the existing GC + root visitor. +7. Mach-O module assembly marks each local `__LLVM_StackMaps` atom + `.no_dead_strip`; otherwise Perry's normal link removes the metadata. + +The parser accepts linker-concatenated stack-map blobs and handles LLVM's +independent alignment before and after the live-out list. Both details caused +real failures during the spike and now have regression coverage. + +LLVM documents this intrinsic and binary format as experimental and explicitly +separate from its GC statepoint machinery: +[Stack maps and patch points](https://llvm.org/docs/StackMaps.html) and +[Garbage collection safepoints](https://llvm.org/docs/Statepoints.html). + +## Correctness results + +The final release artifacts passed: + +- all 8 GC-ratchet probes, with stdout identical to Node; +- all 8 probes again with `PERRY_GC_FORCE_EVACUATE=1` and + `PERRY_GC_VERIFY_EVACUATION=1`; +- the string-retention probe 100 consecutive times after it exposed the + parser/call-site issues; +- all 310 `perry-codegen` library tests; +- 1,570 `perry-runtime` library tests in single-threaded mode (3 ignored), + excluding the existing debug-only `extern "C"` malformed-pop test whose + intentional `debug_assert!` aborts a debug test process; +- 3 stack-map parser tests; +- 3 stack-map lowering/liveness tests; +- 46 object-cache tests, including `PERRY_STACK_MAPS` cache separation. + +Across the full ratchet, median retained heap and heap capacity were identical +for every probe. Promotions were identical. Six probes copied exactly the same +number of objects; the remaining differences were +3, -6, and +2 objects, with +the same final retention and freed bytes. + +The spike found three important correctness requirements: + +- A moving collector must map writable native slots, not merely record pointer + values. Passing alloca addresses produces LLVM `Direct` locations. +- Stack-map parsing must honor the pre-live-out alignment in the v3 format. + Missing it desynchronized any record with an odd number of locations. +- A shadow-slot clear is a liveness change, not a write to the program local. + Zeroing the native local corrupted a value used after its last GC-capable + call. Static per-call liveness is required. + +## Performance results + +Hardware was an Apple M1 Max on macOS 26.5. The GC runs reported load averages +between 19 and 45, so the numbers below are directional only. Each A/B used the +same final compiler and runtime archives, with caches and auto-optimization +disabled. Runtime pairs were interleaved where noted. + +### GC-ratchet wall time + +Three measured runs plus one warmup per mode: + +| Probe | Shadow | Stack map | Delta | +|---|---:|---:|---:| +| Nursery churn | 182.581 ms | 175.766 ms | -3.73% | +| Survivor promotion | 213.684 ms | 213.661 ms | -0.01% | +| Cross-generation writes | 210.161 ms | 203.122 ms | -3.35% | +| Dead after deep stack | 471.128 ms | 484.463 ms | +2.83% | +| Closure capture | 175.824 ms | 163.789 ms | -6.84% | +| String retention | 135.017 ms | 135.645 ms | +0.47% | +| Array grow/evacuate | 171.918 ms | 174.017 ms | +1.22% | +| Map/set side tables | 459.315 ms | 449.043 ms | -2.24% | + +Geometric mean: stack maps were 1.50% faster. This is smaller than the +cross-run noise expected under the recorded host load. An additional +11-pair interleaved run measured: + +- deep active stack: stack maps **1.49% slower**; +- string retention: stack maps **0.07% faster**. + +The deep-stack result is consistent with native unwinding costing more than a +linear walk over the compact shadow buffer. + +### Broader runtime samples + +Eleven interleaved runs per binary: + +| Workload | Shadow | Stack map | Delta | +|---|---:|---:|---:| +| Process startup | 5.351 ms | 5.238 ms | -2.10% | +| Method calls | 93.295 ms | 93.383 ms | +0.09% | +| Function calls | 111.325 ms | 111.055 ms | -0.24% | +| GC pressure | 38.958 ms | 38.874 ms | -0.22% | +| JSON roundtrip | 362.536 ms | 363.680 ms | +0.32% | + +The substantive workloads are flat. Startup's 0.11 ms difference is below +what this host can resolve. + +### Compile time and size + +Five uncached compilations per cell: + +| Workload | Shadow | Stack map | Delta | +|---|---:|---:|---:| +| Startup | 449.71 ms | 449.76 ms | +0.01% | +| Method calls | 472.72 ms | 471.16 ms | -0.33% | +| Function calls | 457.69 ms | 457.73 ms | +0.01% | +| GC pressure | 465.83 ms | 460.47 ms | -1.15% | +| JSON roundtrip | 486.94 ms | 489.84 ms | +0.59% | + +The stack-map section measured 2,192 bytes for method calls, 5,512 bytes for GC +pressure, and 8,568 bytes for JSON roundtrip. On these Mach-O executables the +new segment rounded the file-size increase to roughly 16 KiB. Programs with no +mapped roots emitted no section and had identical file size. + +The modified runtime archive is 18,656 bytes larger than the exact baseline +(30,313,640 versus 30,294,984 bytes). + +## Is the GC simpler? + +Only in a narrow sense. + +The stack-map scanner is 451 lines in this spike versus 752 lines for the +current shadow-stack runtime. A completed replacement could also delete much +of the 563-line inline shadow-slot emitter and remove frame push/pop, TLS buffer +growth, longjmp depth restoration, and slot mirroring. + +The total system is not yet simpler: + +- the compiler gained a textual IR lowering and CFG liveness analysis; +- the linker needs platform-specific metadata retention; +- the runtime depends on an experimental LLVM binary contract and platform + unwinder behavior; +- moving-GC relocation needs writable allocas plus compiler memory fences; +- diagnostics must distinguish missing metadata from a genuinely rootless + frame; +- target support moves from ordinary generated calls to per-object-format + section discovery and per-architecture DWARF register handling. + +This trades locally optimized, explicit machinery for more cross-layer +machinery. It could become simpler after statepoints make relocation and +liveness first-class, but plain stack maps do not reach that point. + +## Is this better positioned? + +Potentially: + +- per-safepoint liveness can be more precise than a mutable activation-wide + registry; +- no shadow-frame push/pop or TLS slot mutation is required on the common + path; +- native frame metadata aligns Perry with established AOT/JIT GC techniques; +- exceptions and non-local exits naturally remove unwound frames from the + root set. + +The current prototype is not yet a platform: + +- scanning is implemented only for Mach-O/macOS; +- every call is conservatively instrumented instead of only GC-capable calls; +- the runtime assumes the matching stack-map PC is within 16 bytes of the + unwound return PC; +- active roots must remain in addressable allocas; +- parameter/`this` bindings still need a fully audited incremental-mark + transition barrier; +- async signals, foreign callbacks, tail calls, `setjmp`/`longjmp`, code + splitting, and non-Apple object formats need dedicated end-to-end tests; +- `PERRY_STACK_MAPS=1` is currently unsafe on unsupported targets because the + runtime scanner intentionally returns no roots there. + +## Recommended next experiment + +The explicit statepoint follow-up is now recorded in +[`statepoint-gc-experiment.md`](statepoint-gc-experiment.md). It validates +relocation correctness but finds no all-around performance win and no +whole-system simplification. + +Do not replace the default shadow stack from this branch. If this direction +continues after the representation work: + +1. Add an explicit safepoint capability table so pure calls do not receive + metadata or optimization fences. +2. Close the incremental parameter/`this` barrier gap and add a probe that + enters a new rooted activation during an in-flight incremental cycle. +3. Fail compilation on unsupported targets before expanding ELF/Windows + section discovery and unwind/register support. +4. Re-run the A/B suite on an idle host with the repository's standard 11-run + methodology and profile both mutator root updates and GC root scanning. diff --git a/docs/statepoint-bridge-probe.ll b/docs/statepoint-bridge-probe.ll new file mode 100644 index 0000000000..1f8b2f2d63 --- /dev/null +++ b/docs/statepoint-bridge-probe.ll @@ -0,0 +1,36 @@ +; Minimal Perry-shaped statepoint probe. +; +; A live NaN-boxed value is carried as an addrspace(1) pointer only across +; the safepoint. The runtime may rewrite its spill slot, gc.relocate reloads +; it, and ptrtoint restores Perry's existing i64 representation. + +target triple = "arm64-apple-macosx15.0.0" + +declare i64 @may_collect(i64) +declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) +declare i64 @llvm.experimental.gc.result.i64(token) +declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token, i32 immarg, i32 immarg) + +define i64 @statepoint_bridge_probe(i64 %bits, i64 %arg) gc "statepoint-example" { +entry: + %root = inttoptr i64 %bits to ptr addrspace(1) + %statepoint = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( + i64 1, + i32 0, + ptr elementtype(i64 (i64)) @may_collect, + i32 1, + i32 0, + i64 %arg, + i32 0, + i32 0 + ) ["gc-live"(ptr addrspace(1) %root)] + %result = call i64 @llvm.experimental.gc.result.i64(token %statepoint) + %root.relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %statepoint, + i32 0, + i32 0 + ) + %bits.relocated = ptrtoint ptr addrspace(1) %root.relocated to i64 + %combined = xor i64 %result, %bits.relocated + ret i64 %combined +} diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md new file mode 100644 index 0000000000..e40772437d --- /dev/null +++ b/docs/statepoint-gc-experiment.md @@ -0,0 +1,250 @@ +# Explicit statepoint GC experiment + +Date: 2026-07-31 + +Branch: `exp/stackmap-viability` + +Base commit: `e2557c1a985cb983ed00aafd1a2c1b31f1570b98` + +## Decision + +The explicit `gc.statepoint` bridge is correct enough to validate the +mechanism, but it is not currently a performance win and does not yet make +Perry's GC simpler. Keep the shadow stack as the default. + +- All eight GC-ratchet probes pass normally and with forced evacuation plus + evacuation verification. +- The full suite emits 1,080 statepoints and 1,562 relocations. It has zero + plain-stack-map fallbacks at GC-relevant calls. +- Runtime is effectively flat versus the shadow stack: -0.27% geometric mean + on a heavily loaded host. It is 1.42% slower than the plain-stack-map arm. +- Uncached compilation is 2.12% slower than shadow-stack compilation. +- Statepoint stack-map payload is 2.01x the plain-map payload across the suite. +- Relocation is better expressed: LLVM now owns the call/result/relocated-value + relationship, and the compiler memory barriers from the plain-map prototype + disappear. +- The whole system remains more complex because native unwinding, stack-map + parsing, textual call rewriting, root liveness, fallbacks, and + platform-specific metadata retention are still required. + +The prototype remains opt-in with `PERRY_STATEPOINTS=1`. The default +shadow-stack path is unchanged. + +## Which statepoint design this tests + +This is the explicit bridge, not LLVM's `RewriteStatepointsForGC` pipeline. +Perry emits the three intrinsics directly: + +1. Load each live NaN-boxed `i64` root from its existing native alloca and + temporarily convert the bits to `ptr addrspace(1)`. +2. Replace the original call with `llvm.experimental.gc.statepoint`. +3. Recover the call's scalar return through `llvm.experimental.gc.result`. +4. Recover every live root through `llvm.experimental.gc.relocate`, convert it + back to `i64`, and store it to the original alloca. + +The runtime collector executes inside the statepoint's callee. It unwinds to +the generated caller, finds LLVM's `Indirect` spill locations in +`__LLVM_STACKMAPS`, and rewrites those words during evacuation. The generated +caller then reloads the rewritten words through `gc.relocate`. + +The small standalone version is +[`statepoint-bridge-probe.ll`](statepoint-bridge-probe.ll). + +This choice intentionally avoids colliding with the representation experiment. +A full `RewriteStatepointsForGC` integration wants managed pointers to be +identifiable throughout SSA and expects a compiler pass to discover and +rewrite safepoints. Perry currently carries GC-capable values as NaN-boxed +`i64` words, so the bridge changes their representation only across one call. + +## Why LLVM calls it experimental + +The `llvm.experimental.*` prefix means LLVM does not promise a permanently +stable IR or binary interface across releases. It does not mean that the +mechanism is an abandoned toy or that production-oriented runtimes cannot use +it. For Perry it creates an engineering requirement: pin and test supported +LLVM versions, verify emitted IR, and treat upgrades as an ABI migration. + +The relevant upstream contracts are +[Garbage collection safepoints](https://llvm.org/docs/Statepoints.html) and +[Stack maps and patch points](https://llvm.org/docs/StackMaps.html). + +## Implementation + +The prototype reuses the precise-root discovery and conservative per-call CFG +liveness built for the plain-stack-map experiment. + +- Functions with roots receive `gc "statepoint-example"`. +- Ordinary direct calls with scalar arguments and scalar/void results are + rewritten explicitly. +- LLVM intrinsics and compiler-only inline assembly are not safepoints. +- Unsupported call forms retain the plain `llvm.experimental.stackmap` + fallback. +- A function containing Perry's setjmp-based `try` lowering uses the plain-map + backend for the whole function. +- The runtime parser accepts plain-map `Direct` alloca addresses and statepoint + `Indirect` spill locations. It deduplicates identical base/derived locations + before visiting roots. +- The module retains each Mach-O `__LLVM_STACKMAPS` atom with + `.no_dead_strip`. +- `PERRY_STATEPOINTS` participates in both build and object cache keys. + +When `PERRY_STATEPOINTS=1` and `PERRY_STACK_MAPS=1` are both present, +statepoints take precedence in eligible functions. + +## Correctness and coverage + +Final release artifacts passed: + +- all 8 GC-ratchet probes with stdout identical to Node; +- all 8 probes with `PERRY_GC_FORCE_EVACUATE=1` and + `PERRY_GC_VERIFY_EVACUATION=1`; +- LLVM 22 verification of all eight generated modules; +- compilation of the minimal bridge with Apple clang 21.0.0 and LLVM 22.1.4; +- 314 `perry-codegen` library tests; +- 5 focused runtime stack-map/statepoint parser and call-site tests; +- the object-cache statepoint environment-key test. + +Final generated-IR coverage: + +| Probe | Statepoints | Relocations | Plain fallbacks | +|---|---:|---:|---:| +| Nursery churn | 152 | 227 | 0 | +| Survivor promotion | 165 | 296 | 0 | +| Cross-generation writes | 168 | 244 | 0 | +| Dead after deep stack | 119 | 135 | 0 | +| Closure capture | 146 | 191 | 0 | +| String retention | 92 | 95 | 0 | +| Array grow/evacuate | 100 | 100 | 0 | +| Map/set side tables | 138 | 274 | 0 | +| **Total** | **1,080** | **1,562** | **0** | + +The zero here describes these probes, not the backend's complete call-form +coverage. Indirect calls, aggregate signatures, unusual call-site attributes, +and setjmp functions can still take the deliberate plain-map fallback. + +Retained heap and heap capacity match the shadow-stack arm byte-for-byte. +Promotions, freed bytes, and cycle counts also match. Two probes have tiny +copy-accounting differences (+3 objects/+208 bytes and -152 bytes), with +identical final retention; the other six match all checked GC counters. + +One apparent intermittent relocation failure during development was a stale +`target/*/libperry_runtime.a`. Perry executables link the +`perry-runtime-static` package, not the `perry-runtime` rlib directly. +Rebuilding only the latter left the old scanner in generated binaries. The +final results rebuild both the compiler and static runtime archive. + +## Performance + +Hardware was an Apple M1 Max on macOS 26.5. The interleaved run reported load +averages of 22.71/25.83/27.77, so these results are directional and should not +be promoted to release claims. + +Each runtime cell is the median of 11 executions. Mode order was interleaved +and rotated after one warmup, and outputs were checked for equality before +timing. + +| Probe | Shadow | Plain stack map | Statepoint | Statepoint vs plain | +|---|---:|---:|---:|---:| +| Nursery churn | 182.026 ms | 177.968 ms | 175.573 ms | -1.35% | +| Survivor promotion | 216.247 ms | 208.667 ms | 208.192 ms | -0.23% | +| Cross-generation writes | 210.743 ms | 203.775 ms | 204.504 ms | +0.36% | +| Dead after deep stack | 460.249 ms | 470.665 ms | 501.690 ms | +6.59% | +| Closure capture | 173.719 ms | 163.076 ms | 162.902 ms | -0.11% | +| String retention | 130.773 ms | 133.166 ms | 136.794 ms | +2.72% | +| Array grow/evacuate | 171.915 ms | 175.039 ms | 172.519 ms | -1.44% | +| Map/set side tables | 460.879 ms | 444.031 ms | 466.627 ms | +5.09% | + +Geometric means versus shadow: + +- plain stack maps: -1.66%; +- statepoints: -0.27%; +- statepoints versus plain maps: +1.42%. + +The deep-stack result is the strongest negative signal. Both native-stack +backends pay for unwinding, while statepoints additionally materialize +relocation spill/reload state around a large number of calls. + +Three uncached compilations per probe measured a +1.47% geometric mean for +plain maps and +2.12% for statepoints versus shadow. Sequential RSS +measurements put statepoints at roughly +1.03% median RSS and +0.70% peak RSS, +but allocator and host noise make those figures less reliable than retained +heap. + +Statepoint metadata is materially larger: + +| Probe | Plain payload | Statepoint payload | +|---|---:|---:| +| Nursery churn | 7,104 B | 13,656 B | +| Survivor promotion | 8,224 B | 16,112 B | +| Cross-generation writes | 7,768 B | 15,040 B | +| Dead after deep stack | 5,128 B | 14,824 B | +| Closure capture | 9,464 B | 18,440 B | +| String retention | 6,016 B | 10,912 B | +| Array grow/evacuate | 5,840 B | 11,480 B | +| Map/set side tables | 7,288 B | 13,840 B | + +The total is 114,304 bytes versus 56,832 bytes, or 2.01x. Most executables are +about 16 KiB larger than shadow after Mach-O segment rounding; closure capture +crosses another segment boundary and is about 33 KiB larger. + +## Is the GC simpler? + +Relocation is simpler to reason about, but the GC system is not simpler yet. + +The improvement is real: a statepoint makes the original call result and every +post-call root explicit SSA results. Plain maps required empty inline assembly +memory barriers to stop LLVM from caching root values across a call whose +stack slots the compiler could not know the collector mutates. + +However, this bridge still needs: + +- Perry's root discovery, logical slots, and conservative CFG liveness; +- addressable root allocas and per-statepoint load/store bridges; +- the LLVM stack-map v3 parser and native unwinder; +- Mach-O section discovery and linker-retention directives; +- call-form parsing and plain-map fallbacks; +- target- and toolchain-specific verification. + +It removes generated shadow-frame push/pop and TLS slot mutation, but replaces +that local machinery with a wider compiler/linker/runtime contract. The +collector itself is nearly unchanged; only its root source changes. + +## Is Perry better positioned? + +Semantically, yes. Operationally, not enough yet to switch. + +Statepoints provide the right vocabulary for a future moving collector: +relocation is explicit, base/derived relationships have a representation, and +a later managed-pointer pipeline can keep roots in SSA rather than forcing +Perry to invent compiler barriers. + +This implementation is still a bridge with important liabilities: + +- arbitrary NaN-boxed bits temporarily masquerade as managed pointers; +- all ordinary calls with live roots are treated as potentially allocating; +- indirect and unusual calls fall back to plain maps; +- `try`/setjmp functions fall back wholesale; +- scanning is macOS/Mach-O-only; +- active-frame matching still uses a 16-byte nearest-PC tolerance; +- the intrinsic and metadata contracts require LLVM-version discipline; +- the current measurements show no all-around speedup. + +## Recommended next step + +Do not replace the shadow stack from this branch. Preserve the prototype as +evidence and wait for the representation work before choosing the production +path. + +After that work lands: + +1. Define a safepoint-capability table so only calls that can enter the + allocator become statepoints. +2. Represent genuine managed references directly instead of converting every + possible NaN-box root through `inttoptr`. +3. Compare direct explicit emission with `RewriteStatepointsForGC` on that + representation. +4. Remove plain-map fallbacks one call form at a time and fail closed on + unsupported targets. +5. Re-run the 11-way interleaved suite on an idle pinned host, with separate + profiles for mutator root maintenance, relocation reloads, unwinding, and + collector root scanning. From c5aead07fb86280883638aa3eb35b2b2a8b6752c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 08:19:49 +0200 Subject: [PATCH 02/53] research(gc): measure and reduce native safepoints --- crates/perry-codegen/src/function.rs | 125 ++++++- crates/perry-codegen/src/gc_call_effects.rs | 113 +++++++ crates/perry-codegen/src/lib.rs | 2 + crates/perry-codegen/src/statepoint_report.rs | 319 ++++++++++++++++++ crates/perry-runtime/src/gc/copying.rs | 4 +- crates/perry-runtime/src/gc/root_words.rs | 11 +- crates/perry-runtime/src/gc/roots.rs | 33 +- .../perry-runtime/src/gc/roots/stack_maps.rs | 65 +++- crates/perry-runtime/src/gc/telemetry.rs | 33 ++ crates/perry-runtime/src/gc/tests/copying.rs | 13 + .../src/gc/tests/telemetry_verifier.rs | 8 + crates/perry-runtime/src/gc/verify.rs | 4 +- .../src/commands/compile/run_pipeline.rs | 34 ++ crates/perry/src/commands/compile/types.rs | 21 ++ crates/perry/src/commands/dev.rs | 1 + crates/perry/src/commands/run/mod.rs | 1 + docs/src/cli/flags.md | 2 + docs/stack-map-gc-experiment.md | 3 +- docs/statepoint-gc-experiment.md | 113 +++++++ 19 files changed, 865 insertions(+), 40 deletions(-) create mode 100644 crates/perry-codegen/src/gc_call_effects.rs create mode 100644 crates/perry-codegen/src/statepoint_report.rs diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 81250e3e8b..da635766ca 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -754,7 +754,7 @@ impl LlFunction { } else { PreciseRootBackend::StackMap }; - lower_precise_roots_to_native_stack(&ir, self.stack_map_slot_count, backend) + lower_precise_roots_to_native_stack(&ir, &self.name, self.stack_map_slot_count, backend) } else { ir }; @@ -948,6 +948,15 @@ enum PreciseRootBackend { Statepoint, } +impl PreciseRootBackend { + fn as_str(self) -> &'static str { + match self { + Self::StackMap => "stack-map", + Self::Statepoint => "statepoint", + } + } +} + #[derive(Debug, Eq, PartialEq)] struct DirectCall<'a> { result: Option<&'a str>, @@ -1046,6 +1055,23 @@ fn parse_direct_statepoint_call(line: &str) -> Option> { }) } +/// Return a direct callee name without the leading `@`. +/// +/// This accepts more call syntax than the statepoint parser because the +/// GC-effect audit only needs to recognize a direct target. Unsupported and +/// indirect forms return `None` and therefore stay conservative. +fn direct_callee_name(line: &str) -> Option<&str> { + let call = line.trim().split_once("call ")?.1; + let args_open = call.find('(')?; + let target = call[..args_open].trim(); + let name = target.split_ascii_whitespace().last()?.strip_prefix('@')?; + (!name.is_empty() + && name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '$'))) + .then_some(name) +} + fn gc_result_suffix(ty: &str) -> Option<&'static str> { match ty { "i1" => Some("i1"), @@ -1159,6 +1185,7 @@ fn emit_statepoint(out: &mut String, call: &DirectCall<'_>, live: &[&String], st /// relocation path. fn lower_precise_roots_to_native_stack( ir: &str, + function_name: &str, slot_count: u32, backend: PreciseRootBackend, ) -> String { @@ -1183,12 +1210,24 @@ fn lower_precise_roots_to_native_stack( let slot_roots = roots; let root_ptrs: Vec = slot_roots.iter().flatten().cloned().collect(); + let mut report = crate::statepoint_report::enabled().then(|| { + crate::statepoint_report::FunctionRecord::new( + function_name, + backend.as_str(), + slot_count, + root_ptrs.len(), + ) + }); if root_ptrs.is_empty() { - return ir + let out = ir .lines() .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) .map(|line| format!("{line}\n")) .collect(); + if let Some(report) = report { + crate::statepoint_report::record(report); + } + return out; } let mut out = String::with_capacity(ir.len() + root_ptrs.len() * 128); @@ -1255,16 +1294,28 @@ fn lower_precise_roots_to_native_stack( .filter_map(|(_, ptr)| ptr.as_ref()) .filter(|ptr| available.contains(*ptr) && initialized.contains(*ptr)) .collect(); + if let Some(report) = report.as_mut() { + report.note_call(live.len()); + } if live.is_empty() { continue; } - if backend == PreciseRootBackend::Statepoint - && (trimmed.contains("@llvm.") || trimmed.contains("call void asm ")) - { - // LLVM intrinsics and zero-instruction compiler barriers cannot - // enter Perry's allocator, so they are not safepoints. The plain - // stack-map prototype instrumented every textual `call`; the - // statepoint path can make this distinction without losing roots. + + let direct_callee = direct_callee_name(line); + let is_compiler_only = direct_callee.is_some_and(|callee| callee.starts_with("llvm.")) + || trimmed.contains("call void asm "); + let cannot_collect = direct_callee.is_some_and(|callee| { + crate::gc_call_effects::classify_direct_callee(callee) + == crate::gc_call_effects::GcCallEffect::CannotCollect + }); + if is_compiler_only || cannot_collect { + // LLVM intrinsics, zero-instruction compiler barriers, and + // runtime helpers in the audited GC-effect table cannot enter + // Perry's allocator. Neither native-stack backend needs metadata + // around them. + if let Some(report) = report.as_mut() { + report.note_skipped(direct_callee.unwrap_or("")); + } continue; } @@ -1274,28 +1325,42 @@ fn lower_precise_roots_to_native_stack( if backend == PreciseRootBackend::Statepoint { if let Some(call) = parse_direct_statepoint_call(line) { emit_statepoint(&mut out, &call, &live, map_id); + if let Some(report) = report.as_mut() { + report.note_statepoint(call.callee.trim_start_matches('@'), live.len()); + } map_id += 1; continue; } } emit_plain_stack_map(&mut out, line, &live, map_id); + if let Some(report) = report.as_mut() { + report.note_plain_stack_map( + direct_callee.unwrap_or(""), + live.len(), + backend == PreciseRootBackend::Statepoint, + ); + } map_id += 1; } + if let Some(report) = report { + crate::statepoint_report::record(report); + } out } #[cfg(test)] mod stack_map_tests { use super::{ - lower_precise_roots_to_native_stack, parse_direct_statepoint_call, PreciseRootBackend, + direct_callee_name, lower_precise_roots_to_native_stack, parse_direct_statepoint_call, + PreciseRootBackend, }; fn lower_stack_maps(input: &str, slots: u32) -> String { - lower_precise_roots_to_native_stack(input, slots, PreciseRootBackend::StackMap) + lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::StackMap) } fn lower_statepoints(input: &str, slots: u32) -> String { - lower_precise_roots_to_native_stack(input, slots, PreciseRootBackend::Statepoint) + lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::Statepoint) } #[test] @@ -1377,6 +1442,15 @@ merge.3: #[test] fn parses_the_scalar_direct_call_subset() { + assert_eq!( + direct_callee_name(" %r7 = call double @foo(i64 %r1, ptr %r2)"), + Some("foo") + ); + assert_eq!( + direct_callee_name(" %r7 = call i64 ()* %fn()"), + None, + "an indirect target must not be inferred from its arguments" + ); assert_eq!( parse_direct_statepoint_call(" %r7 = call double @foo(i64 %r1, ptr %r2)"), Some(super::DirectCall { @@ -1460,4 +1534,31 @@ entry.0: ); assert!(!output.contains("@llvm.experimental.stackmap")); } + + #[test] + fn audited_non_collecting_helpers_are_not_safepoints_in_either_backend() { + let input = r#"define void @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @js_gc_temp_root_push(i64 %arg) + call void @js_write_barrier_root_nanbox(i64 %arg) + call void @js_gc_loop_safepoint() + ret void +} +"#; + for output in [lower_stack_maps(input, 1), lower_statepoints(input, 1)] { + assert!(output.contains("call void @js_gc_temp_root_push(i64 %arg)")); + assert!(output.contains("call void @js_write_barrier_root_nanbox(i64 %arg)")); + assert_eq!( + output.matches("@llvm.experimental.stackmap").count() + + output + .matches("@llvm.experimental.gc.statepoint.p0") + .count(), + 1, + "only the explicit collection boundary should be a safepoint:\n{output}" + ); + } + } } diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs new file mode 100644 index 0000000000..3f960199fd --- /dev/null +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -0,0 +1,113 @@ +//! Perry-GC call effects for native-stack safepoint lowering. +//! +//! This is deliberately narrower than LLVM's memory-effect attributes. A +//! helper may mutate runtime metadata, take a lock, or allocate through the +//! system allocator and still be safe to omit as a Perry GC safepoint. The +//! only question answered here is: can this call enter Perry's collector? +//! +//! Unknown is the safe default. Adding a helper to the allowlist requires +//! auditing the complete runtime call graph for `gc_check_trigger`, +//! `js_gc_collect`, `js_gc_loop_safepoint`, or another route into collection. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum GcCallEffect { + CannotCollect, + Unknown, +} + +/// Classify one direct LLVM callee name, without the leading `@`. +pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { + match name { + // Pure/read-only ABI helpers audited in `module::helper_decl_attrs`. + "js_nanbox_pointer" + | "js_nanbox_get_pointer" + | "js_typed_f64_arg_guard" + | "js_typed_i32_arg_guard" + | "js_typed_i1_arg_guard" + | "js_typed_i1_arg_to_raw" + | "js_typed_i32_arg_to_raw" + | "js_typed_string_arg_guard" + | "js_is_truthy" + | "js_typed_feedback_plain_array_index_get_guard" + | "js_typed_feedback_numeric_array_index_get_guard" + | "js_typed_feedback_plain_array_index_set_guard" + | "js_typed_feedback_numeric_array_index_set_guard" + | "js_typed_feedback_numeric_array_push_guard" + | "js_array_numeric_value_to_raw_f64" + // `gc/roots/temp_roots.rs`: TLS vector operations and an incremental + // marking barrier only. They never run a Perry collection. + | "js_gc_temp_root_push" + | "js_gc_temp_root_get" + | "js_gc_temp_root_set" + | "js_gc_temp_root_truncate" + // `gc/barrier.rs`: remembered-set / incremental-marking maintenance. + | "js_write_barrier" + | "js_write_barrier_slot" + | "js_write_barrier_root_heap_word" + | "js_write_barrier_root_nanbox" + // `gc/layout.rs`: side-table metadata updates only. + | "js_gc_note_slot_layout" + | "js_gc_note_slot_layout_aware" + | "js_gc_init_typed_shape_layout" + | "js_gc_init_unboxed_object_layout" + // `typed_feedback.rs`: counters/registries only. This intentionally + // does not include feedback wrappers that perform the actual object + // get/set operation. + | "js_typed_feedback_record_guard_pass" + | "js_typed_feedback_record_guard_fail" + | "js_typed_feedback_record_fallback_call" + | "js_typed_feedback_class_field_get_guard" + | "js_typed_feedback_class_field_set_guard" + | "js_typed_feedback_observe_property_get" + | "js_typed_feedback_observe_property_set" + // Refcount writes and array-layout observations; none enters GC. + | "js_string_addref" + | "js_string_addref_if_heap_string" + | "js_array_clear_numeric_layout" + | "js_array_note_numeric_write" + | "js_array_is_numeric_f64_layout" + // TLS dynamic-call context only. + | "js_implicit_this_set" + | "js_new_target_get" + | "js_new_target_set" => GcCallEffect::CannotCollect, + _ => GcCallEffect::Unknown, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audited_runtime_bookkeeping_cannot_collect() { + for name in [ + "js_gc_temp_root_push", + "js_write_barrier_root_nanbox", + "js_gc_note_slot_layout", + "js_typed_feedback_record_guard_pass", + "js_string_addref", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::CannotCollect, + "{name}" + ); + } + } + + #[test] + fn collection_and_unknown_calls_stay_conservative() { + for name in [ + "js_gc_collect", + "js_gc_loop_safepoint", + "js_alloc_object", + "user_function", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::Unknown, + "{name}" + ); + } + } +} diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index 6d87dde4fe..45a9a626e9 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod collectors; pub mod expr; pub mod ext_registry; pub mod function; +pub(crate) mod gc_call_effects; pub mod linker; pub(crate) mod loop_purity; pub(crate) mod lower_array_method; @@ -24,6 +25,7 @@ pub(crate) mod nm_install; pub mod opt_report; pub mod runtime_decls; pub(crate) mod setjmp_abi; +pub mod statepoint_report; pub(crate) mod stmt; pub mod strings; pub mod stubs; diff --git a/crates/perry-codegen/src/statepoint_report.rs b/crates/perry-codegen/src/statepoint_report.rs new file mode 100644 index 0000000000..6cd034d460 --- /dev/null +++ b/crates/perry-codegen/src/statepoint_report.rs @@ -0,0 +1,319 @@ +//! Observational root-pressure report for the native-stack GC experiments. +//! +//! `PERRY_STATEPOINT_REPORT=1|text|json` records how many textual calls see +//! live roots, which ones can be omitted after the GC-effect audit, and how +//! much statepoint/stack-map metadata remains. Codegen never reads the data +//! back, so enabling the report cannot affect emitted IR. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::sync::{Mutex, OnceLock}; + +#[derive(Clone, Debug, Default, Eq, PartialEq, serde::Serialize)] +pub struct FunctionRecord { + function: String, + backend: String, + reserved_logical_slots: u32, + bound_native_slots: usize, + textual_calls: u64, + calls_without_live_roots: u64, + calls_with_live_roots: u64, + skipped_non_safepoints: u64, + statepoints: u64, + relocations: u64, + plain_stack_maps: u64, + stack_map_operands: u64, + statepoint_fallbacks: u64, + max_live_roots: usize, + live_roots_histogram: BTreeMap, + statepoints_by_callee: BTreeMap, + skipped_by_callee: BTreeMap, + fallbacks_by_callee: BTreeMap, +} + +impl FunctionRecord { + pub(crate) fn new( + function: &str, + backend: &str, + reserved_logical_slots: u32, + bound_native_slots: usize, + ) -> Self { + Self { + function: function.to_string(), + backend: backend.to_string(), + reserved_logical_slots, + bound_native_slots, + ..Self::default() + } + } + + pub(crate) fn note_call(&mut self, live_roots: usize) { + self.textual_calls += 1; + if live_roots == 0 { + self.calls_without_live_roots += 1; + } else { + self.calls_with_live_roots += 1; + } + } + + pub(crate) fn note_skipped(&mut self, callee: &str) { + self.skipped_non_safepoints += 1; + *self + .skipped_by_callee + .entry(callee.to_string()) + .or_default() += 1; + } + + fn note_emitted_roots(&mut self, live_roots: usize) { + self.max_live_roots = self.max_live_roots.max(live_roots); + *self.live_roots_histogram.entry(live_roots).or_default() += 1; + } + + pub(crate) fn note_statepoint(&mut self, callee: &str, live_roots: usize) { + self.statepoints += 1; + self.relocations += live_roots as u64; + self.note_emitted_roots(live_roots); + *self + .statepoints_by_callee + .entry(callee.to_string()) + .or_default() += 1; + } + + pub(crate) fn note_plain_stack_map( + &mut self, + callee: &str, + live_roots: usize, + is_statepoint_fallback: bool, + ) { + self.plain_stack_maps += 1; + self.stack_map_operands += live_roots as u64; + self.note_emitted_roots(live_roots); + if is_statepoint_fallback { + self.statepoint_fallbacks += 1; + *self + .fallbacks_by_callee + .entry(callee.to_string()) + .or_default() += 1; + } + } +} + +pub fn enabled() -> bool { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_STATEPOINT_REPORT").as_deref(), + Ok("1") | Ok("text") | Ok("json") + ) + }) +} + +static SINK: OnceLock>> = OnceLock::new(); + +fn sink() -> &'static Mutex> { + SINK.get_or_init(|| Mutex::new(Vec::new())) +} + +pub(crate) fn record(record: FunctionRecord) { + if enabled() { + if let Ok(mut records) = sink().lock() { + records.push(record); + } + } +} + +pub fn take_records() -> Vec { + let mut records = match sink().lock() { + Ok(mut records) => std::mem::take(&mut *records), + Err(_) => Vec::new(), + }; + records.sort_by(|a, b| { + (&a.function, &a.backend, a.reserved_logical_slots).cmp(&( + &b.function, + &b.backend, + b.reserved_logical_slots, + )) + }); + records.dedup(); + records +} + +#[derive(Default, serde::Serialize)] +struct Totals { + functions: usize, + reserved_logical_slots: u64, + bound_native_slots: u64, + textual_calls: u64, + calls_without_live_roots: u64, + calls_with_live_roots: u64, + skipped_non_safepoints: u64, + statepoints: u64, + relocations: u64, + plain_stack_maps: u64, + stack_map_operands: u64, + statepoint_fallbacks: u64, + max_live_roots: usize, + live_roots_histogram: BTreeMap, + statepoints_by_callee: BTreeMap, + skipped_by_callee: BTreeMap, + fallbacks_by_callee: BTreeMap, +} + +fn totals(records: &[FunctionRecord]) -> Totals { + let mut out = Totals { + functions: records.len(), + ..Totals::default() + }; + for record in records { + out.reserved_logical_slots += u64::from(record.reserved_logical_slots); + out.bound_native_slots += record.bound_native_slots as u64; + out.textual_calls += record.textual_calls; + out.calls_without_live_roots += record.calls_without_live_roots; + out.calls_with_live_roots += record.calls_with_live_roots; + out.skipped_non_safepoints += record.skipped_non_safepoints; + out.statepoints += record.statepoints; + out.relocations += record.relocations; + out.plain_stack_maps += record.plain_stack_maps; + out.stack_map_operands += record.stack_map_operands; + out.statepoint_fallbacks += record.statepoint_fallbacks; + out.max_live_roots = out.max_live_roots.max(record.max_live_roots); + for (width, count) in &record.live_roots_histogram { + *out.live_roots_histogram.entry(*width).or_default() += count; + } + for (callee, count) in &record.statepoints_by_callee { + *out.statepoints_by_callee.entry(callee.clone()).or_default() += count; + } + for (callee, count) in &record.skipped_by_callee { + *out.skipped_by_callee.entry(callee.clone()).or_default() += count; + } + for (callee, count) in &record.fallbacks_by_callee { + *out.fallbacks_by_callee.entry(callee.clone()).or_default() += count; + } + } + out +} + +fn render_ranked_map(out: &mut String, heading: &str, values: &BTreeMap) { + if values.is_empty() { + return; + } + let mut rows: Vec<_> = values.iter().collect(); + rows.sort_by(|(name_a, count_a), (name_b, count_b)| { + count_b.cmp(count_a).then_with(|| name_a.cmp(name_b)) + }); + let _ = writeln!(out, "{heading}"); + for (name, count) in rows.into_iter().take(25) { + let _ = writeln!(out, " {count:>6} {name}"); + } + out.push('\n'); +} + +pub fn render_text(records: &[FunctionRecord]) -> String { + let totals = totals(records); + let mut out = String::from( + "Perry native-stack GC report (--statepoint-report)\n\ + ==================================================\n\n", + ); + if records.is_empty() { + out.push_str( + "No native-stack lowering records were emitted. Enable PERRY_STACK_MAPS=1\n\ + or PERRY_STATEPOINTS=1 and ensure codegen is not served from cache.\n", + ); + return out; + } + + let emitted = totals.statepoints + totals.plain_stack_maps; + let _ = writeln!( + out, + "{} function(s), {} bound native root slots ({} logical slots reserved)", + totals.functions, totals.bound_native_slots, totals.reserved_logical_slots + ); + let _ = writeln!( + out, + "{} textual calls: {} with live roots, {} without", + totals.textual_calls, totals.calls_with_live_roots, totals.calls_without_live_roots + ); + let _ = writeln!( + out, + "{} safepoints emitted: {} statepoints, {} plain stack maps", + emitted, totals.statepoints, totals.plain_stack_maps + ); + let _ = writeln!( + out, + "{} non-collecting calls skipped; {} statepoint parser fallback(s)", + totals.skipped_non_safepoints, totals.statepoint_fallbacks + ); + let _ = writeln!( + out, + "{} relocations, {} plain-map operands; maximum {} live roots at one safepoint\n", + totals.relocations, totals.stack_map_operands, totals.max_live_roots + ); + + if !totals.live_roots_histogram.is_empty() { + out.push_str("Live roots per emitted safepoint\n"); + for (width, count) in &totals.live_roots_histogram { + let _ = writeln!(out, " {width:>4} root(s): {count:>6} safepoint(s)"); + } + out.push('\n'); + } + render_ranked_map( + &mut out, + "Most frequent explicit statepoint callees", + &totals.statepoints_by_callee, + ); + render_ranked_map( + &mut out, + "Calls omitted by the GC-effect audit", + &totals.skipped_by_callee, + ); + render_ranked_map( + &mut out, + "Plain-map fallbacks in statepoint mode", + &totals.fallbacks_by_callee, + ); + out +} + +#[derive(serde::Serialize)] +struct JsonReport<'a> { + schema_version: u32, + totals: Totals, + functions: &'a [FunctionRecord], +} + +pub fn render_json(records: &[FunctionRecord]) -> String { + serde_json::to_string_pretty(&JsonReport { + schema_version: 1, + totals: totals(records), + functions: records, + }) + .unwrap_or_else(|error| format!("{{\"error\":\"{error}\"}}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_and_json_expose_root_pressure_and_fallbacks() { + let mut record = FunctionRecord::new("probe", "statepoint", 3, 2); + record.note_call(2); + record.note_statepoint("@may_collect", 2); + record.note_call(1); + record.note_skipped("@js_gc_temp_root_get"); + record.note_call(1); + record.note_plain_stack_map("", 1, true); + + let text = render_text(std::slice::from_ref(&record)); + assert!(text.contains("2 bound native root slots")); + assert!(text.contains("1 non-collecting calls skipped")); + assert!(text.contains("1 statepoint parser fallback(s)")); + assert!(text.contains("@js_gc_temp_root_get")); + + let json = render_json(&[record]); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["schema_version"], 1); + assert_eq!(parsed["totals"]["relocations"], 2); + assert_eq!(parsed["totals"]["statepoint_fallbacks"], 1); + } +} diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 01a4091524..f0e055a6d0 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1023,7 +1023,7 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( collector.stats.malloc_sweep_due = malloc_sweep_due; collector.stats.reset_blocks += crate::arena::copying_prepare_to_space(); - visit_mutable_root_slots(|slot| unsafe { + let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { let bits = slot.read(); if let Some(trace) = trace.as_mut() { let pointer_root = collector.ptrs.decode_bits(bits).is_some(); @@ -1046,6 +1046,8 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( } } }); + let mut root_sources = trace.as_mut().map(|trace| &mut trace.root_sources); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); let scanners: Vec = MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()); { diff --git a/crates/perry-runtime/src/gc/root_words.rs b/crates/perry-runtime/src/gc/root_words.rs index 78b5016a3c..705a586bd9 100644 --- a/crates/perry-runtime/src/gc/root_words.rs +++ b/crates/perry-runtime/src/gc/root_words.rs @@ -143,13 +143,12 @@ pub(super) fn decode_root_word(bits: u64) -> Option { }) } -/// Mark the object referenced by a mutable-root slot word — shadow-stack -/// slots (`MutableRootSlotKind::ShadowStack`) and registered module-global -/// roots (`MutableRootSlotKind::GlobalRoot`) alike. +/// Mark the object referenced by a mutable-root slot word — shadow-stack, +/// native stack-map, and registered module-global slots alike. /// -/// Both slot kinds are rewritten by `rewrite_mutable_root_slots` through -/// `try_rewrite_value`, so both must be marked through the same decoder -/// (#6910). Marking is address-identical for the two forms once decoded — +/// All slot kinds are rewritten by `rewrite_mutable_root_slots` through +/// `try_rewrite_value`, so all must be marked through the same decoder +/// (#6910). Marking is address-identical for the forms once decoded — /// `try_mark_raw_root_addr` performs the same validation and mark that /// `try_mark_value` does after unwrapping a NaN box — so a single call /// covers them. diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index ffb6af3749..19b6128d55 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -8,6 +8,7 @@ mod shadow_stack; mod stack_maps; mod temp_roots; pub(super) use stack_maps::initialize as initialize_stack_maps; +pub(super) use stack_maps::record_native_stack_walk_source; pub(super) use runtime_handles::{ new_runtime_handle_root_scan_state, scan_runtime_handle_roots_mut, @@ -1341,13 +1342,14 @@ pub(super) fn atomic_store_ordering( /// Which registry a mutable root slot came from. /// /// The kind selects a *telemetry bucket* only — it must never select a -/// different pointer decoding. Both kinds are marked by -/// `mark_mutable_root_bits` and rewritten by `try_rewrite_value`, and both +/// different pointer decoding. All kinds are marked by +/// `mark_mutable_root_bits` and rewritten by `try_rewrite_value`, and all /// therefore accept a heap reference either NaN-boxed or bare. That symmetry /// is the #6910 invariant; see `gc::root_words`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum MutableRootSlotKind { ShadowStack, + NativeStack, GlobalRoot, } @@ -1372,8 +1374,10 @@ impl MutableRootSlot { /// Visit every live shadow-stack slot. The visitor receives real /// mutable slot addresses so the same walk can support mark-only /// scanning and post-forwarding rewrites. -pub(super) fn visit_shadow_stack_root_slots(mut visit: impl FnMut(MutableRootSlot)) { - stack_maps::visit_stack_map_root_slots(&mut visit); +pub(super) fn visit_shadow_stack_root_slots( + mut visit: impl FnMut(MutableRootSlot), +) -> stack_maps::NativeStackWalkStats { + let native_stack_walk = stack_maps::visit_stack_map_root_slots(&mut visit); SHADOW.with(|cell| unsafe { let s = &mut *cell.get(); if s.len == 0 || s.ptr.is_null() { @@ -1415,6 +1419,7 @@ pub(super) fn visit_shadow_stack_root_slots(mut visit: impl FnMut(MutableRootSlo top = header.value as usize; } }); + native_stack_walk } /// Visit every registered module-global root slot. @@ -1435,9 +1440,12 @@ pub(super) fn visit_global_root_slots(mut visit: impl FnMut(MutableRootSlot)) { /// Visit the root slots whose storage is owned by this runtime and can /// therefore be rewritten after evacuation. -pub(super) fn visit_mutable_root_slots(mut visit: impl FnMut(MutableRootSlot)) { - visit_shadow_stack_root_slots(&mut visit); +pub(super) fn visit_mutable_root_slots( + mut visit: impl FnMut(MutableRootSlot), +) -> stack_maps::NativeStackWalkStats { + let native_stack_walk = visit_shadow_stack_root_slots(&mut visit); visit_global_root_slots(&mut visit); + native_stack_walk } #[derive(Default)] @@ -1462,7 +1470,7 @@ pub(super) fn mark_mutable_root_slots_step( } let mut seen = 0usize; let mut exhausted = true; - visit_shadow_stack_root_slots(|slot| unsafe { + let native_stack_walk = visit_shadow_stack_root_slots(|slot| unsafe { if seen < cursor.shadow_seen { seen += 1; return; @@ -1474,8 +1482,10 @@ pub(super) fn mark_mutable_root_slots_step( let bits = slot.read(); record_mutable_slot_scan_source(slot, bits, valid_ptrs, &mut root_sources); - if let Some(stats) = shadow_stats.as_mut() { - stats.record_scan(bits); + if matches!(slot.kind, MutableRootSlotKind::ShadowStack) { + if let Some(stats) = shadow_stats.as_mut() { + stats.record_scan(bits); + } } if bits != 0 { mark_mutable_root_bits(bits, valid_ptrs); @@ -1484,6 +1494,7 @@ pub(super) fn mark_mutable_root_slots_step( cursor.shadow_seen = seen; remaining -= 1; }); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); if !exhausted { return false; } @@ -1557,6 +1568,7 @@ pub(super) fn root_source_for_mutable_slot( ) -> &mut RootSourceSlotTraceStats { match kind { MutableRootSlotKind::ShadowStack => &mut sources.compiled_shadow, + MutableRootSlotKind::NativeStack => &mut sources.compiled_native, MutableRootSlotKind::GlobalRoot => &mut sources.module_globals, } } @@ -1596,7 +1608,7 @@ pub(super) fn mark_mutable_root_slots( mut shadow_stats: Option<&mut ShadowRootTraceStats>, mut root_sources: Option<&mut RootSourcesTraceStats>, ) { - visit_mutable_root_slots(|slot| unsafe { + let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { let bits = slot.read(); record_mutable_slot_scan_source(slot, bits, valid_ptrs, &mut root_sources); if matches!(slot.kind, MutableRootSlotKind::ShadowStack) { @@ -1609,6 +1621,7 @@ pub(super) fn mark_mutable_root_slots( } mark_mutable_root_bits(bits, valid_ptrs); }); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); } #[inline] diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 436dbb40b5..5c4c0b173a 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -14,6 +14,7 @@ //! there. use super::{MutableRootSlot, MutableRootSlotKind}; +use crate::gc::telemetry::RootSourcesTraceStats; use std::ffi::c_void; use std::sync::OnceLock; @@ -35,6 +36,29 @@ struct StackMapRecord { static STACK_MAPS: OnceLock> = OnceLock::new(); +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(in crate::gc) struct NativeStackWalkStats { + pub(in crate::gc) walks: usize, + pub(in crate::gc) frames_visited: usize, + pub(in crate::gc) records_matched: usize, + pub(in crate::gc) locations_visited: usize, +} + +#[inline] +pub(in crate::gc) fn record_native_stack_walk_source( + stats: NativeStackWalkStats, + root_sources: &mut Option<&mut RootSourcesTraceStats>, +) { + if let Some(sources) = root_sources { + sources.native_stack_maps.record_walk( + stats.walks, + stats.frames_visited, + stats.records_matched, + stats.locations_visited, + ); + } +} + pub(in crate::gc) fn initialize() { let _ = stack_maps(); } @@ -71,12 +95,14 @@ fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { } } -pub(super) fn visit_stack_map_root_slots(visit: &mut impl FnMut(MutableRootSlot)) { +pub(super) fn visit_stack_map_root_slots( + visit: &mut impl FnMut(MutableRootSlot), +) -> NativeStackWalkStats { let maps = stack_maps(); if maps.is_empty() { - return; + return NativeStackWalkStats::default(); } - unwind::visit(maps, visit); + unwind::visit(maps, visit) } fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { @@ -339,16 +365,28 @@ mod unwind { struct WalkState<'a, F> { maps: &'a [StackMapRecord], visit: &'a mut F, + stats: NativeStackWalkStats, } - pub(super) fn visit(maps: &[StackMapRecord], visit: &mut F) { - let mut state = WalkState { maps, visit }; + pub(super) fn visit( + maps: &[StackMapRecord], + visit: &mut F, + ) -> NativeStackWalkStats { + let mut state = WalkState { + maps, + visit, + stats: NativeStackWalkStats { + walks: 1, + ..NativeStackWalkStats::default() + }, + }; unsafe { _Unwind_Backtrace( walk_frame::, (&mut state as *mut WalkState<'_, _>).cast::(), ); } + state.stats } unsafe extern "C" fn walk_frame( @@ -356,6 +394,7 @@ mod unwind { argument: *mut c_void, ) -> i32 { let state = &mut *argument.cast::>(); + state.stats.frames_visited = state.stats.frames_visited.saturating_add(1); let ip = _Unwind_GetIP(context); let Some(candidate_pc) = closest_record_pc(state.maps, ip) else { return 0; @@ -371,8 +410,13 @@ mod unwind { let last = state .maps .partition_point(|record| record.pc <= candidate_pc); + state.stats.records_matched = state + .stats + .records_matched + .saturating_add(last.saturating_sub(first)); for record in &state.maps[first..last] { for location in &record.locations { + state.stats.locations_visited = state.stats.locations_visited.saturating_add(1); let base = _Unwind_GetGR(context, i32::from(location.dwarf_reg)); let address = if location.offset < 0 { base.checked_sub(location.offset.unsigned_abs() as usize) @@ -386,9 +430,7 @@ mod unwind { continue; } (state.visit)(MutableRootSlot { - // Reuse the compiled-frame telemetry bucket so the - // experiment compares root source counts directly. - kind: MutableRootSlotKind::ShadowStack, + kind: MutableRootSlotKind::NativeStack, ptr: address as *mut u64, }); } @@ -401,7 +443,12 @@ mod unwind { mod unwind { use super::*; - pub(super) fn visit(_maps: &[StackMapRecord], _visit: &mut impl FnMut(MutableRootSlot)) {} + pub(super) fn visit( + _maps: &[StackMapRecord], + _visit: &mut impl FnMut(MutableRootSlot), + ) -> NativeStackWalkStats { + NativeStackWalkStats::default() + } } #[cfg(test)] diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index ec41683930..f557d20a55 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -300,13 +300,39 @@ pub(super) struct NativeStackFallbackTraceStats { pub(super) compiled_frame_pinned_bytes: usize, } +#[derive(Clone, Copy, Default)] +pub(super) struct NativeStackMapTraceStats { + pub(super) walks: usize, + pub(super) frames_visited: usize, + pub(super) records_matched: usize, + pub(super) locations_visited: usize, +} + +impl NativeStackMapTraceStats { + #[inline] + pub(super) fn record_walk( + &mut self, + walks: usize, + frames_visited: usize, + records_matched: usize, + locations_visited: usize, + ) { + self.walks = self.walks.saturating_add(walks); + self.frames_visited = self.frames_visited.saturating_add(frames_visited); + self.records_matched = self.records_matched.saturating_add(records_matched); + self.locations_visited = self.locations_visited.saturating_add(locations_visited); + } +} + #[derive(Clone, Copy, Default)] pub(super) struct RootSourcesTraceStats { pub(super) compiled_shadow: RootSourceSlotTraceStats, + pub(super) compiled_native: RootSourceSlotTraceStats, pub(super) module_globals: RootSourceSlotTraceStats, pub(super) runtime_handles: RootSourceSlotTraceStats, pub(super) runtime_mutable_scanners: RootSourceSlotTraceStats, pub(super) ffi_mutable_scanners: RootSourceSlotTraceStats, + pub(super) native_stack_maps: NativeStackMapTraceStats, pub(super) native_stack_fallback: NativeStackFallbackTraceStats, } @@ -1289,10 +1315,17 @@ pub(super) fn root_source_slot_json(stats: RootSourceSlotTraceStats) -> serde_js pub(super) fn root_sources_json(stats: RootSourcesTraceStats) -> serde_json::Value { serde_json::json!({ "compiled_shadow": root_source_slot_json(stats.compiled_shadow), + "compiled_native": root_source_slot_json(stats.compiled_native), "module_globals": root_source_slot_json(stats.module_globals), "runtime_handles": root_source_slot_json(stats.runtime_handles), "runtime_mutable_scanners": root_source_slot_json(stats.runtime_mutable_scanners), "ffi_mutable_scanners": root_source_slot_json(stats.ffi_mutable_scanners), + "native_stack_maps": { + "walks": stats.native_stack_maps.walks, + "frames_visited": stats.native_stack_maps.frames_visited, + "records_matched": stats.native_stack_maps.records_matched, + "locations_visited": stats.native_stack_maps.locations_visited, + }, "native_stack_fallback": { "decision": stats.native_stack_fallback.decision.as_str(), "scanned": stats.native_stack_fallback.scanned, diff --git a/crates/perry-runtime/src/gc/tests/copying.rs b/crates/perry-runtime/src/gc/tests/copying.rs index cffaf0074f..c35ed39488 100644 --- a/crates/perry-runtime/src/gc/tests/copying.rs +++ b/crates/perry-runtime/src/gc/tests/copying.rs @@ -770,6 +770,19 @@ fn root_source_active_shadow_frame_reports_precise_shadow_roots_only() { ); } +#[test] +fn root_source_native_stack_slot_has_a_distinct_telemetry_bucket() { + let mut sources = RootSourcesTraceStats::default(); + root_source_for_mutable_slot(&mut sources, MutableRootSlotKind::NativeStack) + .record_scan(true, true); + root_source_for_mutable_slot(&mut sources, MutableRootSlotKind::NativeStack).record_rewrite(); + + assert_eq!(sources.compiled_native.slots_scanned, 1); + assert_eq!(sources.compiled_native.pointer_roots, 1); + assert_eq!(sources.compiled_native.rewritten_slots, 1); + assert_eq!(sources.compiled_shadow.slots_scanned, 0); +} + #[test] fn test_copied_minor_eligibility_empty_rust_copy_only_scanner_falls_back() { let _guard = CopyingNurseryTestGuard::new(0); diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index 81b5c1a7c6..2f8783e369 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -243,6 +243,14 @@ fn root_heavy_workload_reports_root_sources_and_budgeted_progression() { >= u64::from(roots), "shadow-root telemetry should classify the roots as pointers" ); + assert!( + event["root_sources"]["compiled_native"].is_object(), + "native stack-map roots need their own source bucket" + ); + assert!( + event["root_sources"]["native_stack_maps"]["frames_visited"].is_number(), + "native stack-map telemetry should expose unwinder work" + ); let live_after = (js_shadow_slot_get(0) & POINTER_MASK) as *const crate::StringHeader; unsafe { diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index ef15f56b32..7df3ca728b 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -892,7 +892,7 @@ pub(super) fn rewrite_mutable_root_slots_with_sources( mut shadow_stats: Option<&mut ShadowRootTraceStats>, mut root_sources: Option<&mut RootSourcesTraceStats>, ) { - visit_mutable_root_slots(|slot| unsafe { + let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { let bits = slot.read(); record_mutable_slot_scan_source(slot, bits, valid_ptrs, &mut root_sources); if bits == 0 { @@ -908,6 +908,7 @@ pub(super) fn rewrite_mutable_root_slots_with_sources( } } }); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); } pub(super) fn rewrite_mutable_registered_roots(valid_ptrs: &ValidPointerSet) { @@ -948,6 +949,7 @@ pub(super) fn verify_mutable_root_slots(valid_ptrs: &ValidPointerSet) { if let Some(new_bits) = try_rewrite_value(bits, valid_ptrs) { let surface = match slot.kind { MutableRootSlotKind::ShadowStack => "shadow stack roots", + MutableRootSlotKind::NativeStack => "native stack-map roots", MutableRootSlotKind::GlobalRoot => "global roots", }; panic_stale_forwarded_reference(surface, slot.ptr as usize, bits, new_bits); diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 677ef4563b..4ee2318cf1 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -223,6 +223,31 @@ pub fn run_with_parse_cache( std::env::set_var("PERRY_NO_CACHE", "1"); } + // Native-stack GC root-pressure report. Like `--opt-report`, this is + // observational and must be enabled before rayon starts module codegen. + // Cache reuse is disabled because cached objects bypass the lowering that + // records each function. + let statepoint_report_format = args.statepoint_report.or_else(|| { + match std::env::var("PERRY_STATEPOINT_REPORT").as_deref() { + Ok("json") => Some(StatepointReportFormat::Json), + Ok("1") | Ok("text") => Some(StatepointReportFormat::Text), + _ => None, + } + }); + if let Some(fmt) = statepoint_report_format { + std::env::set_var( + "PERRY_STATEPOINT_REPORT", + match fmt { + StatepointReportFormat::Json => "json", + StatepointReportFormat::Text => "text", + }, + ); + std::env::set_var("PERRY_NO_CACHE", "1"); + // `perry dev` reuses the process; discard records from its previous + // build before starting this one. + let _ = perry_codegen::statepoint_report::take_records(); + } + // Canonicalize the input path first so its `.parent()` is an absolute directory. // Without this, a bare filename like `perry demo.ts` produced `Path::new("").parent()` // → fallback `"."`, and the walk-up loops below (package.json + perry.toml discovery) @@ -4704,6 +4729,15 @@ pub fn run_with_parse_cache( eprintln!("{rendered}"); } + if let Some(fmt) = statepoint_report_format { + let records = perry_codegen::statepoint_report::take_records(); + let rendered = match fmt { + StatepointReportFormat::Json => perry_codegen::statepoint_report::render_json(&records), + StatepointReportFormat::Text => perry_codegen::statepoint_report::render_text(&records), + }; + eprintln!("{rendered}"); + } + // #835 + #846: fold the codegen-side FFI provenance registry into // ctx so the well-known flip and `needs_stdlib` decisions below see // the symbols codegen actually emitted, not just the modules the diff --git a/crates/perry/src/commands/compile/types.rs b/crates/perry/src/commands/compile/types.rs index 94b3ced08e..a15d9fbbc2 100644 --- a/crates/perry/src/commands/compile/types.rs +++ b/crates/perry/src/commands/compile/types.rs @@ -494,6 +494,18 @@ pub struct CompileArgs { /// that codegen actually executes and has something to report. #[arg(long, value_enum, num_args = 0..=1, default_missing_value = "text")] pub opt_report: Option, + + /// Report native-stack GC root pressure for the stack-map/statepoint + /// research backends. Shows calls with live roots, audited calls that + /// cannot collect, statepoint relocation counts, plain stack-map + /// fallbacks, and the live-root-width distribution. + /// + /// Useful with `PERRY_STACK_MAPS=1` or `PERRY_STATEPOINTS=1`. + /// `--statepoint-report=json` emits a stable machine-readable schema. + /// Observational only; cache reuse is disabled for the reporting run so + /// codegen executes and produces records. + #[arg(long, value_enum, num_args = 0..=1, default_missing_value = "text")] + pub statepoint_report: Option, } /// Output format for `--opt-report`. @@ -505,6 +517,15 @@ pub enum OptReportFormat { Json, } +/// Output format for `--statepoint-report`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum StatepointReportFormat { + /// Human-readable root-pressure summary and ranked callees. + Text, + /// Stable JSON schema for tooling. + Json, +} + /// Information about a JavaScript module that will be interpreted at runtime #[derive(Debug, Clone)] pub struct JsModule { diff --git a/crates/perry/src/commands/dev.rs b/crates/perry/src/commands/dev.rs index 62a92d78ff..e2da75f66e 100644 --- a/crates/perry/src/commands/dev.rs +++ b/crates/perry/src/commands/dev.rs @@ -314,6 +314,7 @@ fn build_once( disable_buffer_fast_path: false, explain_lowering: false, opt_report: None, + statepoint_report: None, emit_attest: false, emit_sandbox: false, lockdown: false, diff --git a/crates/perry/src/commands/run/mod.rs b/crates/perry/src/commands/run/mod.rs index 6669976afc..dcfe5d7a53 100644 --- a/crates/perry/src/commands/run/mod.rs +++ b/crates/perry/src/commands/run/mod.rs @@ -226,6 +226,7 @@ pub fn run(args: RunArgs, format: OutputFormat, use_color: bool, verbose: u8) -> disable_buffer_fast_path: false, explain_lowering: false, opt_report: None, + statepoint_report: None, emit_attest: false, emit_sandbox: false, lockdown: false, diff --git a/docs/src/cli/flags.md b/docs/src/cli/flags.md index 537c7b72ca..9761f064ed 100644 --- a/docs/src/cli/flags.md +++ b/docs/src/cli/flags.md @@ -103,6 +103,7 @@ accept either the `$perryfs/` virtual path or the embed-relative key. | `--no-codegen` | Skip the `package.json` `perry.codegen` build-time steps (also `PERRY_SKIP_CODEGEN=1`). See [Project Configuration](../getting-started/project-config.md) | | `--keep-intermediates` | Keep `.o` and `.asm` intermediate files | | `--opt-report[=json]` | Report which values Perry could **not** statically type, why, and whether you can fix it. Text by default; `--opt-report=json` emits a stable schema for tooling. Also settable via `PERRY_OPT_REPORT=1` | +| `--statepoint-report[=json]` | Report native-stack GC root pressure: calls with live roots, audited non-collecting calls omitted, relocations, plain-map fallbacks, and live-root widths. Research-only; use with `PERRY_STACK_MAPS=1` or `PERRY_STATEPOINTS=1` | The `--trace`/`--focus` pair localizes "compiled to the wrong thing" bugs: `perry compile foo.ts --trace hir,llvm --focus parseRow` dumps just the @@ -228,6 +229,7 @@ shrink less, proportionally. | `CI=true` | Auto-skip update checks (set by most CI systems) | | `RUST_LOG` | Debug logging level (`debug`, `info`, `trace`) | | `PERRY_OPT_REPORT` | `1`/`text` or `json` — same as `--opt-report[=json]`, for driving the report from an environment where adding a flag is awkward | +| `PERRY_STATEPOINT_REPORT` | `1`/`text` or `json` — same as `--statepoint-report[=json]`; observational root-pressure reporting for the native-stack GC experiments | ## Configuration Files diff --git a/docs/stack-map-gc-experiment.md b/docs/stack-map-gc-experiment.md index 6b50f868fd..dfa031928d 100644 --- a/docs/stack-map-gc-experiment.md +++ b/docs/stack-map-gc-experiment.md @@ -203,7 +203,8 @@ Potentially: The current prototype is not yet a platform: - scanning is implemented only for Mach-O/macOS; -- every call is conservatively instrumented instead of only GC-capable calls; +- unknown calls remain conservatively instrumented; an audited call-effect + table now omits runtime helpers proven unable to enter Perry's collector; - the runtime assumes the matching stack-map PC is within 16 bytes of the unwound return PC; - active roots must remain in addressable allocas; diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index e40772437d..3bb7853690 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -30,6 +30,119 @@ Perry's GC simpler. Keep the shadow stack as the default. The prototype remains opt-in with `PERRY_STATEPOINTS=1`. The default shadow-stack path is unchanged. +## Follow-up: root pressure and audited safepoints + +The first prototype treated almost every textual call with live roots as a +safepoint. That was correct but needlessly pessimistic. The follow-up adds an +audited GC-call-effect table whose only claim is whether a helper can enter +Perry's collector. Unknown calls remain safepoints. + +This is intentionally separate from LLVM memory effects. Temporary-root +bookkeeping, write barriers, layout notes, feedback counters, and refcount +writes mutate memory, but they do not run a Perry collection and therefore do +not need stack-map metadata. + +`--statepoint-report[=json]` makes the resulting root pressure visible. It +reports per-function logical/bound root slots, calls with live roots, audited +non-collecting calls, statepoints, relocations, plain-map fallbacks, live-root +widths, and callee frequencies. It is observational and disables cache reuse +for the reporting run: + +```sh +PERRY_STATEPOINTS=1 perry compile app.ts --statepoint-report +``` + +On `benchmarks/app-patterns/kernels/batch.ts`, the audit changed: + +| Metric | Before | After | Change | +|---|---:|---:|---:| +| Statepoints | 442 | 219 | -50.5% | +| Relocations | 867 | 403 | -53.5% | +| `__llvm_stackmaps` | 54,968 B | 26,432 B | -51.9% | +| Plain-map fallbacks | 0 | 0 | unchanged | + +The report found 223 calls with live roots that cannot collect. The largest +groups were typed-feedback bookkeeping, class-field guards, temporary-root +push/get/truncate, layout notes, write barriers, and property-observation +records. + +Across the eight GC probes: + +| Probe | Statepoints before | Statepoints after | Relocations after | Calls skipped | +|---|---:|---:|---:|---:| +| Nursery churn | 152 | 65 | 91 | 88 | +| Survivor promotion | 165 | 79 | 132 | 88 | +| Cross-generation writes | 168 | 74 | 95 | 96 | +| Dead after deep stack | 119 | 55 | 59 | 65 | +| Closure capture | 146 | 69 | 81 | 78 | +| String retention | 92 | 45 | 46 | 50 | +| Array grow/evacuate | 100 | 49 | 49 | 52 | +| Map/set side tables | 138 | 66 | 124 | 74 | +| **Total** | **1,080** | **502** | **677** | **591** | + +The same audit applies to the plain-map backend. Its eight-probe metadata +payload is now 26,816 bytes; explicit statepoints use 53,952 bytes. Statepoint +metadata therefore remains 2.01x plain maps even after both shrink by roughly +half. Reducing the number of maybe-pointer roots through representation +promotion remains the larger shared lever. + +All eight probes pass in shadow, plain-map, and statepoint modes with forced +evacuation and relocation verification: 24/24 mode/probe comparisons match +Node's probe/checksum output. + +### Follow-up runtime and compile measurements + +Each runtime cell below is the median of seven executions on the same host and +full-feature runtime artifact. Host variance was high, so the numbers are +directional: + +| Probe | Shadow | Plain stack map | Statepoint | Statepoint vs plain | +|---|---:|---:|---:|---:| +| Nursery churn | 196.698 ms | 194.860 ms | 195.178 ms | +0.16% | +| Survivor promotion | 234.184 ms | 235.600 ms | 233.294 ms | -0.98% | +| Cross-generation writes | 383.002 ms | 330.386 ms | 341.206 ms | +3.27% | +| Dead after deep stack | 1,029.803 ms | 996.121 ms | 1,196.806 ms | +20.15% | +| Closure capture | 230.904 ms | 212.415 ms | 188.984 ms | -11.03% | +| String retention | 151.553 ms | 173.651 ms | 162.272 ms | -6.55% | +| Array grow/evacuate | 251.063 ms | 267.094 ms | 237.260 ms | -11.17% | +| Map/set side tables | 577.092 ms | 563.401 ms | 601.368 ms | +6.74% | + +Geometric means put plain maps at -1.17% versus shadow, statepoints at -1.54% +versus shadow, and statepoints at -0.38% versus plain maps. That small aggregate +statepoint lead is below the noise floor; the deep-stack regression remains a +clear negative signal. + +For uncached `batch.ts` compilation, seven-run medians were 910.3 ms shadow, +946.0 ms plain maps (+3.92%), and 958.3 ms statepoints (+5.28%). + +### Native-root and unwinder telemetry + +Native stack-map roots now have their own `root_sources.compiled_native` +telemetry bucket instead of being incorrectly charged to +`compiled_shadow`. `root_sources.native_stack_maps` also records walks, frames +visited, records matched, and locations visited. + +The forced-evacuation deep-stack probe reported 105 walks, 36,458 frames +visited, 36,139 records matched, and only 104 root locations visited, with a +maximum of 694 frames in one cycle. The walker is therefore a justified +optimization target, but a direct frame-pointer walker is not yet a safe +substitution: current generated AArch64 code saves `x29` without consistently +establishing an `x29` frame chain, and Rust/runtime frames have no matching +contract. A fast path first needs an explicit frame-pointer/unwind ABI for +generated and intervening runtime frames, plus fallback and cross-architecture +tests. + +### Work deliberately left gated + +This follow-up does not alter temporary-root semantics, collection scheduling, +or the conservative native-stack fallback. The representation plan makes the +temp-root correctness work a prerequisite for an explicit-only collection +contract and conservative-scanner removal. Doing that here would overlap the +other agent's work and make failures impossible to attribute. Once that +prerequisite lands, the next experiment is to assert that moving collections +occur only at declared safepoints and then measure whether the conservative +scanner can be deleted. + ## Which statepoint design this tests This is the explicit bridge, not LLVM's `RewriteStatepointsForGC` pipeline. From 416e6a6fa1fda95731f7aee1a0f2b8294fe93a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 21:48:25 +0200 Subject: [PATCH 03/53] research(gc): x29-chain fast walker for native stack-map roots The deep-stack telemetry showed 36,458 frames unwound to visit 104 root locations: _Unwind_Backtrace pays full compact-unwind register recovery on every native frame. Replace it with a raw x29-chain walk when the maps allow it: - codegen emits "frame-pointer"="non-leaf" on generated functions in native-root modes, so the [x29, x30] chain is guaranteed through generated frames (textual-IR input gets no frame-pointer default from the clang driver); - the parser now records each function's stack size; LLVM's AArch64 frame keeps the FP/LR pair at the top of the frame, so SP-relative statepoint spills resolve as fp + 16 - stack_size from the same two chain loads; - chain_walkable is decided once at parse: any location that is not FP-relative or sized-SP-relative disables the fast path for the image; - every anomaly (misaligned, non-increasing, or out-of-bounds frame pointer) abandons the walk and re-runs the platform unwinder; slot visits are idempotent so the fallback is safe; - PERRY_STACKMAP_WALKER=unwind forces the old walker (bisection control); PERRY_STACKMAP_WALKER=verify runs both and panics unless they visit the identical slot set - the liveness gate for the fast walker, since forced-evacuation verification enumerates roots through the same walker and cannot see a frame the walker skipped; - telemetry gains fp_walks/fallback_walks so a run can prove which walker actually executed. Finding recorded for the mode decision: plain-map mode emits Register R#1 locations (root slot address in a caller-saved register) that the parser must drop - those roots are invisible to the collector by construction, which statepoint spill slots cannot exhibit. --- crates/perry-codegen/src/function.rs | 13 +- .../perry-runtime/src/gc/roots/stack_maps.rs | 347 +++++++++++++++++- crates/perry-runtime/src/gc/telemetry.rs | 8 + .../src/gc/tests/telemetry_verifier.rs | 5 + 4 files changed, 355 insertions(+), 18 deletions(-) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index da635766ca..adcff3e8b0 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -635,6 +635,15 @@ impl LlFunction { } else { "" }; + // The native-stack walker recovers frames through the x29 chain, so + // every generated function must link one; without the attribute, + // textual-IR input gets no frame-pointer default from the clang + // driver and LLVM may omit the chain even while saving x29. + let frame_pointer = if crate::codegen::helpers::native_stack_roots_enabled() { + " \"frame-pointer\"=\"non-leaf\"" + } else { + "" + }; let gc_strategy = if self.stack_map_requested && crate::codegen::helpers::statepoints_enabled() && !self.has_try @@ -644,8 +653,8 @@ impl LlFunction { "" }; let mut ir = format!( - "define {}{} @{}({}){}{} {{\n", - linkage, self.return_type, self.name, param_str, attrs, gc_strategy + "define {}{} @{}({}){}{}{} {{\n", + linkage, self.return_type, self.name, param_str, attrs, frame_pointer, gc_strategy ); for (i, blk) in self.blocks.iter().enumerate() { diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 5c4c0b173a..c42f6c7ce4 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -31,10 +31,55 @@ struct StackMapLocation { #[derive(Clone, Debug, Eq, PartialEq)] struct StackMapRecord { pc: usize, + /// The containing function's total frame size from the stack-map header. + /// LLVM's AArch64 frame places the `[x29, x30]` pair at the top of the + /// frame, so a chain walker can reconstruct the body SP as + /// `fp + 16 - stack_size` for SP-relative locations. + stack_size: u64, locations: Vec, } -static STACK_MAPS: OnceLock> = OnceLock::new(); +/// Parsed section plus the facts the fast walker's preconditions need. +/// +/// `chain_walkable` is decided once at parse time: the raw x29-chain walk can +/// recover only the frame pointer (register 29) directly, plus the body SP +/// (register 31) derived from the header's per-function stack size. Any other +/// register anywhere in the maps disables the fast path for the whole image +/// rather than risking a wrong base mid-walk. +#[derive(Debug, Default)] +struct StackMapIndex { + records: Vec, + chain_walkable: bool, + min_pc: usize, + max_pc: usize, +} + +static STACK_MAPS: OnceLock = OnceLock::new(); + +const DWARF_REG_FP_AARCH64: u16 = 29; +const DWARF_REG_SP_AARCH64: u16 = 31; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WalkerMode { + /// x29-chain walk when `chain_walkable`, transparent unwinder fallback otherwise. + Fast, + /// Force the platform unwinder (bisection control). + Unwind, + /// Run both walks and panic unless they visit the identical slot set. + /// This is the only check that can catch a fast walk that silently skips + /// frames: forced-evacuation verification enumerates roots through the + /// same walker, so it cannot see a slot the walker never reached. + Verify, +} + +fn walker_mode() -> WalkerMode { + static MODE: OnceLock = OnceLock::new(); + *MODE.get_or_init(|| match std::env::var("PERRY_STACKMAP_WALKER").as_deref() { + Ok("unwind") => WalkerMode::Unwind, + Ok("verify") => WalkerMode::Verify, + _ => WalkerMode::Fast, + }) +} #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(in crate::gc) struct NativeStackWalkStats { @@ -42,6 +87,8 @@ pub(in crate::gc) struct NativeStackWalkStats { pub(in crate::gc) frames_visited: usize, pub(in crate::gc) records_matched: usize, pub(in crate::gc) locations_visited: usize, + pub(in crate::gc) fp_walks: usize, + pub(in crate::gc) fallback_walks: usize, } #[inline] @@ -55,6 +102,8 @@ pub(in crate::gc) fn record_native_stack_walk_source( stats.frames_visited, stats.records_matched, stats.locations_visited, + stats.fp_walks, + stats.fallback_walks, ); } } @@ -63,17 +112,32 @@ pub(in crate::gc) fn initialize() { let _ = stack_maps(); } -fn stack_maps() -> &'static [StackMapRecord] { - STACK_MAPS - .get_or_init(|| { - let Some(section) = loaded_stack_map_section() else { - return Vec::new(); - }; - let mut records = parse_concatenated_stack_maps(section).unwrap_or_default(); - records.sort_unstable_by_key(|record| record.pc); - records +fn stack_maps() -> &'static StackMapIndex { + STACK_MAPS.get_or_init(|| { + let Some(section) = loaded_stack_map_section() else { + return StackMapIndex::default(); + }; + let mut records = parse_concatenated_stack_maps(section).unwrap_or_default(); + records.sort_unstable_by_key(|record| record.pc); + index_records(records) + }) +} + +fn index_records(records: Vec) -> StackMapIndex { + let chain_walkable = records.iter().all(|record| { + record.locations.iter().all(|location| { + location.dwarf_reg == DWARF_REG_FP_AARCH64 + || (location.dwarf_reg == DWARF_REG_SP_AARCH64 && record.stack_size >= 16) }) - .as_slice() + }); + let min_pc = records.first().map_or(usize::MAX, |record| record.pc); + let max_pc = records.last().map_or(0, |record| record.pc); + StackMapIndex { + records, + chain_walkable, + min_pc, + max_pc, + } } fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { @@ -98,11 +162,63 @@ fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { pub(super) fn visit_stack_map_root_slots( visit: &mut impl FnMut(MutableRootSlot), ) -> NativeStackWalkStats { - let maps = stack_maps(); - if maps.is_empty() { + let index = stack_maps(); + if index.records.is_empty() { return NativeStackWalkStats::default(); } - unwind::visit(maps, visit) + match walker_mode() { + WalkerMode::Unwind => unwind::visit(&index.records, visit), + WalkerMode::Fast => { + if index.chain_walkable { + if let Some(stats) = fp_chain::visit(index, visit) { + return stats; + } + } + let mut stats = unwind::visit(&index.records, visit); + stats.fallback_walks = 1; + stats + } + WalkerMode::Verify => verify_visit(index, visit), + } +} + +/// Debug-only cross-check: the fast walk reads slot addresses without +/// mutating, then the unwinder performs the real visitation while recording +/// what it reached. Any set difference is a missed or invented frame and +/// panics immediately — this is the liveness gate for the fast walker itself. +fn verify_visit( + index: &StackMapIndex, + visit: &mut impl FnMut(MutableRootSlot), +) -> NativeStackWalkStats { + let mut fast_addresses: Vec = Vec::new(); + let fast_stats = fp_chain::visit(index, &mut |slot: MutableRootSlot| { + fast_addresses.push(slot.ptr as usize); + }); + let Some(fast_stats) = fast_stats else { + panic!( + "PERRY_STACKMAP_WALKER=verify: fast walk unavailable \ + (chain_walkable={}, anomaly or unsupported target)", + index.chain_walkable + ); + }; + let mut unwind_addresses: Vec = Vec::new(); + let mut stats = unwind::visit(&index.records, &mut |slot: MutableRootSlot| { + unwind_addresses.push(slot.ptr as usize); + visit(slot); + }); + fast_addresses.sort_unstable(); + fast_addresses.dedup(); + unwind_addresses.sort_unstable(); + unwind_addresses.dedup(); + assert_eq!( + fast_addresses, unwind_addresses, + "PERRY_STACKMAP_WALKER=verify: fast walk visited {} unique slots, \ + unwinder visited {}", + fast_addresses.len(), + unwind_addresses.len() + ); + stats.fp_walks = fast_stats.fp_walks; + stats } fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { @@ -137,8 +253,9 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { let mut expected_records = 0usize; for _ in 0..function_count { let address = read_u64(bytes, offset)? as usize; + let stack_size = read_u64(bytes, offset + 8)?; let records = read_u64(bytes, offset + 16)? as usize; - functions.push((address, records)); + functions.push((address, stack_size, records)); expected_records = expected_records.checked_add(records)?; offset = offset.checked_add(24)?; } @@ -151,7 +268,7 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { } let mut out = Vec::with_capacity(record_count); - for (function_address, function_record_count) in functions { + for (function_address, function_stack_size, function_record_count) in functions { for _ in 0..function_record_count { let instruction_offset = read_u32(bytes, offset + 8)? as usize; let location_count = read_u16(bytes, offset + 14)? as usize; @@ -195,6 +312,7 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { out.push(StackMapRecord { pc: function_address.checked_add(instruction_offset)?, + stack_size: function_stack_size, locations, }); } @@ -451,6 +569,149 @@ mod unwind { } } +/// Raw x29-chain walker. +/// +/// AArch64 prologues under `"frame-pointer"="non-leaf"` are +/// `stp x29, x30, [sp, #-16]!; mov x29, sp`, so every frame's x29 points at +/// a `[caller x29, return address]` pair. One hop is therefore two loads, +/// against a full unwind step (compact-unwind lookup plus register +/// recovery) — this is what turns the measured 350:1 frames-to-roots ratio +/// from a tax into noise. +/// +/// Fail-closed everywhere: a misaligned, non-increasing, or out-of-bounds +/// frame pointer abandons the walk with `None` and the caller re-runs the +/// whole scan through the platform unwinder. Slot visitation is idempotent +/// (a rewritten slot no longer points at a forwarded object), so a partial +/// fast walk followed by a full unwinder walk is safe. +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +mod fp_chain { + use super::*; + + fn current_frame_pointer() -> usize { + let fp: usize; + unsafe { + core::arch::asm!("mov {fp}, x29", fp = out(reg) fp, options(nomem, nostack)); + } + fp + } + + fn stack_top() -> usize { + unsafe extern "C" { + fn pthread_self() -> usize; + fn pthread_get_stackaddr_np(thread: usize) -> *mut c_void; + } + unsafe { pthread_get_stackaddr_np(pthread_self()) as usize } + } + + pub(super) fn visit( + index: &StackMapIndex, + visit: &mut F, + ) -> Option { + if !index.chain_walkable { + return None; + } + let top = stack_top(); + if top == 0 { + return None; + } + let mut stats = NativeStackWalkStats { + walks: 1, + fp_walks: 1, + ..NativeStackWalkStats::default() + }; + let low_pc = index.min_pc.saturating_sub(MAX_SAFEPOINT_RETURN_DELTA); + let high_pc = index.max_pc.saturating_add(MAX_SAFEPOINT_RETURN_DELTA); + let mut fp = current_frame_pointer(); + while fp != 0 { + if fp & 0xF != 0 || fp.checked_add(16)? > top { + return None; + } + let return_address = unsafe { *((fp + 8) as *const usize) }; + let caller_fp = unsafe { *(fp as *const usize) }; + stats.frames_visited = stats.frames_visited.saturating_add(1); + if return_address == 0 { + break; + } + if return_address >= low_pc && return_address <= high_pc { + if let Some(candidate_pc) = closest_record_pc(&index.records, return_address) { + if return_address.abs_diff(candidate_pc) <= MAX_SAFEPOINT_RETURN_DELTA { + // The record describes the caller's frame; its + // locations are relative to the caller's own x29, + // which is exactly the saved word we just read. + if caller_fp == 0 { + return None; + } + let first = index + .records + .partition_point(|record| record.pc < candidate_pc); + let last = index + .records + .partition_point(|record| record.pc <= candidate_pc); + stats.records_matched = stats + .records_matched + .saturating_add(last.saturating_sub(first)); + for record in &index.records[first..last] { + // LLVM's AArch64 frame keeps the [x29, x30] pair + // at the top of the frame, so the caller's body + // SP is its fp + 16 - stack_size. `chain_walkable` + // guaranteed stack_size >= 16 for SP records. + let sp = caller_fp + .checked_add(16) + .and_then(|top| top.checked_sub(record.stack_size as usize)); + for location in &record.locations { + stats.locations_visited = + stats.locations_visited.saturating_add(1); + let base = if location.dwarf_reg == DWARF_REG_FP_AARCH64 { + Some(caller_fp) + } else { + sp + }; + let Some(base) = base else { + return None; + }; + let address = if location.offset < 0 { + base.checked_sub(location.offset.unsigned_abs() as usize) + } else { + base.checked_add(location.offset as usize) + }; + let Some(address) = address else { + continue; + }; + if address == 0 + || address & (std::mem::align_of::() - 1) != 0 + { + continue; + } + visit(MutableRootSlot { + kind: MutableRootSlotKind::NativeStack, + ptr: address as *mut u64, + }); + } + } + } + } + } + if caller_fp != 0 && caller_fp <= fp { + return None; + } + fp = caller_fp; + } + Some(stats) + } +} + +#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] +mod fp_chain { + use super::*; + + pub(super) fn visit( + _index: &StackMapIndex, + _visit: &mut impl FnMut(MutableRootSlot), + ) -> Option { + None + } +} + #[cfg(test)] mod tests { use super::*; @@ -505,6 +766,7 @@ mod tests { records, vec![StackMapRecord { pc: 0x1010, + stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, offset: -8, @@ -537,6 +799,7 @@ mod tests { records, vec![StackMapRecord { pc: 0x1020, + stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, offset: -16, @@ -556,15 +819,67 @@ mod tests { assert!(parse_one_stack_map(&bytes).is_none()); } + #[test] + fn chain_walkable_index_accepts_fp_and_sized_sp_locations_only() { + let fp_record = StackMapRecord { + pc: 0x1000, + stack_size: 0, + locations: vec![StackMapLocation { + dwarf_reg: DWARF_REG_FP_AARCH64, + offset: -8, + }], + }; + let sp_record = StackMapRecord { + pc: 0x2000, + stack_size: 160, + locations: vec![StackMapLocation { + dwarf_reg: DWARF_REG_SP_AARCH64, + offset: 16, + }], + }; + let frameless_sp_record = StackMapRecord { + pc: 0x3000, + stack_size: 0, + locations: vec![StackMapLocation { + dwarf_reg: DWARF_REG_SP_AARCH64, + offset: 8, + }], + }; + let other_reg_record = StackMapRecord { + pc: 0x4000, + stack_size: 160, + locations: vec![StackMapLocation { + dwarf_reg: 1, + offset: 0, + }], + }; + + let walkable = index_records(vec![fp_record.clone(), sp_record.clone()]); + assert!(walkable.chain_walkable); + assert_eq!(walkable.min_pc, 0x1000); + assert_eq!(walkable.max_pc, 0x2000); + + assert!( + !index_records(vec![fp_record.clone(), frameless_sp_record]).chain_walkable, + "an SP location without a usable frame size must disable the fast walk" + ); + assert!( + !index_records(vec![fp_record, other_reg_record]).chain_walkable, + "any non-FP/SP register must disable the fast walk" + ); + } + #[test] fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { let maps = vec![ StackMapRecord { pc: 0x1000, + stack_size: 32, locations: Vec::new(), }, StackMapRecord { pc: 0x1020, + stack_size: 32, locations: Vec::new(), }, ]; diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index f557d20a55..d3af9f1d88 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -306,6 +306,8 @@ pub(super) struct NativeStackMapTraceStats { pub(super) frames_visited: usize, pub(super) records_matched: usize, pub(super) locations_visited: usize, + pub(super) fp_walks: usize, + pub(super) fallback_walks: usize, } impl NativeStackMapTraceStats { @@ -316,11 +318,15 @@ impl NativeStackMapTraceStats { frames_visited: usize, records_matched: usize, locations_visited: usize, + fp_walks: usize, + fallback_walks: usize, ) { self.walks = self.walks.saturating_add(walks); self.frames_visited = self.frames_visited.saturating_add(frames_visited); self.records_matched = self.records_matched.saturating_add(records_matched); self.locations_visited = self.locations_visited.saturating_add(locations_visited); + self.fp_walks = self.fp_walks.saturating_add(fp_walks); + self.fallback_walks = self.fallback_walks.saturating_add(fallback_walks); } } @@ -1325,6 +1331,8 @@ pub(super) fn root_sources_json(stats: RootSourcesTraceStats) -> serde_json::Val "frames_visited": stats.native_stack_maps.frames_visited, "records_matched": stats.native_stack_maps.records_matched, "locations_visited": stats.native_stack_maps.locations_visited, + "fp_walks": stats.native_stack_maps.fp_walks, + "fallback_walks": stats.native_stack_maps.fallback_walks, }, "native_stack_fallback": { "decision": stats.native_stack_fallback.decision.as_str(), diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index 2f8783e369..4f2f0aa79e 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -251,6 +251,11 @@ fn root_heavy_workload_reports_root_sources_and_budgeted_progression() { event["root_sources"]["native_stack_maps"]["frames_visited"].is_number(), "native stack-map telemetry should expose unwinder work" ); + assert!( + event["root_sources"]["native_stack_maps"]["fp_walks"].is_number() + && event["root_sources"]["native_stack_maps"]["fallback_walks"].is_number(), + "native stack-map telemetry should expose which walker ran" + ); let live_after = (js_shadow_slot_get(0) & POINTER_MASK) as *const crate::StringHeader; unsafe { From 2e50534a52aa0d9d25555568ad2be0cb8f00291b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 21:55:32 +0200 Subject: [PATCH 04/53] docs: record x29-chain walker results and the plain-map Register-location finding --- docs/statepoint-gc-experiment.md | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 3bb7853690..4baa4424cd 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -115,6 +115,51 @@ clear negative signal. For uncached `batch.ts` compilation, seven-run medians were 910.3 ms shadow, 946.0 ms plain maps (+3.92%), and 958.3 ms statepoints (+5.28%). +### Follow-up: x29-chain fast walker (2026-07-31, after rebase onto #7114) + +The deep-stack telemetry below (36,458 frames unwound for 104 root +locations) identified `_Unwind_Backtrace` as the walker bottleneck: full +compact-unwind register recovery on every native frame. The branch now walks +the raw x29 chain instead — two loads per frame — enabled by two facts: + +1. Generated functions now carry `"frame-pointer"="non-leaf"`. Textual-IR + input gets no frame-pointer default from the clang driver, which is why + generated code previously saved x29 without establishing a chain. +2. Statepoint spills are SP-relative (`Indirect [R#31 + N]`) on AArch64 + regardless of frame-pointer attributes, but the stack-map header records + each function's frame size, and LLVM's AArch64 frame keeps the + `[x29, x30]` pair at the top of the frame — so the body SP is always + `fp + 16 - stack_size` from the same chain loads. + +The fast path is fail-closed twice over. At parse time, any location that is +not FP-relative or sized-SP-relative marks the whole image not chain-walkable. +At walk time, a misaligned, non-increasing, or out-of-bounds frame pointer +abandons the walk and re-runs the platform unwinder (slot visits are +idempotent, so a partial fast walk followed by a full unwinder walk is safe). +`PERRY_STACKMAP_WALKER=unwind` forces the old walker as a bisection control; +`PERRY_STACKMAP_WALKER=verify` runs both and panics unless they visit the +identical slot set. Verify exists because forced-evacuation verification +enumerates roots through the same walker and therefore cannot catch a walker +that silently skips frames — this is CLAUDE.md gate-failure mode 4 applied to +the walker itself. Telemetry gains `fp_walks`/`fallback_walks` so any run can +prove which walker executed. + +Results (loaded host, directional): 24/24 correctness matrix, 16/16 +verify-mode probe runs (fast walk engaged and byte-identical to the unwinder +everywhere), and the deep-stack statepoint probe improves 4.8% end-to-end +against the unwinder walker on the same binary with interleaved reps. + +A finding for the mode decision fell out of the register census: plain-map +mode emits `Register R#1` locations — the root slot's address materialized in +a caller-saved register at the map point. No parser can soundly use that +location (the register is clobbered by the callee and unrecoverable at GC +time), so those roots are structurally invisible to the collector. LLVM's +stackmap intrinsic offers no way to force the address into memory; statepoint +spill slots cannot exhibit the problem. Plain maps are therefore unsound by +construction at a small but nonzero rate (3 of 60 locations on the deep-stack +probe), which strengthens the case for deleting the plain-map arm once +statepoints match it on the walker-sensitive workloads. + ### Native-root and unwinder telemetry Native stack-map roots now have their own `root_sources.compiled_native` From 90169acd2166f005214a22e8f89f73795af11337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 22:06:57 +0200 Subject: [PATCH 05/53] research(gc): explicit-safepoint collection contract (PERRY_GC_SAFEPOINT_ONLY) The contract: a collection that skips the conservative stack scan consumes only precise roots, and with native stack maps active those exist only at mapped PCs - so such a collection may only begin at a declared safepoint (loop back-edge poll, outermost microtask-pump boundary); anywhere else it must scan conservatively. Today that property is emergent - every possibly- collecting call happens to be mapped. The contract makes it enforced, which is what allows call sites to become unmapped. Runtime: - GC_AT_DECLARED_SAFEPOINT thread-local + RAII guard, set by the moving- minor safepoint drain (covers both the loop poll and the microtask boundary) and by the contract poll extension. - Enforcement at the root-scan subphase: an undeclared precise-root cycle either has the conservative scan forced for that cycle (heal mode, =1 - sound: the scan restores liveness and a conservatively-scanned cycle is non-moving) or panics (=strict, the gate mode that proves enforcement is live). The alloc-point valve and manual gc() force the scan already and are exempt by construction. - Under the contract, loop polls also drain non-nursery triggers via gc_check_trigger so full collections migrate to declared safepoints. Codegen: - New audited GcCallEffect::AllocNoReentry class: helpers that may allocate (and so arm a trigger) but never collect synchronously and never re-enter generated JS. Under the contract their call sites need no statepoint; without it they remain safepoints. First audited set: closure/object allocation, js_array_push_f64/length/slice_values. - PERRY_GC_SAFEPOINT_ONLY participates in build and object cache keys. Census note (batch.ts): the bulk of remaining statepoints are property- access diamonds that can re-enter via getters and must stay mapped; the contract's reach is bounded by re-entry, and deleting those calls is representation selection's job (Ptr), not the contract's. The two compose: repsel removes the calls, the contract unmaps what allocation traffic remains. --- crates/perry-codegen/src/codegen/helpers.rs | 17 +++++ crates/perry-codegen/src/function.rs | 15 +++- crates/perry-codegen/src/gc_call_effects.rs | 49 +++++++++++++ crates/perry-runtime/src/gc/cycle.rs | 30 +++++++- crates/perry-runtime/src/gc/policy.rs | 73 ++++++++++++++++++- crates/perry-runtime/src/gc/roots.rs | 1 + .../perry-runtime/src/gc/roots/stack_maps.rs | 7 ++ .../perry/src/commands/compile/build_cache.rs | 1 + .../src/commands/compile/object_cache.rs | 7 ++ .../object_cache/object_cache_tests.rs | 1 + 10 files changed, 197 insertions(+), 4 deletions(-) diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index a9ae533752..535773d0fb 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -115,6 +115,23 @@ pub(crate) fn native_stack_roots_enabled() -> bool { stack_maps_enabled() || statepoints_enabled() } +/// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract +/// (research, `exp/stackmap-viability`). The runtime enforces that a +/// precise-root collection only begins at a declared safepoint; under that +/// guarantee, audited allocate-but-never-reenter helpers +/// (`GcCallEffect::AllocNoReentry`) need no statepoint. Participates in both +/// build and object cache keys. +pub(crate) fn gc_safepoint_only_contract_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref(), + Ok("1") | Ok("on") | Ok("true") | Ok("strict") + ) + }) +} + /// Inline shadow-slot store gate (#7088). Default ON. /// /// When enabled, a store to a GC-rooted local is emitted as an address diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index adcff3e8b0..ab48d927b7 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -1314,8 +1314,19 @@ fn lower_precise_roots_to_native_stack( let is_compiler_only = direct_callee.is_some_and(|callee| callee.starts_with("llvm.")) || trimmed.contains("call void asm "); let cannot_collect = direct_callee.is_some_and(|callee| { - crate::gc_call_effects::classify_direct_callee(callee) - == crate::gc_call_effects::GcCallEffect::CannotCollect + match crate::gc_call_effects::classify_direct_callee(callee) { + crate::gc_call_effects::GcCallEffect::CannotCollect => true, + // Under the explicit-safepoint contract the runtime + // guarantees these helpers' triggers never consume this + // frame's precise roots (they defer to a declared safepoint + // or collect behind a forced conservative scan), so the + // call site needs no metadata. Without the contract they + // stay safepoints. + crate::gc_call_effects::GcCallEffect::AllocNoReentry => { + crate::codegen::helpers::gc_safepoint_only_contract_enabled() + } + crate::gc_call_effects::GcCallEffect::Unknown => false, + } }); if is_compiler_only || cannot_collect { // LLVM intrinsics, zero-instruction compiler barriers, and diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 3f960199fd..8313ed408a 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -12,6 +12,17 @@ #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum GcCallEffect { CannotCollect, + /// May allocate — and therefore arm a GC trigger — but never runs a + /// collection synchronously inside the call and never re-enters generated + /// JS (no getters, setters, valueOf/toString coercion, or callbacks). + /// + /// Only meaningful under `PERRY_GC_SAFEPOINT_ONLY`: the runtime contract + /// guarantees any trigger these helpers arm either defers to a declared + /// safepoint (moving) or collects behind a forced conservative scan (the + /// alloc-point valve), so the caller's precise frame roots are never + /// consumed at this call site and it needs no statepoint. Without the + /// contract these remain safepoints. + AllocNoReentry, Unknown, } @@ -70,6 +81,17 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_implicit_this_set" | "js_new_target_get" | "js_new_target_set" => GcCallEffect::CannotCollect, + // Audited allocate-but-never-reenter helpers (2026-07-31): each body + // was checked for closure invocation, coercion (valueOf/toString), + // and accessor dispatch — none present, and none takes a receiver + // that could route through user code (`js_array_length` takes a + // typed `*const ArrayHeader`, not a JSValue). The forced-evacuation + // probe gates backstop the audit. + "js_closure_alloc_singleton" + | "js_object_alloc_class_inline_keys" + | "js_array_push_f64" + | "js_array_length" + | "js_array_slice_values" => GcCallEffect::AllocNoReentry, _ => GcCallEffect::Unknown, } } @@ -110,4 +132,31 @@ mod tests { ); } } + + #[test] + fn audited_alloc_helpers_are_contract_only_non_safepoints() { + for name in ["js_closure_alloc_singleton", "js_array_push_f64"] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::AllocNoReentry, + "{name}" + ); + } + // Re-entering helpers must never be in the AllocNoReentry class: + // a poll can fire inside the callback/getter with this frame + // mid-stack, and the caller's roots must be findable. + for name in [ + "js_array_map", + "js_array_sort_with_comparator", + "js_number_coerce", + "js_dynamic_string_or_number_add", + "js_object_get_field_by_name_f64", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::Unknown, + "{name}" + ); + } + } } diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index e2bfc8c762..1b643a1151 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -571,7 +571,35 @@ impl RootScanCycleState { self.subphase = RootScanSubphase::MutableSlots; return false; } - let conservative_scan_decision = conservative_stack_scan_decision(); + let mut conservative_scan_decision = conservative_stack_scan_decision(); + // PERRY_GC_SAFEPOINT_ONLY contract: a collection that skips + // the conservative scan consumes only precise roots, and with + // native stack maps active those exist only at mapped PCs — + // so it may begin only at a declared safepoint (loop poll, + // outermost microtask boundary). Heal mode forces the scan + // for the offending cycle (sound and non-moving); strict + // mode panics so gates can prove the enforcement is live. + // The alloc-point valve and manual gc() already force the + // scan and are exempt by construction. + if !matches!( + conservative_scan_decision, + ConservativeStackScanDecision::Scan + ) && super::roots::native_stack_maps_active() + && !super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) + { + match super::policy::gc_safepoint_only_contract() { + super::policy::SafepointOnlyContract::Off => {} + super::policy::SafepointOnlyContract::Heal => { + conservative_scan_decision = ConservativeStackScanDecision::Scan; + } + super::policy::SafepointOnlyContract::Strict => { + panic!( + "PERRY_GC_SAFEPOINT_ONLY: precise-root collection \ + began outside a declared safepoint" + ); + } + } + } // #5029: minors retain old-gen conservative discoveries // pin-only (no trace) — see try_mark_conservative_word. let conservative_root_stats = mark_stack_roots_for_decision( diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index f603374221..ec905904c0 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -681,6 +681,65 @@ thread_local! { /// last set — the baseline the deferral slack is measured from (#7024). /// Meaningless while `GC_SAFEPOINT_PENDING` is false. pub(super) static GC_SAFEPOINT_DEFER_ARENA_BASE: Cell = const { Cell::new(0) }; + /// True while a DECLARED safepoint drain is running: a loop back-edge + /// poll, the outermost microtask-pump moving minor, or an explicit + /// `gc()`. Consumed by the `PERRY_GC_SAFEPOINT_ONLY` contract assert in + /// the root-scan subphase. + pub(super) static GC_AT_DECLARED_SAFEPOINT: Cell = const { Cell::new(false) }; +} + +/// `PERRY_GC_SAFEPOINT_ONLY` — research contract for the native-root modes +/// (`exp/stackmap-viability`): a collection that skips the conservative stack +/// scan consumes only precise roots, and with native stack maps active those +/// roots exist only at mapped PCs — so such a collection may begin only at a +/// declared safepoint; anywhere else it must scan conservatively. Codegen +/// reads the same env to stop emitting statepoints around audited +/// allocate-but-never-reenter helpers; the enforcement in `cycle.rs` is what +/// turns the property from emergent (every possibly-collecting call happens +/// to be mapped) into enforced. +/// +/// `1`/`on`/`true` — HEAL: an undeclared precise-root cycle has the +/// conservative scan forced for that cycle (sound: the scan restores +/// liveness, and a conservatively-scanned cycle is non-moving). This is the +/// measuring mode: alloc-point full collections are legitimate today and +/// simply pay the scan. +/// `strict` — PANIC on any undeclared precise-root cycle. This is the gate +/// mode that proves the enforcement is live. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SafepointOnlyContract { + Off, + Heal, + Strict, +} + +pub(super) fn gc_safepoint_only_contract() -> SafepointOnlyContract { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| match std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref() { + Ok("1") | Ok("on") | Ok("true") => SafepointOnlyContract::Heal, + Ok("strict") => SafepointOnlyContract::Strict, + _ => SafepointOnlyContract::Off, + }) +} + +/// RAII marker for a declared-safepoint drain. Nesting-safe: restores the +/// previous value so a poll firing inside a manual `gc()` cannot clear it. +pub(super) struct DeclaredSafepointGuard { + prev: bool, +} + +impl DeclaredSafepointGuard { + pub(super) fn enter() -> Self { + let prev = GC_AT_DECLARED_SAFEPOINT.with(|flag| flag.replace(true)); + Self { prev } + } +} + +impl Drop for DeclaredSafepointGuard { + fn drop(&mut self) { + let prev = self.prev; + GC_AT_DECLARED_SAFEPOINT.with(|flag| flag.set(prev)); + } } /// Committed arena bytes a deferred nursery trigger may allocate **past the @@ -1697,6 +1756,7 @@ pub(crate) fn gc_safepoint_moving_minor() { // We are handling this safepoint (collect or find nothing due): clear the // deferral flag set by the alloc-point arm (Phase 2/3). GC_SAFEPOINT_PENDING.with(|p| p.set(false)); + let _declared = DeclaredSafepointGuard::enter(); let kind = match gc_budgeted_due_trigger() { Some(BudgetedGcTrigger::ArenaBytes) => GcTriggerKind::ArenaBytes, Some(BudgetedGcTrigger::MallocCount) => GcTriggerKind::MallocCount, @@ -1775,7 +1835,18 @@ pub extern "C" fn js_gc_loop_safepoint() { if !GC_SAFEPOINT_PENDING.with(Cell::get) && !super::gc_zeal_enabled() { return; } - gc_safepoint_moving_minor(); + if GC_SAFEPOINT_PENDING.with(Cell::get) { + gc_safepoint_moving_minor(); + return; + } + // Under the safepoint-only contract, non-nursery (full/old-gen) triggers + // should preferentially drain at declared safepoints, where precise roots + // make the conservative heal unnecessary. The dueness check inside + // `gc_check_trigger` is cheap and this branch is research-mode only. + if gc_safepoint_only_contract() != SafepointOnlyContract::Off { + let _declared = DeclaredSafepointGuard::enter(); + gc_check_trigger(); + } } struct BudgetedGcStepGuard; diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index 19b6128d55..2086639bb2 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -8,6 +8,7 @@ mod shadow_stack; mod stack_maps; mod temp_roots; pub(super) use stack_maps::initialize as initialize_stack_maps; +pub(super) use stack_maps::native_maps_active as native_stack_maps_active; pub(super) use stack_maps::record_native_stack_walk_source; pub(super) use runtime_handles::{ diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index c42f6c7ce4..4b1978fb1d 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -112,6 +112,13 @@ pub(in crate::gc) fn initialize() { let _ = stack_maps(); } +/// Whether this image carries any native stack-map records — i.e. whether +/// precise frame roots depend on mapped PCs at all. Consumed by the +/// `PERRY_GC_SAFEPOINT_ONLY` contract assert. +pub(in crate::gc) fn native_maps_active() -> bool { + !stack_maps().records.is_empty() +} + fn stack_maps() -> &'static StackMapIndex { STACK_MAPS.get_or_init(|| { let Some(section) = loaded_stack_map_section() else { diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 7c6b8f054c..032804347f 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -37,6 +37,7 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_SHADOW_STACK", "PERRY_STACK_MAPS", "PERRY_STATEPOINTS", + "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 77facaaafa..b0f658d1e2 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -807,6 +807,13 @@ fn compute_object_cache_key_with_env( "env_statepoints", env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), ); + // Explicit-safepoint contract: flips audited AllocNoReentry helpers + // between statepoint and plain call. Two arms sharing a cached object + // would make the contract's metadata reduction unmeasurable. + h.field( + "env_gc_safepoint_only", + env_var("PERRY_GC_SAFEPOINT_ONLY").as_deref().unwrap_or(""), + ); // #7088: flips the shadow-slot store between an inline sequence and the // `js_shadow_slot_*` calls. Two arms that shared a cached object would // silently measure the same code. diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index 672bcee4a0..6e22f84eb4 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -585,6 +585,7 @@ fn key_changes_with_codegen_env_vars() { "PERRY_SHADOW_STACK", "PERRY_STACK_MAPS", "PERRY_STATEPOINTS", + "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", "PERRY_UNBOXED_OBJECT_FIELDS", From 72f5980c622653f5ca48c95c47e709430242f70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 23:26:48 +0200 Subject: [PATCH 06/53] docs: explicit-safepoint contract design, enforcement levels, and census bound --- docs/statepoint-gc-experiment.md | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 4baa4424cd..c8939d88a4 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -177,6 +177,51 @@ contract. A fast path first needs an explicit frame-pointer/unwind ABI for generated and intervening runtime frames, plus fallback and cross-architecture tests. +### Follow-up: the explicit-safepoint collection contract (PERRY_GC_SAFEPOINT_ONLY) + +The prerequisite that gated this experiment — the #7114 temp-root +correctness fix — landed on main during the first prototype session, so the +contract experiment ran after rebasing onto it. + +**The contract.** A collection that skips the conservative stack scan +consumes only precise roots; with native stack maps active, precise frame +roots exist only at mapped PCs. Therefore such a collection may only begin +at a declared safepoint (a loop back-edge poll or the outermost +microtask-pump boundary) — anywhere else it must scan conservatively. The +runtime already routes moving minors to those safepoints (the #7024 +deferral machinery), so today the property is *emergent*: it holds because +every possibly-collecting call happens to be mapped. The contract makes it +*enforced* — a thread-local declared-safepoint flag plus a check at the +root-scan subphase — and enforcement is what makes it sound to stop mapping +call sites. + +Two enforcement levels: `PERRY_GC_SAFEPOINT_ONLY=1` (heal — an undeclared +precise-root cycle gets the conservative scan forced for that cycle, which +restores liveness and keeps it non-moving) and `=strict` (panic — the gate +mode that proves the enforcement is live, per the four-ways-a-gate-cannot- +fail rule). Manual `gc()` and the alloc-point slack valve force the scan +already and are exempt by construction. Under the contract, loop polls also +drain non-nursery triggers so full collections migrate to declared +safepoints. + +**What it unmaps.** A new audited `GcCallEffect::AllocNoReentry` class: +helpers that may allocate (arming a trigger) but never collect synchronously +and never re-enter generated JS. Under the contract their call sites need no +statepoint — any trigger they arm either defers to a declared safepoint or +collects behind the forced scan. First audited set: singleton closure +allocation, class-object allocation, `js_array_push_f64`, `js_array_length`, +`js_array_slice_values`. + +**The census result that bounds the idea.** On `batch.ts`, 217 statepoints +break down as roughly 85 property-access diamonds (getter re-entry possible +— must stay mapped), ~40 coercion/setter/throw paths (re-entry — stay), ~10 +generated-to-generated calls and polls (stay by definition), and only ~25-30 +pure-allocation sites the contract can unmap. **Re-entry, not allocation, is +what bounds the contract's reach on object-heavy code.** Deleting the +property-access calls is representation selection's job (`Ptr`); the +contract unmaps what allocation traffic remains. The two campaigns compose +rather than compete. + ### Work deliberately left gated This follow-up does not alter temporary-root semantics, collection scheduling, From ec9fcac971540de53ce3f2380f9d2e41ea3271e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 23:29:09 +0200 Subject: [PATCH 07/53] research(gc): enforce the safepoint contract on the copying-minor path The copying minor evaluates eligibility in copying.rs and never reaches the cycle.rs root-scan subphase - so the first enforcement point missed exactly the MOVING path the contract exists to police. Add the same check at eligibility evaluation: outside a declared safepoint a copying minor either falls back to the non-moving cycle (heal - whose scan the cycle.rs heal then forces) or panics (strict). --- crates/perry-runtime/src/gc/copying.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index f0e055a6d0..21eb41c21e 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -836,6 +836,31 @@ impl CopiedMinorEligibility { malloc_sweep_due, ); } + // PERRY_GC_SAFEPOINT_ONLY contract: this is the MOVING path, the one + // the contract exists to police. A copying minor consumes precise + // roots, so outside a declared safepoint it must not run: heal mode + // falls back (the non-moving cycle then has its scan forced by the + // cycle.rs heal, which is why reusing the ConservativeStack reason is + // accurate); strict mode panics so gates can prove enforcement. + if !matches!( + super::policy::gc_safepoint_only_contract(), + super::policy::SafepointOnlyContract::Off + ) && super::roots::native_stack_maps_active() + && !super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) + { + if super::policy::gc_safepoint_only_contract() + == super::policy::SafepointOnlyContract::Strict + { + panic!( + "PERRY_GC_SAFEPOINT_ONLY: copying minor began outside a \ + declared safepoint" + ); + } + return Self::fallback( + CopiedMinorFallbackReason::ConservativeStack, + malloc_sweep_due, + ); + } let ptrs = CopyingPointerSet::new(); let (copy_only_reason, legacy_root_stats) = Self::copy_only_root_preflight_reason(&ptrs); if let Some(reason) = copy_only_reason { From faefb5deeca4189a4135dd72279e33fea1b4346a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 07:05:52 +0200 Subject: [PATCH 08/53] fix(gc): heal the safepoint contract through the shared scan override The first enforcement healed by overriding a LOCAL decision variable in the root-scan subphase. Copying-minor eligibility and evacuation pinning read conservative_stack_scan_decision() globally, concluded there were no conservative roots to pin, and PERRY_GC_FORCE_EVACUATE moved objects that raw native-stack words still pointed at - probe 04 span forever in corrupted mutator code (109 CPU-minutes, zero GC frames in 1,489 samples). Consolidate to one chokepoint: contract_scan_heal_guard() at the synchronous collection entries returns a cycle-long ManualGcScanGuard, so every consumer of the scan decision sees the same healed answer. Strict mode panics at the same chokepoint. Deletes both scattered enforcement sites - net less code than the broken version. --- crates/perry-runtime/src/gc/copying.rs | 25 ------------------- crates/perry-runtime/src/gc/cycle.rs | 30 +---------------------- crates/perry-runtime/src/gc/mod.rs | 8 ++++++ crates/perry-runtime/src/gc/policy.rs | 34 ++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 54 deletions(-) diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 21eb41c21e..f0e055a6d0 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -836,31 +836,6 @@ impl CopiedMinorEligibility { malloc_sweep_due, ); } - // PERRY_GC_SAFEPOINT_ONLY contract: this is the MOVING path, the one - // the contract exists to police. A copying minor consumes precise - // roots, so outside a declared safepoint it must not run: heal mode - // falls back (the non-moving cycle then has its scan forced by the - // cycle.rs heal, which is why reusing the ConservativeStack reason is - // accurate); strict mode panics so gates can prove enforcement. - if !matches!( - super::policy::gc_safepoint_only_contract(), - super::policy::SafepointOnlyContract::Off - ) && super::roots::native_stack_maps_active() - && !super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) - { - if super::policy::gc_safepoint_only_contract() - == super::policy::SafepointOnlyContract::Strict - { - panic!( - "PERRY_GC_SAFEPOINT_ONLY: copying minor began outside a \ - declared safepoint" - ); - } - return Self::fallback( - CopiedMinorFallbackReason::ConservativeStack, - malloc_sweep_due, - ); - } let ptrs = CopyingPointerSet::new(); let (copy_only_reason, legacy_root_stats) = Self::copy_only_root_preflight_reason(&ptrs); if let Some(reason) = copy_only_reason { diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 1b643a1151..e2bfc8c762 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -571,35 +571,7 @@ impl RootScanCycleState { self.subphase = RootScanSubphase::MutableSlots; return false; } - let mut conservative_scan_decision = conservative_stack_scan_decision(); - // PERRY_GC_SAFEPOINT_ONLY contract: a collection that skips - // the conservative scan consumes only precise roots, and with - // native stack maps active those exist only at mapped PCs — - // so it may begin only at a declared safepoint (loop poll, - // outermost microtask boundary). Heal mode forces the scan - // for the offending cycle (sound and non-moving); strict - // mode panics so gates can prove the enforcement is live. - // The alloc-point valve and manual gc() already force the - // scan and are exempt by construction. - if !matches!( - conservative_scan_decision, - ConservativeStackScanDecision::Scan - ) && super::roots::native_stack_maps_active() - && !super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) - { - match super::policy::gc_safepoint_only_contract() { - super::policy::SafepointOnlyContract::Off => {} - super::policy::SafepointOnlyContract::Heal => { - conservative_scan_decision = ConservativeStackScanDecision::Scan; - } - super::policy::SafepointOnlyContract::Strict => { - panic!( - "PERRY_GC_SAFEPOINT_ONLY: precise-root collection \ - began outside a declared safepoint" - ); - } - } - } + let conservative_scan_decision = conservative_stack_scan_decision(); // #5029: minors retain old-gen conservative discoveries // pin-only (no trace) — see try_mark_conservative_word. let conservative_root_stats = mark_stack_roots_for_decision( diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index a37055764e..f9ed2ed745 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -102,6 +102,10 @@ pub fn gc_collect_minor() -> u64 { } pub(super) fn gc_collect_minor_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome { + // PERRY_GC_SAFEPOINT_ONLY: held for the whole collection so every + // consumer of the scan decision (root scan, copying eligibility, + // evacuation pinning, verifier) sees the same healed answer. + let _contract_heal = policy::contract_scan_heal_guard(); gc_drain_active_budgeted_cycle(); // Barriers-off ⇒ the remembered set is not being maintained, and a // minor's black-leafed old parents would hide live children. Route @@ -314,6 +318,10 @@ fn gc_collect_inner_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome } fn gc_collect_full_mark_sweep_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome { + // PERRY_GC_SAFEPOINT_ONLY: see gc_collect_minor_with_trigger. Manual + // gc() engages its own force_full_scan first, which this detects as + // already-Scan and no-ops. + let _contract_heal = policy::contract_scan_heal_guard(); gc_drain_active_budgeted_cycle(); GC_TRIGGER_BUMPED.with(|c| c.set(false)); GcCycleState::new_full(trigger).run_to_completion() diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index ec905904c0..649d7c9f08 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -722,6 +722,40 @@ pub(super) fn gc_safepoint_only_contract() -> SafepointOnlyContract { }) } +/// Contract enforcement chokepoint, called once at every synchronous +/// collection entry. When an undeclared precise-root collection is about to +/// begin, heal mode returns a scan-override guard that must be held for the +/// WHOLE collection: it flips the thread-local override that every consumer +/// of `conservative_stack_scan_decision()` reads — the root-scan subphase, +/// copying-minor eligibility, and the evacuation verifier alike. A previous +/// revision healed by overriding a local variable inside the root-scan +/// subphase only; copying-minor eligibility still read the global decision, +/// concluded there were no conservative roots to pin, and forced evacuation +/// moved objects that raw native-stack words still pointed at. +pub(super) fn contract_scan_heal_guard() -> Option { + if gc_safepoint_only_contract() == SafepointOnlyContract::Off { + return None; + } + if !super::roots::native_stack_maps_active() + || GC_AT_DECLARED_SAFEPOINT.with(Cell::get) + { + return None; + } + if matches!( + super::roots::conservative_stack_scan_decision(), + super::roots::ConservativeStackScanDecision::Scan + ) { + return None; + } + if gc_safepoint_only_contract() == SafepointOnlyContract::Strict { + panic!( + "PERRY_GC_SAFEPOINT_ONLY: precise-root collection began outside \ + a declared safepoint" + ); + } + Some(super::roots::ManualGcScanGuard::force_full_scan()) +} + /// RAII marker for a declared-safepoint drain. Nesting-safe: restores the /// previous value so a poll firing inside a manual `gc()` cannot clear it. pub(super) struct DeclaredSafepointGuard { From d03827611634c533107785018cd13b8c64805f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 07:34:01 +0200 Subject: [PATCH 09/53] fix(gc): delete the per-poll trigger drain from the safepoint contract Draining non-nursery triggers at every allocating loop back-edge turned nursery-churn loops into per-iteration collection work - O(n^2), probe 01 burned 20 CPU-minutes on a 200ms workload (sample: dominant runtime frames + TLS + memmove = collection work per iteration, unlike the split-brain hang's pure-mutator signature). The extension was an optimization, not a soundness requirement: an undeclared full at an alloc point heals with one conservative scan. Polls return to their single job - draining the pending moving minor. --- crates/perry-runtime/src/gc/policy.rs | 13 +------------ docs/statepoint-gc-experiment.md | 8 +++++--- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 649d7c9f08..6ce8056131 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1869,18 +1869,7 @@ pub extern "C" fn js_gc_loop_safepoint() { if !GC_SAFEPOINT_PENDING.with(Cell::get) && !super::gc_zeal_enabled() { return; } - if GC_SAFEPOINT_PENDING.with(Cell::get) { - gc_safepoint_moving_minor(); - return; - } - // Under the safepoint-only contract, non-nursery (full/old-gen) triggers - // should preferentially drain at declared safepoints, where precise roots - // make the conservative heal unnecessary. The dueness check inside - // `gc_check_trigger` is cheap and this branch is research-mode only. - if gc_safepoint_only_contract() != SafepointOnlyContract::Off { - let _declared = DeclaredSafepointGuard::enter(); - gc_check_trigger(); - } + gc_safepoint_moving_minor(); } struct BudgetedGcStepGuard; diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index c8939d88a4..2a05552a3c 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -200,9 +200,11 @@ precise-root cycle gets the conservative scan forced for that cycle, which restores liveness and keeps it non-moving) and `=strict` (panic — the gate mode that proves the enforcement is live, per the four-ways-a-gate-cannot- fail rule). Manual `gc()` and the alloc-point slack valve force the scan -already and are exempt by construction. Under the contract, loop polls also -drain non-nursery triggers so full collections migrate to declared -safepoints. +already and are exempt by construction. (An earlier revision also drained +non-nursery triggers at every allocating loop back-edge; that turned churn +loops into per-iteration collection work — O(n²) — and was deleted. The heal +alone is sufficient: undeclared full collections simply pay one conservative +scan.) **What it unmaps.** A new audited `GcCallEffect::AllocNoReentry` class: helpers that may allocate (arming a trigger) but never collect synchronously From f74c0ce7bef6c76f1245598b9848cd0cc7233edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 07:40:46 +0200 Subject: [PATCH 10/53] docs: record contract gate results and the three bugs the gates caught --- docs/statepoint-gc-experiment.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 2a05552a3c..e9399758e3 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -214,6 +214,20 @@ collects behind the forced scan. First audited set: singleton closure allocation, class-object allocation, `js_array_push_f64`, `js_array_length`, `js_array_slice_values`. +**Measured results (loaded host, correctness-grade).** All gates green at +`4e3d5c70e`: 16/16 probe cells (forced evacuation + walker-verify under the +contract), strict-mode enforcement fired on the deliberately unsound +configuration (`PERRY_GC_SCAVENGE=1` + polls off — a precise-root minor at +an unmapped alloc point aborts with the contract panic), and max RSS is +unchanged by deferral (27/27 MB and 36/36 MB on the two churn-heaviest +probes). The audited five-helper set removes 7.8% of `batch.ts` statepoints +(217 → 200) and 7.5% of relocations. Getting the contract here found and +fixed three implementation bugs, each caught by a gate: enforcement that +missed the copying-minor path, a heal that overrode a local decision while +copying eligibility read the global one (real memory corruption under +forced evacuation), and a per-poll trigger drain that turned churn loops +into O(n²) collection work. All three fixes deleted code. + **The census result that bounds the idea.** On `batch.ts`, 217 statepoints break down as roughly 85 property-access diamonds (getter re-entry possible — must stay mapped), ~40 coercion/setter/throw paths (re-entry — stay), ~10 From 1984a3961143a7089d5609df826c150a0e188b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 07:48:53 +0200 Subject: [PATCH 11/53] docs: quiet-host matrix results from the reserved M1 mini Deep-stack closed (walker-attributed via the unwind control arm), compile +5.3% claim withdrawn, RSS flat, statepoints at-worst-tied on wall clock; metadata remains the only losing axis. 10ms timer quantum caveat recorded. --- docs/statepoint-gc-experiment.md | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index e9399758e3..d9c16533a5 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -249,6 +249,59 @@ prerequisite lands, the next experiment is to assert that moving collections occur only at declared safepoints and then measure whether the conservative scanner can be deleted. +## Quiet-host matrix (2026-08-01, reserved Mac mini) + +First measurement of this experiment not taken on a loaded host: Apple M1 +(4P+4E), macOS 26.5.1, the gc-ratchet pinned-baseline platform, reserved +with baseline load 1.4–1.9 from release infrastructure only (recorded +per-rep). Artifacts shipped SHA-pinned from `4e3d5c70e` (no cargo on the +host); all four arms hash-distinct per probe; 8×4 forced-evacuation preflight +plus walker-verify and the strict-enforcement gate all green there before +any timing. 11 interleaved reps, rotated arm order, `/usr/bin/time -l`. +Caveat: 10 ms timer granularity puts ±1 quantum (≈2–9% on these probe +durations) on any single cell; medians were stable across reps. + +Runtime, median seconds (spread), delta vs shadow: + +| Probe | Shadow | Plain map | Statepoint | Contract | +|---|---:|---:|---:|---:| +| Nursery churn | 0.160 | 0.160 (+0.0%) | 0.160 (+0.0%) | 0.160 (+0.0%) | +| Survivor promotion | 0.190 | 0.180 (−5.3%) | 0.190 (+0.0%) | 0.190 (+0.0%) | +| Cross-gen writes | 0.190 | 0.180 (−5.3%) | 0.180 (−5.3%) | 0.190 (+0.0%) | +| **Dead after deep stack** | 0.430 | 0.410 (−4.7%) | **0.410 (−4.7%)** | 0.410 (−4.7%) | +| Closure capture | 0.140 | 0.130 (−7.1%) | 0.130 (−7.1%) | 0.130 (−7.1%) | +| String retention | 0.110 | 0.110 (+0.0%) | 0.120 (+9.1%, one quantum) | 0.120 | +| Array grow/evacuate | 0.150 | 0.150 (+0.0%) | 0.150 (+0.0%) | 0.150 (+0.0%) | +| Map/set side tables | 0.430 | 0.430 (+0.0%) | 0.430 (+0.0%) | 0.430 (+0.0%) | + +Geometric means vs shadow: plain maps −2.83%, statepoints −1.10%, +contract −0.43%. + +**The deep-stack weakness is closed.** The probe that was ~20% slower on +the loaded host is now 4.7% *faster* than shadow, and the attribution is +exact: the same statepoint binary with `PERRY_STACKMAP_WALKER=unwind` runs +at 0.430 — precise shadow parity — so the x29-chain walker is the entire +difference. + +Max RSS: every cell within ±0.8% of shadow (ratchet-comparable platform). +Uncached `batch.ts` compile: shadow 0.590 s, statepoints 0.570 s (−3.4%) — +the loaded-host "+5.3% slower to compile" claim did not survive quiet +measurement and is withdrawn. + +Metadata (`__llvm_stackmaps`, summed over the eight probes): plain maps +42,936 B, statepoints 81,104 B (1.89×), contract 73,952 B (−8.8% vs +statepoint). Generated `__text` is ~5.8 KB smaller across the eight +binaries in the native arms (probe code is small; the shadow-stack text +delta scales with generated code, per #7108's 13.3% on a real app). + +Standing conclusion after this matrix: on wall-clock, RSS, and compile +time, statepoints are at worst tied with the shadow stack on this +hardware; metadata remains the only losing axis, and it is the axis +repsel promotion shrinks. The plain-map arm no longer earns its keep as +anything but a control: statepoints match it within quantization, and it +is structurally unsound (`Register R#1`). Small-hardware and Linux +numbers still require the ELF scanner port. + ## Which statepoint design this tests This is the explicit bridge, not LLVM's `RewriteStatepointsForGC` pipeline. From cda6ec0f7ab41a2757c70f686d75eeb8ade6728b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 07:56:55 +0200 Subject: [PATCH 12/53] research(gc): delete the plain-map user mode; elide statepoints at noreturn sites PERRY_STACK_MAPS is gone per the GC knob kill-policy: after the quiet-host matrix it was a losing mode (statepoints match it within timer quantization) and it is structurally unsound - LLVM's stackmap intrinsic can record a root slot's address as Register R#N (caller-saved, unrecoverable at collection time), leaving those roots invisible to the collector. The plain-map lowering survives only as statepoint mode's internal fallback for try/setjmp functions; shrinking that fallback set is tracked follow-up work. The env leaves both cache-key sets with it. New audited GcCallEffect::NeverReturns class: every js_throw* helper funnels into exception::js_throw (-> !), so control never returns to the call site, no relocation is ever consumed, and the frame's roots are dead past the call - the site needs no metadata in any mode. Deeper frames carry their own records; values the helper holds are its own frame's responsibility, as for every helper call. batch.ts carries 19 such sites. --- crates/perry-codegen/src/codegen/helpers.rs | 39 ++++++------------- crates/perry-codegen/src/function.rs | 4 ++ crates/perry-codegen/src/gc_call_effects.rs | 9 +++++ crates/perry-codegen/src/statepoint_report.rs | 2 +- .../perry/src/commands/compile/build_cache.rs | 1 - .../src/commands/compile/object_cache.rs | 8 +--- .../object_cache/object_cache_tests.rs | 1 - crates/perry/src/commands/compile/types.rs | 2 +- 8 files changed, 29 insertions(+), 37 deletions(-) diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index 535773d0fb..bd06b3739f 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -74,34 +74,19 @@ pub(super) fn shadow_stack_enabled() -> bool { }) } -/// Research-only precise-root backend using LLVM's -/// `llvm.experimental.stackmap` intrinsic. -/// -/// `PERRY_STACK_MAPS=1` keeps the existing pointer-local/liveness analysis but -/// changes the storage and discovery mechanism: roots remain in their native -/// frame allocas and LLVM records those writable locations at call sites. The -/// runtime can then unwind the native stack and visit the exact slots without -/// a parallel heap-backed shadow stack. -/// -/// This is intentionally opt-in while the experiment establishes correctness, -/// target coverage, and performance. It is independent of -/// `PERRY_SHADOW_STACK=0`: the latter still disables precise-root analysis -/// entirely, while this selects the backend used when that analysis is on. -pub(crate) fn stack_maps_enabled() -> bool { - matches!( - std::env::var("PERRY_STACK_MAPS").as_deref(), - Ok("1") | Ok("on") | Ok("true") - ) -} - /// Research-only moving-GC backend using LLVM's explicit statepoint -/// relocation sequence. +/// relocation sequence (`PERRY_STATEPOINTS=1`). /// -/// This is separate from `PERRY_STACK_MAPS` so the two native-stack -/// prototypes can be measured independently. Statepoint mode still consumes -/// LLVM's stack-map section at runtime, but supported calls are represented -/// by `gc.statepoint` / `gc.result` / `gc.relocate` instead of a standalone -/// metadata marker plus compiler memory barriers. +/// The standalone plain-stack-map mode (`PERRY_STACK_MAPS`) was deleted per +/// the GC knob kill-policy after the quiet-host matrix: statepoints matched +/// it within timer quantization, and it is structurally unsound — LLVM's +/// stackmap intrinsic can record a root slot's address as `Register R#N` +/// (caller-saved, unrecoverable at collection time), making those roots +/// invisible to the collector by construction. The plain-map LOWERING +/// survives only as this mode's internal fallback for `try`/setjmp +/// functions and unsupported call forms. The Register hazard exists there +/// too, which is why shrinking the fallback set is the remaining +/// correctness work for this backend, tracked in the experiment doc. pub(crate) fn statepoints_enabled() -> bool { matches!( std::env::var("PERRY_STATEPOINTS").as_deref(), @@ -112,7 +97,7 @@ pub(crate) fn statepoints_enabled() -> bool { /// Whether precise roots should use a native-stack metadata backend rather /// than Perry's heap-backed shadow frame. pub(crate) fn native_stack_roots_enabled() -> bool { - stack_maps_enabled() || statepoints_enabled() + statepoints_enabled() } /// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index ab48d927b7..7954b13601 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -1316,6 +1316,10 @@ fn lower_precise_roots_to_native_stack( let cannot_collect = direct_callee.is_some_and(|callee| { match crate::gc_call_effects::classify_direct_callee(callee) { crate::gc_call_effects::GcCallEffect::CannotCollect => true, + // Control never returns here: no relocation is consumed and + // the frame's roots are dead past the call. Deeper frames + // carry their own records. + crate::gc_call_effects::GcCallEffect::NeverReturns => true, // Under the explicit-safepoint contract the runtime // guarantees these helpers' triggers never consume this // frame's precise roots (they defer to a declared safepoint diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 8313ed408a..44a6c67f89 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -23,6 +23,14 @@ pub(crate) enum GcCallEffect { /// consumed at this call site and it needs no statepoint. Without the /// contract these remain safepoints. AllocNoReentry, + /// The callee never returns to this call site (audited 2026-08-01: every + /// `js_throw*` helper funnels into `exception::js_throw`, which is + /// `-> !` — the `f64` results are unreachable ABI shape). No relocation + /// can ever be consumed downstream and the frame's roots are dead past + /// the call, so the site needs no metadata in ANY mode. Values the + /// helper itself holds are its own frame's responsibility + /// (`RuntimeHandleScope`/temp roots), exactly as for every helper call. + NeverReturns, Unknown, } @@ -92,6 +100,7 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_array_push_f64" | "js_array_length" | "js_array_slice_values" => GcCallEffect::AllocNoReentry, + name if name.starts_with("js_throw") => GcCallEffect::NeverReturns, _ => GcCallEffect::Unknown, } } diff --git a/crates/perry-codegen/src/statepoint_report.rs b/crates/perry-codegen/src/statepoint_report.rs index 6cd034d460..733fc08923 100644 --- a/crates/perry-codegen/src/statepoint_report.rs +++ b/crates/perry-codegen/src/statepoint_report.rs @@ -216,7 +216,7 @@ pub fn render_text(records: &[FunctionRecord]) -> String { ); if records.is_empty() { out.push_str( - "No native-stack lowering records were emitted. Enable PERRY_STACK_MAPS=1\n\ + "No native-stack lowering records were emitted. Enable PERRY_STATEPOINTS=1\n\ or PERRY_STATEPOINTS=1 and ensure codegen is not served from cache.\n", ); return out; diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 032804347f..36f69b581d 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -35,7 +35,6 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_LLVM_CLANG", "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", - "PERRY_STACK_MAPS", "PERRY_STATEPOINTS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index b0f658d1e2..1db7bff07e 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -229,7 +229,7 @@ fn stable_type_key(ty: &perry_hir::types::Type) -> String { /// We also mix in environment variables that `perry-codegen` reads /// at compile time but that aren't part of `CompileOptions`: /// `PERRY_DEBUG_INIT`, `PERRY_DEBUG_SYMBOLS`, `PERRY_LLVM_CLANG`, -/// `PERRY_WRITE_BARRIERS`, `PERRY_SHADOW_STACK`, `PERRY_STACK_MAPS`, +/// `PERRY_WRITE_BARRIERS`, `PERRY_SHADOW_STACK`, /// `PERRY_STATEPOINTS`, /// `PERRY_DISABLE_BUFFER_FAST_PATH`, `PERRY_VERIFY_NATIVE_REGIONS`, /// `PERRY_UNBOXED_OBJECT_FIELDS`, and `PERRY_TARGET_CPU`. See the env-var @@ -760,7 +760,7 @@ fn compute_object_cache_key_with_env( // calls at heap-store sites (codegen.rs / expr.rs). // - PERRY_SHADOW_STACK=0/off/false suppresses generated frame/slot // roots at function entry and pointer local stores. - // - PERRY_STACK_MAPS=1 lowers those precise roots to LLVM native-frame + // - (historical) PERRY_STACK_MAPS lowered precise roots to plain LLVM stackmap records; deleted, statepoint fallback keeps the lowering internal. PERRY_STATEPOINTS=1 lowers roots to native-frame // stack maps instead of the runtime shadow stack. // - PERRY_STATEPOINTS=1 replaces supported calls with LLVM statepoint // relocation sequences and uses native stack maps for the remainder. @@ -799,10 +799,6 @@ fn compute_object_cache_key_with_env( "env_shadow_stack", env_var("PERRY_SHADOW_STACK").as_deref().unwrap_or(""), ); - h.field( - "env_stack_maps", - env_var("PERRY_STACK_MAPS").as_deref().unwrap_or(""), - ); h.field( "env_statepoints", env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index 6e22f84eb4..93e44fd58f 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -583,7 +583,6 @@ fn key_changes_with_codegen_env_vars() { "PERRY_LLVM_CLANG", "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", - "PERRY_STACK_MAPS", "PERRY_STATEPOINTS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", diff --git a/crates/perry/src/commands/compile/types.rs b/crates/perry/src/commands/compile/types.rs index a15d9fbbc2..0f266205a2 100644 --- a/crates/perry/src/commands/compile/types.rs +++ b/crates/perry/src/commands/compile/types.rs @@ -500,7 +500,7 @@ pub struct CompileArgs { /// cannot collect, statepoint relocation counts, plain stack-map /// fallbacks, and the live-root-width distribution. /// - /// Useful with `PERRY_STACK_MAPS=1` or `PERRY_STATEPOINTS=1`. + /// Useful with `PERRY_STATEPOINTS=1`. /// `--statepoint-report=json` emits a stable machine-readable schema. /// Observational only; cache reuse is disabled for the reporting run so /// codegen executes and produces records. From a85c05f2edb49ebdb6d7fc4f49839d3d97344b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 08:01:15 +0200 Subject: [PATCH 13/53] docs: post-matrix follow-through - mode deletion, noreturn elision, metadata trajectory --- docs/statepoint-gc-experiment.md | 37 ++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index d9c16533a5..8edc02b042 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -297,10 +297,39 @@ delta scales with generated code, per #7108's 13.3% on a real app). Standing conclusion after this matrix: on wall-clock, RSS, and compile time, statepoints are at worst tied with the shadow stack on this hardware; metadata remains the only losing axis, and it is the axis -repsel promotion shrinks. The plain-map arm no longer earns its keep as -anything but a control: statepoints match it within quantization, and it -is structurally unsound (`Register R#1`). Small-hardware and Linux -numbers still require the ELF scanner port. +repsel promotion shrinks. Small-hardware and Linux numbers still require +the ELF scanner port. + +## Post-matrix follow-through (2026-08-01, `897e0f53b`) + +Two changes landed after the matrix, both gate-verified (16/16 forced +evacuation + walker-verify, strict-enforcement gate fires, RSS flat): + +1. **The plain-map user mode is deleted** per the GC knob kill-policy: + after the quiet matrix it was a losing configuration (statepoints match + it within timer quantization), and it is structurally unsound — LLVM's + stackmap intrinsic can record a root slot's address as `Register R#N`, + caller-saved and unrecoverable at collection time, so those roots are + invisible to the collector by construction. The lowering survives only + as statepoint mode's internal `try`/setjmp fallback; shrinking that + fallback set is the remaining correctness work for the backend. +2. **Noreturn call sites carry no metadata** (`GcCallEffect::NeverReturns`): + every `js_throw*` helper funnels into `exception::js_throw` (`-> !`), so + control never returns, no relocation is ever consumed, and the frame's + roots are dead past the call. Sound in any mode; deeper frames carry + their own records. + +Metadata trajectory on `batch.ts` statepoints, all without any +representation-selection improvement: 442 (first prototype) → 217 +(call-effect audit) → 198 (noreturn elision) → **181 under the contract — +−59% total**. Each remaining big step is identified: the property-access +diamonds (~85 sites) fall to repsel `Ptr`, and the v3 format itself +wastes ~36 B/record on constant locations plus ~12 B/root on base/derived +duplication that a Perry-owned compact section could reclaim — but the +compact-section design needs either post-link fixup surgery or a +per-function (not per-safepoint) precision model, both of which are real +projects with open soundness questions, recorded here so the next session +starts from the design constraints rather than rediscovering them. ## Which statepoint design this tests From 51dfa619f8f6c4c2a7ea7fa2be13a3c3a75fd0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 08:18:32 +0200 Subject: [PATCH 14/53] research(gc): compact per-function root metadata (PERRY_COMPACT_ROOTS) The file-size lever that does not wait for repsel. One stackmap intrinsic in the entry block records every root alloca as a stable Direct location; calls carry only zero-instruction memory barriers. Precision drops from per-safepoint to per-function - sound because root allocas are already zero-initialized at entry, so visiting a stale slot can only over-retain, never corrupt. Metadata falls from ~64 B/safepoint + 24 B/root-pair to ~40 B/function + 12 B/slot: on the #7108 real-app model, 4.5-16.6 MB becomes ~120 KB - below the shadow stack's 439 KB of hot text. - Every generated function is lowered (rootless ones get a zero-operand entry record) so region matching can never attribute a frame to a neighboring function; block-local root slots fall back to the statepoint backend per function; has_try needs no exclusion because there is no per-call rewriting to conflict with setjmp. - A __perry_gen_end sentinel object is linked after every generated object; its magic-ID record is both the region's exclusive upper bound and the runtime's compact-mode signal. - The runtime matches frames by region (greatest record PC at or below the return address, bounded by the sentinel) instead of the +-16-byte per-safepoint heuristic; both walkers share the new match_records. - Fail-closed: the parser counts register-recorded locations, and a compact image refuses to run with any present - in compact mode the entry record is the only description of the frame, so a register root would be silently invisible. - PERRY_COMPACT_ROOTS participates in build and object cache keys. Known pre-existing failure, not from this change: the branch's gc::tests::shadow_stack_ops::out_of_range_frame_pop_is_ignored aborts (panic inside a nounwind path, shadow_stack.rs:531, last touched by main's #7088) - fails identically without this diff. --- crates/perry-codegen/src/codegen/helpers.rs | 18 +- crates/perry-codegen/src/function.rs | 174 +++++++++++++- crates/perry-codegen/src/lib.rs | 1 + .../perry-runtime/src/gc/roots/stack_maps.rs | 218 ++++++++++++++---- .../perry/src/commands/compile/build_cache.rs | 1 + .../src/commands/compile/object_cache.rs | 4 + .../object_cache/object_cache_tests.rs | 1 + .../src/commands/compile/run_pipeline.rs | 31 +++ 8 files changed, 390 insertions(+), 58 deletions(-) diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index bd06b3739f..7e95ec52c6 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -94,10 +94,26 @@ pub(crate) fn statepoints_enabled() -> bool { ) } +/// `PERRY_COMPACT_ROOTS=1` — compact per-function native-root metadata +/// (research). One entry stackmap per generated function records every root +/// alloca as a stable Direct location; calls carry only memory barriers. +/// See `PreciseRootBackend::CompactEntry`. Takes precedence over +/// `PERRY_STATEPOINTS` when both are set. +pub(crate) fn compact_roots_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_COMPACT_ROOTS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + /// Whether precise roots should use a native-stack metadata backend rather /// than Perry's heap-backed shadow frame. pub(crate) fn native_stack_roots_enabled() -> bool { - statepoints_enabled() + statepoints_enabled() || compact_roots_enabled() } /// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 7954b13601..7dadf57149 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -757,7 +757,20 @@ impl LlFunction { // Research backend: turn the existing shadow-slot binding IR into // native-frame stack maps only after lowering is complete, when every // lazily-reserved scalar root and every call site is visible. - let ir = if self.stack_map_requested { + // + // Compact mode lowers EVERY generated function — rootless ones get a + // zero-operand entry record so the runtime's region matching can + // never attribute their frames to a neighboring rooted function. It + // also has no has_try exclusion: with no per-call rewriting there is + // nothing to conflict with the setjmp lowering. + let ir = if crate::codegen::helpers::compact_roots_enabled() { + lower_precise_roots_to_native_stack( + &ir, + &self.name, + self.stack_map_slot_count, + PreciseRootBackend::CompactEntry, + ) + } else if self.stack_map_requested { let backend = if crate::codegen::helpers::statepoints_enabled() && !self.has_try { PreciseRootBackend::Statepoint } else { @@ -955,6 +968,20 @@ fn stack_map_active_slots( enum PreciseRootBackend { StackMap, Statepoint, + /// Compact per-function mode (`PERRY_COMPACT_ROOTS=1`): ONE stackmap + /// intrinsic in the entry block records every root alloca as a stable + /// Direct FP-relative location; calls carry only zero-instruction memory + /// barriers (store-before / reload-after), no per-safepoint metadata. + /// Precision drops from per-safepoint to per-function — sound because + /// every root alloca is zero-initialized at entry, so visiting a stale + /// slot can only over-retain, never corrupt. Metadata cost falls from + /// ~64 B/safepoint + 24 B/root-pair to ~40 B/function + 12 B/slot. + /// + /// The walker matches frames by function REGION: every generated + /// function (rooted or not) carries an entry record, and a sentinel + /// `__perry_gen_end` object linked last bounds the region, so a foreign + /// frame can never false-match a generated record. + CompactEntry, } impl PreciseRootBackend { @@ -962,10 +989,26 @@ impl PreciseRootBackend { match self { Self::StackMap => "stack-map", Self::Statepoint => "statepoint", + Self::CompactEntry => "compact-entry", } } } +/// Record ID of the `__perry_gen_end` sentinel. Its presence in the parsed +/// section is what flips the runtime walker into compact region matching, +/// and its PC is the exclusive upper bound of generated code. +pub const COMPACT_SENTINEL_STACKMAP_ID: u64 = 0x9E44_C0DE_0DEC_1DED; + +/// A textual-IR basic-block label line (`entry.0:` — one token, ends with a +/// colon, not a comment). +fn is_block_label(line: &str) -> bool { + let trimmed = line.trim(); + !trimmed.is_empty() + && trimmed.ends_with(':') + && !trimmed.contains(' ') + && !trimmed.starts_with(';') +} + #[derive(Debug, Eq, PartialEq)] struct DirectCall<'a> { result: Option<&'a str>, @@ -1228,11 +1271,23 @@ fn lower_precise_roots_to_native_stack( ) }); if root_ptrs.is_empty() { - let out = ir - .lines() - .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) - .map(|line| format!("{line}\n")) - .collect(); + let mut out = String::with_capacity(ir.len() + 96); + let mut entry_record_emitted = backend != PreciseRootBackend::CompactEntry; + for line in ir.lines() { + if parse_shadow_bind(line).is_some() || parse_shadow_set(line).is_some() { + continue; + } + out.push_str(line); + out.push('\n'); + // Compact mode: a rootless function still needs its entry record + // as a region boundary for the runtime's frame matching. + if !entry_record_emitted && is_block_label(line) { + out.push_str( + " call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0)\n", + ); + entry_record_emitted = true; + } + } if let Some(report) = report { crate::statepoint_report::record(report); } @@ -1243,8 +1298,25 @@ fn lower_precise_roots_to_native_stack( let mut available = std::collections::HashSet::::new(); let mut initialized = std::collections::HashSet::::new(); let mut map_id = 0u64; + let mut compact_entry_emitted = backend != PreciseRootBackend::CompactEntry; + let mut labels_seen = 0usize; for (line_idx, line) in lines.iter().enumerate() { + // Compact mode requires every root alloca to be recordable from the + // entry block. A block-local slot (rare scalar-replacement shapes) + // would leave the entry record incomplete, so the whole function + // falls back to the per-safepoint statepoint backend instead. + if !compact_entry_emitted && is_block_label(line) { + labels_seen += 1; + if labels_seen >= 2 { + return lower_precise_roots_to_native_stack( + ir, + function_name, + slot_count, + PreciseRootBackend::Statepoint, + ); + } + } if parse_shadow_bind(line).is_some() { // Compile-time marker only. The real slot is already populated by // the local store immediately preceding this old bind. @@ -1284,6 +1356,20 @@ fn lower_precise_roots_to_native_stack( } } + // Compact mode: the moment every root alloca exists (still inside + // the entry block, or the label check above would have bailed), emit + // the function's single stackmap with the whole slot set. Escaping + // every root address through the intrinsic also pins the allocas as + // address-taken for the rest of the pipeline. + if !compact_entry_emitted && initialized.len() == root_ptrs.len() { + let operands: Vec = root_ptrs.iter().map(|p| format!("ptr {p}")).collect(); + out.push_str(&format!( + " call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0, {})\n", + operands.join(", ") + )); + compact_entry_emitted = true; + } + // Insert before calls, not after. Rebuild the tail when the line just // appended is a call so the intrinsic's instruction offset is the // actual call-site offset in the final machine function. @@ -1346,6 +1432,17 @@ fn lower_precise_roots_to_native_stack( // Move the call line behind the intrinsic. let call_len = line.len() + 1; out.truncate(out.len() - call_len); + if backend == PreciseRootBackend::CompactEntry { + // No per-safepoint metadata. The pre-call barrier forces live + // root stores to be materialized before the callee can collect; + // the post-call barrier forces reloads of anything the collector + // may have rewritten in the entry-recorded slots. + out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); + out.push_str(line); + out.push('\n'); + out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); + continue; + } if backend == PreciseRootBackend::Statepoint { if let Some(call) = parse_direct_statepoint_call(line) { emit_statepoint(&mut out, &call, &live, map_id); @@ -1387,6 +1484,71 @@ mod stack_map_tests { lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::Statepoint) } + fn lower_compact(input: &str, slots: u32) -> String { + lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::CompactEntry) + } + + #[test] + fn compact_emits_one_entry_record_and_barrier_wrapped_calls() { + let input = r#"define i64 @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 @may_collect() + call void @may_collect_again() + ret i64 %r1 +} +"#; + let output = lower_compact(input, 1); + // Exactly one stackmap, in the entry block, carrying the whole slot set. + assert_eq!(output.matches("@llvm.experimental.stackmap").count(), 1); + assert!(output.contains("stackmap(i64 0, i32 0, ptr %r0)")); + let map_at = output.find("@llvm.experimental.stackmap").unwrap(); + let first_call = output.find("@may_collect()").unwrap(); + assert!(map_at < first_call, "entry record must precede the first call"); + // Calls carry pre+post barriers and no metadata. + assert!(output.contains( + "call void asm sideeffect \"\", \"~{memory}\"()\n %r1 = call i64 \ + @may_collect()\n call void asm sideeffect \"\", \"~{memory}\"()" + )); + assert!(!output.contains("gc.statepoint")); + } + + #[test] + fn compact_rootless_function_still_gets_a_region_record() { + let input = r#"define void @probe() { +entry.0: + call void @leaf() + ret void +} +"#; + let output = lower_compact(input, 0); + assert_eq!(output.matches("@llvm.experimental.stackmap").count(), 1); + assert!(output.contains("stackmap(i64 0, i32 0)\n call void @leaf()")); + } + + #[test] + fn compact_falls_back_to_statepoints_for_block_local_slots() { + let input = r#"define void @probe(i1 %cond) { +entry.0: + br i1 %cond, label %then.1, label %exit.2 +then.1: + %r0 = alloca i64 + store i64 5, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @may_collect() + br label %exit.2 +exit.2: + ret void +} +"#; + let output = lower_compact(input, 1); + // Block-local root alloca: the compact entry record cannot cover it, + // so the function must take the per-safepoint statepoint backend. + assert!(output.contains("gc.statepoint")); + } + #[test] fn lowers_bind_and_liveness_clear_to_native_frame_maps() { let input = r#"define i64 @probe(i64 %arg) { diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index 45a9a626e9..17e0b4387f 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod collectors; pub mod expr; pub mod ext_registry; pub mod function; +pub use function::COMPACT_SENTINEL_STACKMAP_ID; pub(crate) mod gc_call_effects; pub mod linker; pub(crate) mod loop_purity; diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 4b1978fb1d..5630f87802 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -19,9 +19,15 @@ use std::ffi::c_void; use std::sync::OnceLock; const STACK_MAP_VERSION: u8 = 3; +const LOCATION_REGISTER: u8 = 1; const LOCATION_DIRECT: u8 = 2; const LOCATION_INDIRECT: u8 = 3; const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; +/// Mirrors `perry_codegen::function::COMPACT_SENTINEL_STACKMAP_ID`: the +/// record ID of the `__perry_gen_end` sentinel that a compact-roots build +/// links after every generated object. Its presence flips region matching +/// on, and its PC is the exclusive upper bound of generated code. +const COMPACT_SENTINEL_STACKMAP_ID: u64 = 0x9E44_C0DE_0DEC_1DED; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct StackMapLocation { dwarf_reg: u16, @@ -31,6 +37,7 @@ struct StackMapLocation { #[derive(Clone, Debug, Eq, PartialEq)] struct StackMapRecord { pc: usize, + id: u64, /// The containing function's total frame size from the stack-map header. /// LLVM's AArch64 frame places the `[x29, x30]` pair at the top of the /// frame, so a chain walker can reconstruct the body SP as @@ -52,6 +59,16 @@ struct StackMapIndex { chain_walkable: bool, min_pc: usize, max_pc: usize, + /// `Some(gen_end_pc)` when the `__perry_gen_end` sentinel is present: + /// this is a compact-roots build, frames match by function region + /// (greatest record PC at or below the return address, bounded by + /// `gen_end_pc`) instead of the ±16-byte per-safepoint heuristic. + compact_gen_end: Option, + /// Locations the parser had to drop because LLVM recorded them in a + /// register. Harmlessly redundant in per-safepoint modes (statepoint + /// spills are memory); fatal in compact mode, where the entry record is + /// the only description of the frame — checked at init. + dropped_register_locations: usize, } static STACK_MAPS: OnceLock = OnceLock::new(); @@ -124,13 +141,25 @@ fn stack_maps() -> &'static StackMapIndex { let Some(section) = loaded_stack_map_section() else { return StackMapIndex::default(); }; - let mut records = parse_concatenated_stack_maps(section).unwrap_or_default(); + let (mut records, dropped) = parse_concatenated_stack_maps(section).unwrap_or_default(); records.sort_unstable_by_key(|record| record.pc); - index_records(records) + let index = index_records(records, dropped); + if index.compact_gen_end.is_some() && index.dropped_register_locations != 0 { + // A compact build's entry record is the ONLY description of its + // frame. A register-recorded root would be silently invisible to + // the collector, so this configuration must not run at all. + panic!( + "PERRY_COMPACT_ROOTS: {} register-recorded root location(s) \ + in a compact-roots image — refusing to run with invisible \ + roots", + index.dropped_register_locations + ); + } + index }) } -fn index_records(records: Vec) -> StackMapIndex { +fn index_records(records: Vec, dropped_register_locations: usize) -> StackMapIndex { let chain_walkable = records.iter().all(|record| { record.locations.iter().all(|location| { location.dwarf_reg == DWARF_REG_FP_AARCH64 @@ -139,11 +168,49 @@ fn index_records(records: Vec) -> StackMapIndex { }); let min_pc = records.first().map_or(usize::MAX, |record| record.pc); let max_pc = records.last().map_or(0, |record| record.pc); + let compact_gen_end = records + .iter() + .find(|record| record.id == COMPACT_SENTINEL_STACKMAP_ID) + .map(|record| record.pc); StackMapIndex { records, chain_walkable, min_pc, max_pc, + compact_gen_end, + dropped_register_locations, + } +} + +impl StackMapIndex { + /// The records describing the frame whose return address is `ip`. + /// + /// Compact builds match by function region: the greatest record PC at or + /// below `ip`, valid only inside `[first record, __perry_gen_end)` — a + /// foreign (runtime) frame can never match because generated code is + /// linked contiguously ahead of the sentinel. Per-safepoint builds keep + /// the ±16-byte nearest-PC match, which can select several records at + /// one PC. + fn match_records(&self, ip: usize) -> &[StackMapRecord] { + if let Some(gen_end) = self.compact_gen_end { + if ip < self.min_pc || ip >= gen_end { + return &[]; + } + let idx = self.records.partition_point(|record| record.pc <= ip); + let Some(idx) = idx.checked_sub(1) else { + return &[]; + }; + return &self.records[idx..idx + 1]; + } + let Some(candidate_pc) = closest_record_pc(&self.records, ip) else { + return &[]; + }; + if ip.abs_diff(candidate_pc) > MAX_SAFEPOINT_RETURN_DELTA { + return &[]; + } + let first = self.records.partition_point(|record| record.pc < candidate_pc); + let last = self.records.partition_point(|record| record.pc <= candidate_pc); + &self.records[first..last] } } @@ -174,14 +241,14 @@ pub(super) fn visit_stack_map_root_slots( return NativeStackWalkStats::default(); } match walker_mode() { - WalkerMode::Unwind => unwind::visit(&index.records, visit), + WalkerMode::Unwind => unwind::visit(index, visit), WalkerMode::Fast => { if index.chain_walkable { if let Some(stats) = fp_chain::visit(index, visit) { return stats; } } - let mut stats = unwind::visit(&index.records, visit); + let mut stats = unwind::visit(index, visit); stats.fallback_walks = 1; stats } @@ -209,7 +276,7 @@ fn verify_visit( ); }; let mut unwind_addresses: Vec = Vec::new(); - let mut stats = unwind::visit(&index.records, &mut |slot: MutableRootSlot| { + let mut stats = unwind::visit(index, &mut |slot: MutableRootSlot| { unwind_addresses.push(slot.ptr as usize); visit(slot); }); @@ -228,8 +295,9 @@ fn verify_visit( stats } -fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { +fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option<(Vec, usize)> { let mut all = Vec::new(); + let mut dropped_registers = 0usize; let mut base = 0usize; while base < bytes.len() { // Linkers preserve the input section's 8-byte alignment. Ignore a @@ -237,17 +305,18 @@ fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { if bytes[base..].iter().all(|byte| *byte == 0) { break; } - let (mut records, consumed) = parse_one_stack_map(&bytes[base..])?; + let (mut records, consumed, dropped) = parse_one_stack_map(&bytes[base..])?; if consumed == 0 { return None; } all.append(&mut records); + dropped_registers = dropped_registers.saturating_add(dropped); base = base.checked_add(consumed)?; } - Some(all) + Some((all, dropped_registers)) } -fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { +fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize, usize)> { if read_u8(bytes, 0)? != STACK_MAP_VERSION { return None; } @@ -275,8 +344,10 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { } let mut out = Vec::with_capacity(record_count); + let mut dropped_registers = 0usize; for (function_address, function_stack_size, function_record_count) in functions { for _ in 0..function_record_count { + let record_id = read_u64(bytes, offset)?; let instruction_offset = read_u32(bytes, offset + 8)? as usize; let location_count = read_u16(bytes, offset + 14)? as usize; offset = offset.checked_add(16)?; @@ -287,6 +358,9 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { let size = read_u16(bytes, offset + 2)?; let dwarf_reg = read_u16(bytes, offset + 4)?; let location_offset = read_i32(bytes, offset + 8)?; + if kind == LOCATION_REGISTER && size == 8 { + dropped_registers = dropped_registers.saturating_add(1); + } if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { let location = StackMapLocation { dwarf_reg, @@ -319,12 +393,13 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { out.push(StackMapRecord { pc: function_address.checked_add(instruction_offset)?, + id: record_id, stack_size: function_stack_size, locations, }); } } - Some((out, offset)) + Some((out, offset, dropped_registers)) } fn align_up(value: usize, alignment: usize) -> Option { @@ -488,17 +563,17 @@ mod unwind { } struct WalkState<'a, F> { - maps: &'a [StackMapRecord], + index: &'a StackMapIndex, visit: &'a mut F, stats: NativeStackWalkStats, } pub(super) fn visit( - maps: &[StackMapRecord], + index: &StackMapIndex, visit: &mut F, ) -> NativeStackWalkStats { let mut state = WalkState { - maps, + index, visit, stats: NativeStackWalkStats { walks: 1, @@ -521,25 +596,12 @@ mod unwind { let state = &mut *argument.cast::>(); state.stats.frames_visited = state.stats.frames_visited.saturating_add(1); let ip = _Unwind_GetIP(context); - let Some(candidate_pc) = closest_record_pc(state.maps, ip) else { - return 0; - }; - let delta = ip.abs_diff(candidate_pc); - if delta > MAX_SAFEPOINT_RETURN_DELTA { + let matched = state.index.match_records(ip); + if matched.is_empty() { return 0; } - - let first = state - .maps - .partition_point(|record| record.pc < candidate_pc); - let last = state - .maps - .partition_point(|record| record.pc <= candidate_pc); - state.stats.records_matched = state - .stats - .records_matched - .saturating_add(last.saturating_sub(first)); - for record in &state.maps[first..last] { + state.stats.records_matched = state.stats.records_matched.saturating_add(matched.len()); + for record in matched { for location in &record.locations { state.stats.locations_visited = state.stats.locations_visited.saturating_add(1); let base = _Unwind_GetGR(context, i32::from(location.dwarf_reg)); @@ -569,7 +631,7 @@ mod unwind { use super::*; pub(super) fn visit( - _maps: &[StackMapRecord], + _index: &StackMapIndex, _visit: &mut impl FnMut(MutableRootSlot), ) -> NativeStackWalkStats { NativeStackWalkStats::default() @@ -627,7 +689,12 @@ mod fp_chain { ..NativeStackWalkStats::default() }; let low_pc = index.min_pc.saturating_sub(MAX_SAFEPOINT_RETURN_DELTA); - let high_pc = index.max_pc.saturating_add(MAX_SAFEPOINT_RETURN_DELTA); + // Compact builds: a frame's return address can sit anywhere inside + // the last real function's body, past its entry record — the + // sentinel PC is the true upper bound. + let high_pc = index + .compact_gen_end + .unwrap_or_else(|| index.max_pc.saturating_add(MAX_SAFEPOINT_RETURN_DELTA)); let mut fp = current_frame_pointer(); while fp != 0 { if fp & 0xF != 0 || fp.checked_add(16)? > top { @@ -640,24 +707,18 @@ mod fp_chain { break; } if return_address >= low_pc && return_address <= high_pc { - if let Some(candidate_pc) = closest_record_pc(&index.records, return_address) { - if return_address.abs_diff(candidate_pc) <= MAX_SAFEPOINT_RETURN_DELTA { + let matched = index.match_records(return_address); + { + if !matched.is_empty() { // The record describes the caller's frame; its // locations are relative to the caller's own x29, // which is exactly the saved word we just read. if caller_fp == 0 { return None; } - let first = index - .records - .partition_point(|record| record.pc < candidate_pc); - let last = index - .records - .partition_point(|record| record.pc <= candidate_pc); - stats.records_matched = stats - .records_matched - .saturating_add(last.saturating_sub(first)); - for record in &index.records[first..last] { + stats.records_matched = + stats.records_matched.saturating_add(matched.len()); + for record in matched { // LLVM's AArch64 frame keeps the [x29, x30] pair // at the top of the frame, so the caller's body // SP is its fp + 16 - stack_size. `chain_walkable` @@ -767,12 +828,14 @@ mod tests { #[test] fn parses_direct_mutable_frame_location() { let bytes = one_map(0x1000, 42, 0x10, -8); - let (records, consumed) = parse_one_stack_map(&bytes).expect("valid stack map"); + let (records, consumed, dropped) = parse_one_stack_map(&bytes).expect("valid stack map"); + assert_eq!(dropped, 0); assert_eq!(consumed, bytes.len()); assert_eq!( records, vec![StackMapRecord { pc: 0x1010, + id: 42, stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, @@ -786,7 +849,8 @@ mod tests { fn parses_linker_concatenated_input_sections() { let mut bytes = one_map(0x1000, 42, 0x10, -8); bytes.extend_from_slice(&one_map(0x2000, 43, 0x20, -16)); - let records = parse_concatenated_stack_maps(&bytes).expect("concatenated maps"); + let (records, dropped) = parse_concatenated_stack_maps(&bytes).expect("concatenated maps"); + assert_eq!(dropped, 0); assert_eq!(records.len(), 2); assert_eq!(records[0].pc, 0x1010); assert_eq!(records[1].pc, 0x2020); @@ -800,12 +864,14 @@ mod tests { 0x20, &[(LOCATION_INDIRECT, -16), (LOCATION_INDIRECT, -16)], ); - let (records, consumed) = parse_one_stack_map(&bytes).expect("valid statepoint map"); + let (records, consumed, dropped) = parse_one_stack_map(&bytes).expect("valid statepoint map"); + assert_eq!(dropped, 0); assert_eq!(consumed, bytes.len()); assert_eq!( records, vec![StackMapRecord { pc: 0x1020, + id: 7, stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, @@ -830,6 +896,7 @@ mod tests { fn chain_walkable_index_accepts_fp_and_sized_sp_locations_only() { let fp_record = StackMapRecord { pc: 0x1000, + id: 0, stack_size: 0, locations: vec![StackMapLocation { dwarf_reg: DWARF_REG_FP_AARCH64, @@ -838,6 +905,7 @@ mod tests { }; let sp_record = StackMapRecord { pc: 0x2000, + id: 0, stack_size: 160, locations: vec![StackMapLocation { dwarf_reg: DWARF_REG_SP_AARCH64, @@ -846,6 +914,7 @@ mod tests { }; let frameless_sp_record = StackMapRecord { pc: 0x3000, + id: 0, stack_size: 0, locations: vec![StackMapLocation { dwarf_reg: DWARF_REG_SP_AARCH64, @@ -854,6 +923,7 @@ mod tests { }; let other_reg_record = StackMapRecord { pc: 0x4000, + id: 0, stack_size: 160, locations: vec![StackMapLocation { dwarf_reg: 1, @@ -861,31 +931,77 @@ mod tests { }], }; - let walkable = index_records(vec![fp_record.clone(), sp_record.clone()]); + let walkable = index_records(vec![fp_record.clone(), sp_record.clone()], 0); assert!(walkable.chain_walkable); assert_eq!(walkable.min_pc, 0x1000); assert_eq!(walkable.max_pc, 0x2000); assert!( - !index_records(vec![fp_record.clone(), frameless_sp_record]).chain_walkable, + !index_records(vec![fp_record.clone(), frameless_sp_record], 0).chain_walkable, "an SP location without a usable frame size must disable the fast walk" ); assert!( - !index_records(vec![fp_record, other_reg_record]).chain_walkable, + !index_records(vec![fp_record, other_reg_record], 0).chain_walkable, "any non-FP/SP register must disable the fast walk" ); } + #[test] + fn compact_index_matches_by_function_region() { + let fp_loc = |offset| StackMapLocation { + dwarf_reg: DWARF_REG_FP_AARCH64, + offset, + }; + let f1 = StackMapRecord { + pc: 0x1000, + id: 0, + stack_size: 48, + locations: vec![fp_loc(-8)], + }; + let f2 = StackMapRecord { + pc: 0x2000, + id: 0, + stack_size: 32, + locations: vec![fp_loc(-16)], + }; + let sentinel = StackMapRecord { + pc: 0x3000, + id: COMPACT_SENTINEL_STACKMAP_ID, + stack_size: 16, + locations: Vec::new(), + }; + + let compact = index_records(vec![f1.clone(), f2, sentinel], 0); + assert_eq!(compact.compact_gen_end, Some(0x3000)); + // Deep inside a function body, far past any ±16 window. + assert_eq!(compact.match_records(0x1abc)[0].pc, 0x1000); + assert_eq!(compact.match_records(0x2ff8)[0].pc, 0x2000); + // Outside the generated region: below the first function, at or + // beyond the sentinel — foreign frames must never match. + assert!(compact.match_records(0x0fff).is_empty()); + assert!(compact.match_records(0x3000).is_empty()); + assert!(compact.match_records(0x9000).is_empty()); + + // Without the sentinel the classic ±16 per-safepoint rule holds and + // a mid-body address matches nothing. + let classic = index_records(vec![f1], 0); + assert!(classic.compact_gen_end.is_none()); + assert!(classic.match_records(0x1abc).is_empty()); + assert_eq!(classic.match_records(0x1008)[0].pc, 0x1000); + } + #[test] fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { let maps = vec![ StackMapRecord { pc: 0x1000, + id: 0, stack_size: 32, locations: Vec::new(), }, StackMapRecord { pc: 0x1020, + id: 0, stack_size: 32, locations: Vec::new(), }, diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 36f69b581d..2b97d7ea74 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -36,6 +36,7 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_STATEPOINTS", + "PERRY_COMPACT_ROOTS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 1db7bff07e..cdce03eabb 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -803,6 +803,10 @@ fn compute_object_cache_key_with_env( "env_statepoints", env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), ); + h.field( + "env_compact_roots", + env_var("PERRY_COMPACT_ROOTS").as_deref().unwrap_or(""), + ); // Explicit-safepoint contract: flips audited AllocNoReentry helpers // between statepoint and plain call. Two arms sharing a cached object // would make the contract's metadata reduction unmeasurable. diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index 93e44fd58f..deebe7bc93 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -584,6 +584,7 @@ fn key_changes_with_codegen_env_vars() { "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_STATEPOINTS", + "PERRY_COMPACT_ROOTS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 4ee2318cf1..8b3f4b6918 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -5885,6 +5885,37 @@ pub fn run_with_parse_cache( None }; + // Compact-roots sentinel (`PERRY_COMPACT_ROOTS=1`): a tiny object linked + // AFTER every generated object. Its stackmap record's PC is the exclusive + // upper bound of generated __text (ld64 lays same-section atoms out in + // input order), which is what makes the runtime's per-function region + // matching unable to attribute a runtime/foreign frame to the last + // generated function. Its magic record ID doubles as the runtime's + // "this is a compact build" signal. + if matches!( + std::env::var("PERRY_COMPACT_ROOTS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) { + let sentinel_id = perry_codegen::COMPACT_SENTINEL_STACKMAP_ID as i64; + let sentinel_ll = format!( + "module asm \".no_dead_strip __LLVM_StackMaps\"\n\n\ + define void @__perry_gen_end() \"frame-pointer\"=\"non-leaf\" {{\n\ + entry:\n\ + \x20 call void (i64, i32, ...) @llvm.experimental.stackmap(i64 {sentinel_id}, i32 0)\n\ + \x20 ret void\n\ + }}\n\n\ + declare void @llvm.experimental.stackmap(i64, i32, ...)\n\n\ + @llvm.used = appending global [1 x ptr] [ptr @__perry_gen_end], section \"llvm.metadata\"\n" + ); + let sentinel_bytes = + perry_codegen::linker::compile_ll_to_object(&sentinel_ll, target.as_deref())?; + let sentinel_path = object_output_dir.join("_perry_gen_end.o"); + fs::write(&sentinel_path, &sentinel_bytes)?; + obj_cleanup_paths.push(sentinel_path.clone()); + obj_paths.push(sentinel_path); + obj_fingerprints.push(None); + } + // Build & run the per-platform link command. Tier 2.1 final extraction // (v0.5.342) — see crates/perry/src/commands/compile/link.rs. let link_cache_status = build_and_run_link( From c98b95606067a0ef335d57810a534d9359817b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 09:40:52 +0200 Subject: [PATCH 15/53] research(gc): delete the compact per-function mode - measured negative result The per-function metadata thesis was built (bd066d62b), measured (424-680 B vs 5.3-8.9 KB per probe, 10-13x), and disproven: a ten-line churn loop deterministically corrupts under moving minors. The forensic chain - retention clears, callee-saved clobbers, dead-slot zeroing, and finally disabling walker visits entirely, all bit-identical failures - proves the corruption vector is not the metadata machinery at all: the mutator reads from-space through stale heap-derived values in optimized SSA, which only relocation semantics can restore (the same module carries 79 gc.relocate under the statepoint backend). Barriers constrain memory ordering, not dataflow. Design law recorded in the doc: with an optimizing compiler between source and safepoint, root metadata without relocation semantics is unsound - per-call plain maps merely made the window small enough for probes to pass; per-function maps made it wide enough to fail in ten lines. The compact 10-13x is only reachable via RS4GC-style managed SSA or repsel shrinking the recorded set. Kept from the detour (mode-independent): the match_records refactor in the walker, the copy-minor diag line (trigger kind + declared-safepoint flag), and GcTriggerKind's Debug derive. --- crates/perry-codegen/src/codegen/helpers.rs | 18 +- crates/perry-codegen/src/function.rs | 173 +--------------- crates/perry-codegen/src/lib.rs | 1 - crates/perry-runtime/src/gc/copying.rs | 6 +- crates/perry-runtime/src/gc/policy.rs | 2 +- .../perry-runtime/src/gc/roots/stack_maps.rs | 184 ++++-------------- .../perry/src/commands/compile/build_cache.rs | 1 - .../src/commands/compile/object_cache.rs | 4 - .../object_cache/object_cache_tests.rs | 1 - .../src/commands/compile/run_pipeline.rs | 31 --- docs/statepoint-gc-experiment.md | 53 ++++- 11 files changed, 93 insertions(+), 381 deletions(-) diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index 7e95ec52c6..bd06b3739f 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -94,26 +94,10 @@ pub(crate) fn statepoints_enabled() -> bool { ) } -/// `PERRY_COMPACT_ROOTS=1` — compact per-function native-root metadata -/// (research). One entry stackmap per generated function records every root -/// alloca as a stable Direct location; calls carry only memory barriers. -/// See `PreciseRootBackend::CompactEntry`. Takes precedence over -/// `PERRY_STATEPOINTS` when both are set. -pub(crate) fn compact_roots_enabled() -> bool { - use std::sync::OnceLock; - static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(|| { - matches!( - std::env::var("PERRY_COMPACT_ROOTS").as_deref(), - Ok("1") | Ok("on") | Ok("true") - ) - }) -} - /// Whether precise roots should use a native-stack metadata backend rather /// than Perry's heap-backed shadow frame. pub(crate) fn native_stack_roots_enabled() -> bool { - statepoints_enabled() || compact_roots_enabled() + statepoints_enabled() } /// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 7dadf57149..635958e4ec 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -758,19 +758,7 @@ impl LlFunction { // native-frame stack maps only after lowering is complete, when every // lazily-reserved scalar root and every call site is visible. // - // Compact mode lowers EVERY generated function — rootless ones get a - // zero-operand entry record so the runtime's region matching can - // never attribute their frames to a neighboring rooted function. It - // also has no has_try exclusion: with no per-call rewriting there is - // nothing to conflict with the setjmp lowering. - let ir = if crate::codegen::helpers::compact_roots_enabled() { - lower_precise_roots_to_native_stack( - &ir, - &self.name, - self.stack_map_slot_count, - PreciseRootBackend::CompactEntry, - ) - } else if self.stack_map_requested { + let ir = if self.stack_map_requested { let backend = if crate::codegen::helpers::statepoints_enabled() && !self.has_try { PreciseRootBackend::Statepoint } else { @@ -968,20 +956,8 @@ fn stack_map_active_slots( enum PreciseRootBackend { StackMap, Statepoint, - /// Compact per-function mode (`PERRY_COMPACT_ROOTS=1`): ONE stackmap - /// intrinsic in the entry block records every root alloca as a stable - /// Direct FP-relative location; calls carry only zero-instruction memory - /// barriers (store-before / reload-after), no per-safepoint metadata. - /// Precision drops from per-safepoint to per-function — sound because - /// every root alloca is zero-initialized at entry, so visiting a stale - /// slot can only over-retain, never corrupt. Metadata cost falls from - /// ~64 B/safepoint + 24 B/root-pair to ~40 B/function + 12 B/slot. - /// - /// The walker matches frames by function REGION: every generated - /// function (rooted or not) carries an entry record, and a sentinel - /// `__perry_gen_end` object linked last bounds the region, so a foreign - /// frame can never false-match a generated record. - CompactEntry, + + } impl PreciseRootBackend { @@ -989,25 +965,10 @@ impl PreciseRootBackend { match self { Self::StackMap => "stack-map", Self::Statepoint => "statepoint", - Self::CompactEntry => "compact-entry", } } } -/// Record ID of the `__perry_gen_end` sentinel. Its presence in the parsed -/// section is what flips the runtime walker into compact region matching, -/// and its PC is the exclusive upper bound of generated code. -pub const COMPACT_SENTINEL_STACKMAP_ID: u64 = 0x9E44_C0DE_0DEC_1DED; - -/// A textual-IR basic-block label line (`entry.0:` — one token, ends with a -/// colon, not a comment). -fn is_block_label(line: &str) -> bool { - let trimmed = line.trim(); - !trimmed.is_empty() - && trimmed.ends_with(':') - && !trimmed.contains(' ') - && !trimmed.starts_with(';') -} #[derive(Debug, Eq, PartialEq)] struct DirectCall<'a> { @@ -1271,23 +1232,11 @@ fn lower_precise_roots_to_native_stack( ) }); if root_ptrs.is_empty() { - let mut out = String::with_capacity(ir.len() + 96); - let mut entry_record_emitted = backend != PreciseRootBackend::CompactEntry; - for line in ir.lines() { - if parse_shadow_bind(line).is_some() || parse_shadow_set(line).is_some() { - continue; - } - out.push_str(line); - out.push('\n'); - // Compact mode: a rootless function still needs its entry record - // as a region boundary for the runtime's frame matching. - if !entry_record_emitted && is_block_label(line) { - out.push_str( - " call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0)\n", - ); - entry_record_emitted = true; - } - } + let out = ir + .lines() + .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) + .map(|line| format!("{line}\n")) + .collect(); if let Some(report) = report { crate::statepoint_report::record(report); } @@ -1298,25 +1247,8 @@ fn lower_precise_roots_to_native_stack( let mut available = std::collections::HashSet::::new(); let mut initialized = std::collections::HashSet::::new(); let mut map_id = 0u64; - let mut compact_entry_emitted = backend != PreciseRootBackend::CompactEntry; - let mut labels_seen = 0usize; for (line_idx, line) in lines.iter().enumerate() { - // Compact mode requires every root alloca to be recordable from the - // entry block. A block-local slot (rare scalar-replacement shapes) - // would leave the entry record incomplete, so the whole function - // falls back to the per-safepoint statepoint backend instead. - if !compact_entry_emitted && is_block_label(line) { - labels_seen += 1; - if labels_seen >= 2 { - return lower_precise_roots_to_native_stack( - ir, - function_name, - slot_count, - PreciseRootBackend::Statepoint, - ); - } - } if parse_shadow_bind(line).is_some() { // Compile-time marker only. The real slot is already populated by // the local store immediately preceding this old bind. @@ -1356,19 +1288,6 @@ fn lower_precise_roots_to_native_stack( } } - // Compact mode: the moment every root alloca exists (still inside - // the entry block, or the label check above would have bailed), emit - // the function's single stackmap with the whole slot set. Escaping - // every root address through the intrinsic also pins the allocas as - // address-taken for the rest of the pipeline. - if !compact_entry_emitted && initialized.len() == root_ptrs.len() { - let operands: Vec = root_ptrs.iter().map(|p| format!("ptr {p}")).collect(); - out.push_str(&format!( - " call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0, {})\n", - operands.join(", ") - )); - compact_entry_emitted = true; - } // Insert before calls, not after. Rebuild the tail when the line just // appended is a call so the intrinsic's instruction offset is the @@ -1432,17 +1351,6 @@ fn lower_precise_roots_to_native_stack( // Move the call line behind the intrinsic. let call_len = line.len() + 1; out.truncate(out.len() - call_len); - if backend == PreciseRootBackend::CompactEntry { - // No per-safepoint metadata. The pre-call barrier forces live - // root stores to be materialized before the callee can collect; - // the post-call barrier forces reloads of anything the collector - // may have rewritten in the entry-recorded slots. - out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); - out.push_str(line); - out.push('\n'); - out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); - continue; - } if backend == PreciseRootBackend::Statepoint { if let Some(call) = parse_direct_statepoint_call(line) { emit_statepoint(&mut out, &call, &live, map_id); @@ -1484,71 +1392,6 @@ mod stack_map_tests { lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::Statepoint) } - fn lower_compact(input: &str, slots: u32) -> String { - lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::CompactEntry) - } - - #[test] - fn compact_emits_one_entry_record_and_barrier_wrapped_calls() { - let input = r#"define i64 @probe(i64 %arg) { -entry.0: - %r0 = alloca i64 - store i64 %arg, ptr %r0 - call void @js_shadow_slot_bind(i32 0, ptr %r0) - %r1 = call i64 @may_collect() - call void @may_collect_again() - ret i64 %r1 -} -"#; - let output = lower_compact(input, 1); - // Exactly one stackmap, in the entry block, carrying the whole slot set. - assert_eq!(output.matches("@llvm.experimental.stackmap").count(), 1); - assert!(output.contains("stackmap(i64 0, i32 0, ptr %r0)")); - let map_at = output.find("@llvm.experimental.stackmap").unwrap(); - let first_call = output.find("@may_collect()").unwrap(); - assert!(map_at < first_call, "entry record must precede the first call"); - // Calls carry pre+post barriers and no metadata. - assert!(output.contains( - "call void asm sideeffect \"\", \"~{memory}\"()\n %r1 = call i64 \ - @may_collect()\n call void asm sideeffect \"\", \"~{memory}\"()" - )); - assert!(!output.contains("gc.statepoint")); - } - - #[test] - fn compact_rootless_function_still_gets_a_region_record() { - let input = r#"define void @probe() { -entry.0: - call void @leaf() - ret void -} -"#; - let output = lower_compact(input, 0); - assert_eq!(output.matches("@llvm.experimental.stackmap").count(), 1); - assert!(output.contains("stackmap(i64 0, i32 0)\n call void @leaf()")); - } - - #[test] - fn compact_falls_back_to_statepoints_for_block_local_slots() { - let input = r#"define void @probe(i1 %cond) { -entry.0: - br i1 %cond, label %then.1, label %exit.2 -then.1: - %r0 = alloca i64 - store i64 5, ptr %r0 - call void @js_shadow_slot_bind(i32 0, ptr %r0) - call void @may_collect() - br label %exit.2 -exit.2: - ret void -} -"#; - let output = lower_compact(input, 1); - // Block-local root alloca: the compact entry record cannot cover it, - // so the function must take the per-safepoint statepoint backend. - assert!(output.contains("gc.statepoint")); - } - #[test] fn lowers_bind_and_liveness_clear_to_native_frame_maps() { let input = r#"define i64 @probe(i64 %arg) { diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index 17e0b4387f..45a9a626e9 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -11,7 +11,6 @@ pub(crate) mod collectors; pub mod expr; pub mod ext_registry; pub mod function; -pub use function::COMPACT_SENTINEL_STACKMAP_ID; pub(crate) mod gc_call_effects; pub mod linker; pub(crate) mod loop_purity; diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index f0e055a6d0..96a6d1839c 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1245,12 +1245,14 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( maybe_schedule_old_reclaim_after_copied_minor(); if std::env::var_os("PERRY_GC_DIAG").is_some() { eprintln!( - "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={}", + "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} trigger={:?} declared_safepoint={}", collector.stats.copied_objects, collector.stats.copied_bytes, collector.stats.promoted_objects, collector.stats.promoted_bytes, - freed_bytes + freed_bytes, + _trigger_kind, + super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) ); } Some(CopiedMinorFastPathOutcome { diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 6ce8056131..39f63d0e6b 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -536,7 +536,7 @@ impl GcCollectionKind { } } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub(super) enum GcTriggerKind { ArenaBytes, MallocCount, diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 5630f87802..1b8df6921c 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -19,15 +19,9 @@ use std::ffi::c_void; use std::sync::OnceLock; const STACK_MAP_VERSION: u8 = 3; -const LOCATION_REGISTER: u8 = 1; const LOCATION_DIRECT: u8 = 2; const LOCATION_INDIRECT: u8 = 3; const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; -/// Mirrors `perry_codegen::function::COMPACT_SENTINEL_STACKMAP_ID`: the -/// record ID of the `__perry_gen_end` sentinel that a compact-roots build -/// links after every generated object. Its presence flips region matching -/// on, and its PC is the exclusive upper bound of generated code. -const COMPACT_SENTINEL_STACKMAP_ID: u64 = 0x9E44_C0DE_0DEC_1DED; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct StackMapLocation { dwarf_reg: u16, @@ -37,7 +31,6 @@ struct StackMapLocation { #[derive(Clone, Debug, Eq, PartialEq)] struct StackMapRecord { pc: usize, - id: u64, /// The containing function's total frame size from the stack-map header. /// LLVM's AArch64 frame places the `[x29, x30]` pair at the top of the /// frame, so a chain walker can reconstruct the body SP as @@ -59,18 +52,9 @@ struct StackMapIndex { chain_walkable: bool, min_pc: usize, max_pc: usize, - /// `Some(gen_end_pc)` when the `__perry_gen_end` sentinel is present: - /// this is a compact-roots build, frames match by function region - /// (greatest record PC at or below the return address, bounded by - /// `gen_end_pc`) instead of the ±16-byte per-safepoint heuristic. - compact_gen_end: Option, - /// Locations the parser had to drop because LLVM recorded them in a - /// register. Harmlessly redundant in per-safepoint modes (statepoint - /// spills are memory); fatal in compact mode, where the entry record is - /// the only description of the frame — checked at init. - dropped_register_locations: usize, } + static STACK_MAPS: OnceLock = OnceLock::new(); const DWARF_REG_FP_AARCH64: u16 = 29; @@ -141,25 +125,13 @@ fn stack_maps() -> &'static StackMapIndex { let Some(section) = loaded_stack_map_section() else { return StackMapIndex::default(); }; - let (mut records, dropped) = parse_concatenated_stack_maps(section).unwrap_or_default(); + let mut records = parse_concatenated_stack_maps(section).unwrap_or_default(); records.sort_unstable_by_key(|record| record.pc); - let index = index_records(records, dropped); - if index.compact_gen_end.is_some() && index.dropped_register_locations != 0 { - // A compact build's entry record is the ONLY description of its - // frame. A register-recorded root would be silently invisible to - // the collector, so this configuration must not run at all. - panic!( - "PERRY_COMPACT_ROOTS: {} register-recorded root location(s) \ - in a compact-roots image — refusing to run with invisible \ - roots", - index.dropped_register_locations - ); - } - index + index_records(records) }) } -fn index_records(records: Vec, dropped_register_locations: usize) -> StackMapIndex { +fn index_records(records: Vec) -> StackMapIndex { let chain_walkable = records.iter().all(|record| { record.locations.iter().all(|location| { location.dwarf_reg == DWARF_REG_FP_AARCH64 @@ -168,49 +140,11 @@ fn index_records(records: Vec, dropped_register_locations: usize }); let min_pc = records.first().map_or(usize::MAX, |record| record.pc); let max_pc = records.last().map_or(0, |record| record.pc); - let compact_gen_end = records - .iter() - .find(|record| record.id == COMPACT_SENTINEL_STACKMAP_ID) - .map(|record| record.pc); StackMapIndex { records, chain_walkable, min_pc, max_pc, - compact_gen_end, - dropped_register_locations, - } -} - -impl StackMapIndex { - /// The records describing the frame whose return address is `ip`. - /// - /// Compact builds match by function region: the greatest record PC at or - /// below `ip`, valid only inside `[first record, __perry_gen_end)` — a - /// foreign (runtime) frame can never match because generated code is - /// linked contiguously ahead of the sentinel. Per-safepoint builds keep - /// the ±16-byte nearest-PC match, which can select several records at - /// one PC. - fn match_records(&self, ip: usize) -> &[StackMapRecord] { - if let Some(gen_end) = self.compact_gen_end { - if ip < self.min_pc || ip >= gen_end { - return &[]; - } - let idx = self.records.partition_point(|record| record.pc <= ip); - let Some(idx) = idx.checked_sub(1) else { - return &[]; - }; - return &self.records[idx..idx + 1]; - } - let Some(candidate_pc) = closest_record_pc(&self.records, ip) else { - return &[]; - }; - if ip.abs_diff(candidate_pc) > MAX_SAFEPOINT_RETURN_DELTA { - return &[]; - } - let first = self.records.partition_point(|record| record.pc < candidate_pc); - let last = self.records.partition_point(|record| record.pc <= candidate_pc); - &self.records[first..last] } } @@ -233,6 +167,24 @@ fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { } } +impl StackMapIndex { + /// The records describing the frame whose return address is `ip`: the + /// ±16-byte nearest-PC match, which can select several records at one PC + /// (plain maps sit just before the call, statepoints exactly at the + /// return address). + fn match_records(&self, ip: usize) -> &[StackMapRecord] { + let Some(candidate_pc) = closest_record_pc(&self.records, ip) else { + return &[]; + }; + if ip.abs_diff(candidate_pc) > MAX_SAFEPOINT_RETURN_DELTA { + return &[]; + } + let first = self.records.partition_point(|record| record.pc < candidate_pc); + let last = self.records.partition_point(|record| record.pc <= candidate_pc); + &self.records[first..last] + } +} + pub(super) fn visit_stack_map_root_slots( visit: &mut impl FnMut(MutableRootSlot), ) -> NativeStackWalkStats { @@ -295,9 +247,8 @@ fn verify_visit( stats } -fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option<(Vec, usize)> { +fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { let mut all = Vec::new(); - let mut dropped_registers = 0usize; let mut base = 0usize; while base < bytes.len() { // Linkers preserve the input section's 8-byte alignment. Ignore a @@ -305,18 +256,17 @@ fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option<(Vec, u if bytes[base..].iter().all(|byte| *byte == 0) { break; } - let (mut records, consumed, dropped) = parse_one_stack_map(&bytes[base..])?; + let (mut records, consumed) = parse_one_stack_map(&bytes[base..])?; if consumed == 0 { return None; } all.append(&mut records); - dropped_registers = dropped_registers.saturating_add(dropped); base = base.checked_add(consumed)?; } - Some((all, dropped_registers)) + Some(all) } -fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize, usize)> { +fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { if read_u8(bytes, 0)? != STACK_MAP_VERSION { return None; } @@ -344,10 +294,8 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize, usiz } let mut out = Vec::with_capacity(record_count); - let mut dropped_registers = 0usize; for (function_address, function_stack_size, function_record_count) in functions { for _ in 0..function_record_count { - let record_id = read_u64(bytes, offset)?; let instruction_offset = read_u32(bytes, offset + 8)? as usize; let location_count = read_u16(bytes, offset + 14)? as usize; offset = offset.checked_add(16)?; @@ -358,9 +306,6 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize, usiz let size = read_u16(bytes, offset + 2)?; let dwarf_reg = read_u16(bytes, offset + 4)?; let location_offset = read_i32(bytes, offset + 8)?; - if kind == LOCATION_REGISTER && size == 8 { - dropped_registers = dropped_registers.saturating_add(1); - } if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { let location = StackMapLocation { dwarf_reg, @@ -393,13 +338,12 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize, usiz out.push(StackMapRecord { pc: function_address.checked_add(instruction_offset)?, - id: record_id, stack_size: function_stack_size, locations, }); } } - Some((out, offset, dropped_registers)) + Some((out, offset)) } fn align_up(value: usize, alignment: usize) -> Option { @@ -689,12 +633,7 @@ mod fp_chain { ..NativeStackWalkStats::default() }; let low_pc = index.min_pc.saturating_sub(MAX_SAFEPOINT_RETURN_DELTA); - // Compact builds: a frame's return address can sit anywhere inside - // the last real function's body, past its entry record — the - // sentinel PC is the true upper bound. - let high_pc = index - .compact_gen_end - .unwrap_or_else(|| index.max_pc.saturating_add(MAX_SAFEPOINT_RETURN_DELTA)); + let high_pc = index.max_pc.saturating_add(MAX_SAFEPOINT_RETURN_DELTA); let mut fp = current_frame_pointer(); while fp != 0 { if fp & 0xF != 0 || fp.checked_add(16)? > top { @@ -828,14 +767,12 @@ mod tests { #[test] fn parses_direct_mutable_frame_location() { let bytes = one_map(0x1000, 42, 0x10, -8); - let (records, consumed, dropped) = parse_one_stack_map(&bytes).expect("valid stack map"); - assert_eq!(dropped, 0); + let (records, consumed) = parse_one_stack_map(&bytes).expect("valid stack map"); assert_eq!(consumed, bytes.len()); assert_eq!( records, vec![StackMapRecord { pc: 0x1010, - id: 42, stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, @@ -849,8 +786,7 @@ mod tests { fn parses_linker_concatenated_input_sections() { let mut bytes = one_map(0x1000, 42, 0x10, -8); bytes.extend_from_slice(&one_map(0x2000, 43, 0x20, -16)); - let (records, dropped) = parse_concatenated_stack_maps(&bytes).expect("concatenated maps"); - assert_eq!(dropped, 0); + let records = parse_concatenated_stack_maps(&bytes).expect("concatenated maps"); assert_eq!(records.len(), 2); assert_eq!(records[0].pc, 0x1010); assert_eq!(records[1].pc, 0x2020); @@ -864,14 +800,12 @@ mod tests { 0x20, &[(LOCATION_INDIRECT, -16), (LOCATION_INDIRECT, -16)], ); - let (records, consumed, dropped) = parse_one_stack_map(&bytes).expect("valid statepoint map"); - assert_eq!(dropped, 0); + let (records, consumed) = parse_one_stack_map(&bytes).expect("valid statepoint map"); assert_eq!(consumed, bytes.len()); assert_eq!( records, vec![StackMapRecord { pc: 0x1020, - id: 7, stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, @@ -896,7 +830,6 @@ mod tests { fn chain_walkable_index_accepts_fp_and_sized_sp_locations_only() { let fp_record = StackMapRecord { pc: 0x1000, - id: 0, stack_size: 0, locations: vec![StackMapLocation { dwarf_reg: DWARF_REG_FP_AARCH64, @@ -905,7 +838,6 @@ mod tests { }; let sp_record = StackMapRecord { pc: 0x2000, - id: 0, stack_size: 160, locations: vec![StackMapLocation { dwarf_reg: DWARF_REG_SP_AARCH64, @@ -914,7 +846,6 @@ mod tests { }; let frameless_sp_record = StackMapRecord { pc: 0x3000, - id: 0, stack_size: 0, locations: vec![StackMapLocation { dwarf_reg: DWARF_REG_SP_AARCH64, @@ -923,7 +854,6 @@ mod tests { }; let other_reg_record = StackMapRecord { pc: 0x4000, - id: 0, stack_size: 160, locations: vec![StackMapLocation { dwarf_reg: 1, @@ -931,77 +861,31 @@ mod tests { }], }; - let walkable = index_records(vec![fp_record.clone(), sp_record.clone()], 0); + let walkable = index_records(vec![fp_record.clone(), sp_record.clone()]); assert!(walkable.chain_walkable); assert_eq!(walkable.min_pc, 0x1000); assert_eq!(walkable.max_pc, 0x2000); assert!( - !index_records(vec![fp_record.clone(), frameless_sp_record], 0).chain_walkable, + !index_records(vec![fp_record.clone(), frameless_sp_record]).chain_walkable, "an SP location without a usable frame size must disable the fast walk" ); assert!( - !index_records(vec![fp_record, other_reg_record], 0).chain_walkable, + !index_records(vec![fp_record, other_reg_record]).chain_walkable, "any non-FP/SP register must disable the fast walk" ); } - #[test] - fn compact_index_matches_by_function_region() { - let fp_loc = |offset| StackMapLocation { - dwarf_reg: DWARF_REG_FP_AARCH64, - offset, - }; - let f1 = StackMapRecord { - pc: 0x1000, - id: 0, - stack_size: 48, - locations: vec![fp_loc(-8)], - }; - let f2 = StackMapRecord { - pc: 0x2000, - id: 0, - stack_size: 32, - locations: vec![fp_loc(-16)], - }; - let sentinel = StackMapRecord { - pc: 0x3000, - id: COMPACT_SENTINEL_STACKMAP_ID, - stack_size: 16, - locations: Vec::new(), - }; - - let compact = index_records(vec![f1.clone(), f2, sentinel], 0); - assert_eq!(compact.compact_gen_end, Some(0x3000)); - // Deep inside a function body, far past any ±16 window. - assert_eq!(compact.match_records(0x1abc)[0].pc, 0x1000); - assert_eq!(compact.match_records(0x2ff8)[0].pc, 0x2000); - // Outside the generated region: below the first function, at or - // beyond the sentinel — foreign frames must never match. - assert!(compact.match_records(0x0fff).is_empty()); - assert!(compact.match_records(0x3000).is_empty()); - assert!(compact.match_records(0x9000).is_empty()); - - // Without the sentinel the classic ±16 per-safepoint rule holds and - // a mid-body address matches nothing. - let classic = index_records(vec![f1], 0); - assert!(classic.compact_gen_end.is_none()); - assert!(classic.match_records(0x1abc).is_empty()); - assert_eq!(classic.match_records(0x1008)[0].pc, 0x1000); - } - #[test] fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { let maps = vec![ StackMapRecord { pc: 0x1000, - id: 0, stack_size: 32, locations: Vec::new(), }, StackMapRecord { pc: 0x1020, - id: 0, stack_size: 32, locations: Vec::new(), }, diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 2b97d7ea74..36f69b581d 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -36,7 +36,6 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_STATEPOINTS", - "PERRY_COMPACT_ROOTS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index cdce03eabb..1db7bff07e 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -803,10 +803,6 @@ fn compute_object_cache_key_with_env( "env_statepoints", env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), ); - h.field( - "env_compact_roots", - env_var("PERRY_COMPACT_ROOTS").as_deref().unwrap_or(""), - ); // Explicit-safepoint contract: flips audited AllocNoReentry helpers // between statepoint and plain call. Two arms sharing a cached object // would make the contract's metadata reduction unmeasurable. diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index deebe7bc93..93e44fd58f 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -584,7 +584,6 @@ fn key_changes_with_codegen_env_vars() { "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_STATEPOINTS", - "PERRY_COMPACT_ROOTS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 8b3f4b6918..4ee2318cf1 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -5885,37 +5885,6 @@ pub fn run_with_parse_cache( None }; - // Compact-roots sentinel (`PERRY_COMPACT_ROOTS=1`): a tiny object linked - // AFTER every generated object. Its stackmap record's PC is the exclusive - // upper bound of generated __text (ld64 lays same-section atoms out in - // input order), which is what makes the runtime's per-function region - // matching unable to attribute a runtime/foreign frame to the last - // generated function. Its magic record ID doubles as the runtime's - // "this is a compact build" signal. - if matches!( - std::env::var("PERRY_COMPACT_ROOTS").as_deref(), - Ok("1") | Ok("on") | Ok("true") - ) { - let sentinel_id = perry_codegen::COMPACT_SENTINEL_STACKMAP_ID as i64; - let sentinel_ll = format!( - "module asm \".no_dead_strip __LLVM_StackMaps\"\n\n\ - define void @__perry_gen_end() \"frame-pointer\"=\"non-leaf\" {{\n\ - entry:\n\ - \x20 call void (i64, i32, ...) @llvm.experimental.stackmap(i64 {sentinel_id}, i32 0)\n\ - \x20 ret void\n\ - }}\n\n\ - declare void @llvm.experimental.stackmap(i64, i32, ...)\n\n\ - @llvm.used = appending global [1 x ptr] [ptr @__perry_gen_end], section \"llvm.metadata\"\n" - ); - let sentinel_bytes = - perry_codegen::linker::compile_ll_to_object(&sentinel_ll, target.as_deref())?; - let sentinel_path = object_output_dir.join("_perry_gen_end.o"); - fs::write(&sentinel_path, &sentinel_bytes)?; - obj_cleanup_paths.push(sentinel_path.clone()); - obj_paths.push(sentinel_path); - obj_fingerprints.push(None); - } - // Build & run the per-platform link command. Tier 2.1 final extraction // (v0.5.342) — see crates/perry/src/commands/compile/link.rs. let link_cache_status = build_and_run_link( diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 8edc02b042..aff8719b59 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -322,14 +322,51 @@ evacuation + walker-verify, strict-enforcement gate fires, RSS flat): Metadata trajectory on `batch.ts` statepoints, all without any representation-selection improvement: 442 (first prototype) → 217 (call-effect audit) → 198 (noreturn elision) → **181 under the contract — -−59% total**. Each remaining big step is identified: the property-access -diamonds (~85 sites) fall to repsel `Ptr`, and the v3 format itself -wastes ~36 B/record on constant locations plus ~12 B/root on base/derived -duplication that a Perry-owned compact section could reclaim — but the -compact-section design needs either post-link fixup surgery or a -per-function (not per-safepoint) precision model, both of which are real -projects with open soundness questions, recorded here so the next session -starts from the design constraints rather than rediscovering them. +−59% total**. The remaining big step is the property-access diamonds +(~85 sites), which fall to repsel `Ptr`. + +## The compact per-function experiment — a measured NEGATIVE result + +The per-function precision model was then built and disproven +(implementation at `bd066d62b`, deleted from the tip afterward — an unsound +mode must not survive as a configuration a future bisect will trust). + +**The thesis**: one entry stackmap per generated function recording every +root alloca as a stable Direct location; calls carry only memory barriers; +a `__perry_gen_end` sentinel object linked last bounds the generated region +so the runtime can match frames by region instead of per-safepoint PCs. +**The size result was real**: 424–680 B of metadata per probe binary versus +5.3–8.9 KB for statepoints — 10–13× — with `__text` mostly smaller too. + +**The correctness result kills it.** A ten-line churn loop +(object escapes into a ring, two field reads) deterministically computes +wrong values. The forensics chain, recorded because each step eliminated a +plausible-but-wrong theory: retention-clear lowering (no effect), +callee-saved register clobbers at barriers (no effect, bit-identical +failure), dead-slot zeroing before every GC-capable call (no effect, +bit-identical), and finally disabling the walker's visits entirely — +**still bit-identical corruption**, proving the stack-map machinery was +never the vector. The corrupted fields contain forwarding-stub and +header-age-bit patterns: the mutator reads from-space through a stale +pointer that lives in optimized SSA, not in any root slot. The same module +compiles to **79 `gc.relocate`s** under the statepoint backend — each one a +place where LLVM held a heap-derived value whose post-collection identity +only relocation semantics can restore. `asm "~{memory}"` constrains memory +ordering, not dataflow; no barrier discipline reaches values the optimizer +carries in registers and rematerializes. + +**Conclusion, stated as the design law this branch keeps re-deriving:** +*with an optimizing compiler between the source and the safepoint, root +metadata without relocation semantics is unsound — per-call plain maps +merely made the window small enough for probes to pass, and per-function +compact maps made it wide enough to fail in ten lines.* This upgrades +#7108's argument ("only statepoint describes the frame during the call") +from analysis to demonstration, and it means the metadata floor for a +sound non-statepoint scheme does not exist: the choice is statepoint-style +relocation (per-safepoint records, ~2× plain maps, the measured −59% +trajectory) or the shadow stack. The compact 10–13× is only reachable via +`RewriteStatepointsForGC`-style managed-pointer SSA — the toolchain +decision #7108 costed — or repsel shrinking the recorded set. ## Which statepoint design this tests From 5fa1a06b5d75d2b526d32d60d7c4378b57e7d841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 09:52:36 +0200 Subject: [PATCH 16/53] docs: real-app remeasurement - metadata 3.83MB (below model floor), text recovery 150KB not 439KB, shadow is the measured three-axis optimum today --- docs/statepoint-gc-experiment.md | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index aff8719b59..e5d39d9fb3 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -355,6 +355,39 @@ only relocation semantics can restore. `asm "~{memory}"` constrains memory ordering, not dataflow; no barrier discipline reaches values the optimizer carries in registers and rematerializes. +## Real-app remeasurement (test-drizzle-pg, 133 modules, 2026-08-01) + +#7108's size model, re-taken as a direct measurement on the same +application with every in-branch reduction live: + +| Arm | file | `__text` | `__llvm_stackmaps` | +|---|---:|---:|---:| +| shadow (default) | 28,474,576 | 20,376,748 | 0 | +| statepoint | 32,206,720 | 20,227,252 | 4,025,336 | +| statepoint + contract | 32,008,560 | 20,226,728 | **3,832,384** | + +Two model corrections, one in each direction. The metadata came in at +**3.83 MB — below the refined model band's 4.5 MB floor** (the audit, +noreturn elision, and contract compose better on real dependency code than +the all-roots-live worst case assumed). But the text actually recovered is +**150 KB, not the 439 KB** #7108 reported — that figure measured +`PERRY_SHADOW_STACK=0` (rooting fully off) as the floor, while real +statepoint codegen keeps spill/reload work. Net file-size cost of the best +native arm on a real app: **+3.53 MB (+12.4%) versus shadow — a ~25× +imbalance that no audited elision closes.** The contract's real-app effect +is −4.8% metadata (probe-scale was −8.8%; dependency code has +proportionally fewer audited-helper sites). + +**Standing verdict, now measured on every axis:** the shadow stack is the +three-axis optimum shipping today — wall-clock tied within timer +quantization, RSS tied, file-size won by 3.5 MB on a real application. +The statepoint backend is correctness-superior (the forgot-to-root class +is structurally impossible), speed-competitive, and 59% leaner in metadata +than its own first prototype — and its remaining 25× size gap is proven +(not projected) to close only through repsel promotion shrinking the +maybe-pointer root set, or RS4GC managed-pointer SSA. Both are tracked; +neither is this branch's to deliver. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From 03d0763b0033a5253dc4a5c07e8d476e4c4e7423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 09:56:01 +0200 Subject: [PATCH 17/53] docs: shadow-frame elision census - 7.7% of framed functions, 4.0% of shadow traffic; measured-and-not-pursued --- docs/statepoint-gc-experiment.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index e5d39d9fb3..2e038f56ee 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -388,6 +388,22 @@ than its own first prototype — and its remaining 25× size gap is proven maybe-pointer root set, or RS4GC managed-pointer SSA. Both are tracked; neither is this branch's to deliver. +### The transfer question, also measured: can the audit shrink the SHADOW stack? + +The audited call-effect facts apply to shadow bookkeeping in principle — a +function whose every call is provably non-collecting needs no shadow frame +at all, by the same soundness argument the statepoint elisions passed gates +with. Census on the real app's traced IR (instrument validated against +#7108's function totals): **118 of 1,535 shadow-framed functions qualify +(7.7%), covering only 4.0% of shadow-op IR lines.** The elidable functions +are small leaves; the cost lives in large functions with genuine collecting +calls. Whole-frame elision would recover well under 1% of generated text — +recorded here as measured-and-not-pursued (finer per-region elision is +complexity the win does not justify). The shadow stack's remaining text +cost is, as the repsel campaign already measured from the other side, +bookkeeping for values that cannot yet be proven non-pointers — one more +place every road converges on representation selection. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From b905b703c7981a7b3bb84560ef9b50c6a754ae62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:01:09 +0200 Subject: [PATCH 18/53] research(gc): second AllocNoReentry audit round - four admitted, two excluded with transitive-reentry evidence Admitted: js_ctor_return_override (inspects the returned value, calls nothing), js_array_indexOf_jsvalue (strict equality never runs user code), js_validate_array_comparator / js_validate_array_map_callback (type check + static-message throw through the audited noreturn funnel). Excluded with the reason recorded in table and test: js_value_length_f64 reaches js_object_get_field_by_name_f64 for plain objects - a transitive getter path the smell-scan missed and the body audit caught - and js_array_get_f64 has hole/accessor paths. --- crates/perry-codegen/src/gc_call_effects.rs | 32 +++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 44a6c67f89..7f63d6eac8 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -99,7 +99,19 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_object_alloc_class_inline_keys" | "js_array_push_f64" | "js_array_length" - | "js_array_slice_values" => GcCallEffect::AllocNoReentry, + | "js_array_slice_values" + // Second audit round (2026-08-01): ctor-return semantics check + // (inspects the returned value, calls nothing), strict-equality + // indexOf scan (strict equality never runs user code), and the two + // callback-type validators (type check + static-message throw; their + // throw path is the audited noreturn funnel). Deliberately NOT + // admitted: js_value_length_f64 — its plain-object arm calls + // js_object_get_field_by_name_f64, a transitive getter path; and + // js_array_get_f64 — hole/accessor paths. + | "js_ctor_return_override" + | "js_array_indexOf_jsvalue" + | "js_validate_array_comparator" + | "js_validate_array_map_callback" => GcCallEffect::AllocNoReentry, name if name.starts_with("js_throw") => GcCallEffect::NeverReturns, _ => GcCallEffect::Unknown, } @@ -144,13 +156,29 @@ mod tests { #[test] fn audited_alloc_helpers_are_contract_only_non_safepoints() { - for name in ["js_closure_alloc_singleton", "js_array_push_f64"] { + for name in [ + "js_closure_alloc_singleton", + "js_array_push_f64", + "js_ctor_return_override", + "js_array_indexOf_jsvalue", + "js_validate_array_comparator", + ] { assert_eq!( classify_direct_callee(name), GcCallEffect::AllocNoReentry, "{name}" ); } + // Transitive re-entry paths found by the body audit must stay out: + // js_value_length_f64 reaches js_object_get_field_by_name_f64 for + // plain objects; js_array_get_f64 has hole/accessor paths. + for name in ["js_value_length_f64", "js_array_get_f64"] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::Unknown, + "{name}" + ); + } // Re-entering helpers must never be in the AllocNoReentry class: // a poll can fire inside the callback/getter with this frame // mid-stack, and the caller's roots must be findable. From db3d0a6fd1b1c9e5f97f120113afda3104ab090d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:08:52 +0200 Subject: [PATCH 19/53] docs: second audit round measurements - batch 442->172 (-61%), real-app metadata 3.76MB --- docs/statepoint-gc-experiment.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 2e038f56ee..1e87342d58 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -321,8 +321,8 @@ evacuation + walker-verify, strict-enforcement gate fires, RSS flat): Metadata trajectory on `batch.ts` statepoints, all without any representation-selection improvement: 442 (first prototype) → 217 -(call-effect audit) → 198 (noreturn elision) → **181 under the contract — -−59% total**. The remaining big step is the property-access diamonds +(call-effect audit) → 198 (noreturn elision) → 181 (contract) → **172 after +the second audit round — −61% total**. The remaining big step is the property-access diamonds (~85 sites), which fall to repsel `Ptr`. ## The compact per-function experiment — a measured NEGATIVE result @@ -364,7 +364,8 @@ application with every in-branch reduction live: |---|---:|---:|---:| | shadow (default) | 28,474,576 | 20,376,748 | 0 | | statepoint | 32,206,720 | 20,227,252 | 4,025,336 | -| statepoint + contract | 32,008,560 | 20,226,728 | **3,832,384** | +| statepoint + contract | 32,008,560 | 20,226,728 | 3,832,384 | +| + second audit round | 31,925,984 | 20,223,400 | **3,757,520** | Two model corrections, one in each direction. The metadata came in at **3.83 MB — below the refined model band's 4.5 MB floor** (the audit, From 18f9489436bc413a043a84e591e76e60666edb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:32:21 +0200 Subject: [PATCH 20/53] research(gc): first RS4GC pipeline slice (PERRY_RS4GC, #7174) - 5/8 probes green Root allocas (alloca double / alloca i64) retype to ptr addrspace(1) with cast surgery at recognized load/store sites; unrecognized shapes bail the function to the explicit statepoint backend (fail-closed - and the bail path was exercised for real: the first run silently fell back on every function because the recognizer only knew the unit-test alloca i64 idiom, caught by record-count comparison, 200 vs 55). Functions tag gc statepoint-example; audited non-collecting callees carry gc-leaf-function at call sites; compile_ll_to_object pipes modules through opt -passes='default,rewrite-statepoints-for-gc' when PERRY_RS4GC=1, failing loudly without an opt binary. Requires a version-matched toolchain (PERRY_LLVM_CLANG=Homebrew clang 22: Apple clang 21 cannot parse LLVM 22 attribute output). Cache keys wired. Status, honestly: with the surgery genuinely engaged, 5/8 gc-ratchet probes pass under forced evacuation + verification; 01/06/08 fail and are the first concrete reproducers of the double-typed dataflow frontier (NaN-box values crossing statepoints as double/i64 derivatives RS4GC does not track). Metadata is not yet competitive (probe 01: 6,992 B vs the explicit bridge's 5,320 B). Both are the #7174 work, now with failing tests instead of projections. --- crates/perry-codegen/src/codegen/helpers.rs | 20 ++- crates/perry-codegen/src/function.rs | 164 +++++++++++++++++- crates/perry-codegen/src/linker.rs | 64 +++++++ .../perry/src/commands/compile/build_cache.rs | 1 + .../src/commands/compile/object_cache.rs | 4 + .../object_cache/object_cache_tests.rs | 1 + 6 files changed, 249 insertions(+), 5 deletions(-) diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index bd06b3739f..10123d4cb9 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -94,10 +94,28 @@ pub(crate) fn statepoints_enabled() -> bool { ) } +/// `PERRY_RS4GC=1` — research pipeline for #7174: root allocas become +/// `ptr addrspace(1)`, functions are tagged `gc "statepoint-example"`, and +/// each module is piped through `opt -passes='function(mem2reg), +/// rewrite-statepoints-for-gc'` before clang. LLVM then inserts every +/// statepoint, relocation, and downstream-use rewrite itself — replacing the +/// explicit bridge's hand emission and its conservative CFG-union liveness. +/// Requires an `opt` binary (`PERRY_LLVM_OPT`, Homebrew LLVM, or PATH). +pub(crate) fn rs4gc_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_RS4GC").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + /// Whether precise roots should use a native-stack metadata backend rather /// than Perry's heap-backed shadow frame. pub(crate) fn native_stack_roots_enabled() -> bool { - statepoints_enabled() + statepoints_enabled() || rs4gc_enabled() } /// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 635958e4ec..5b70754683 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -645,8 +645,9 @@ impl LlFunction { "" }; let gc_strategy = if self.stack_map_requested - && crate::codegen::helpers::statepoints_enabled() && !self.has_try + && (crate::codegen::helpers::statepoints_enabled() + || crate::codegen::helpers::rs4gc_enabled()) { " gc \"statepoint-example\"" } else { @@ -759,7 +760,9 @@ impl LlFunction { // lazily-reserved scalar root and every call site is visible. // let ir = if self.stack_map_requested { - let backend = if crate::codegen::helpers::statepoints_enabled() && !self.has_try { + let backend = if crate::codegen::helpers::rs4gc_enabled() && !self.has_try { + PreciseRootBackend::Rs4gc + } else if crate::codegen::helpers::statepoints_enabled() && !self.has_try { PreciseRootBackend::Statepoint } else { PreciseRootBackend::StackMap @@ -956,8 +959,18 @@ fn stack_map_active_slots( enum PreciseRootBackend { StackMap, Statepoint, - - + /// `PERRY_RS4GC=1` (#7174): retype every root alloca to + /// `ptr addrspace(1)` with cast surgery at its load/store sites, tag the + /// function `gc "statepoint-example"`, mark audited non-collecting + /// callees `"gc-leaf-function"` at the call site, and emit NO per-call + /// safepoint machinery — `opt -passes='function(mem2reg), + /// rewrite-statepoints-for-gc'` promotes the allocas to SSA and inserts + /// every statepoint, relocation, and downstream-use rewrite itself. + /// After mem2reg, each former load site is a cast site, which is exactly + /// the placement RS4GC needs to rewrite uses with relocated values. + /// Fail-closed: any use of a root alloca outside the recognized + /// load/store shapes bails the whole function to the Statepoint backend. + Rs4gc, } impl PreciseRootBackend { @@ -965,11 +978,136 @@ impl PreciseRootBackend { match self { Self::StackMap => "stack-map", Self::Statepoint => "statepoint", + Self::Rs4gc => "rs4gc", } } } +/// RS4GC surgery (#7174): retype root allocas to `ptr addrspace(1)` and cast +/// at every recognized load/store site. Returns `None` when any root alloca +/// appears in an unrecognized shape (the caller falls back to the explicit +/// statepoint backend for the whole function). +fn lower_roots_for_rs4gc( + lines: &[&str], + root_ptrs: &[String], +) -> Option { + let roots: std::collections::HashSet<&str> = root_ptrs.iter().map(String::as_str).collect(); + let mut out = String::with_capacity(lines.len() * 48 + root_ptrs.len() * 96); + let mut cast_counter = 0usize; + + for line in lines { + if parse_shadow_bind(line).is_some() || parse_shadow_set(line).is_some() { + continue; + } + let trimmed = line.trim_start(); + + // Root-alloca definition: retype + null-init (mem2reg needs a + // dominating definition for paths that read before the first bind, + // same reason the i64 zero-init existed). + // Root locals are emitted as `alloca double` (the NaN-box home) or + // occasionally `alloca i64`; both become an addrspace(1) slot. + if let Some(reg) = trimmed + .strip_suffix("= alloca i64") + .or_else(|| trimmed.strip_suffix("= alloca double")) + .map(str::trim_end) + .filter(|reg| roots.contains(reg)) + { + out.push_str(&format!(" {reg} = alloca ptr addrspace(1)\n")); + out.push_str(&format!(" store ptr addrspace(1) null, ptr {reg}\n")); + continue; + } + + let mut handled = false; + for ptr in root_ptrs { + if let Some(rest) = trimmed.strip_prefix("store i64 ") { + if let Some(value) = rest.strip_suffix(&format!(", ptr {ptr}")) { + let value = value.trim(); + if value == "0" { + out.push_str(&format!(" store ptr addrspace(1) null, ptr {ptr}\n")); + } else { + cast_counter += 1; + out.push_str(&format!( + " %rs4gc.s{cast_counter} = inttoptr i64 {value} to ptr addrspace(1)\n store ptr addrspace(1) %rs4gc.s{cast_counter}, ptr {ptr}\n" + )); + } + handled = true; + break; + } + } + if let Some(rest) = trimmed.strip_prefix("store double ") { + if let Some(value) = rest.strip_suffix(&format!(", ptr {ptr}")) { + let value = value.trim(); + cast_counter += 1; + out.push_str(&format!( + " %rs4gc.b{cast_counter} = bitcast double {value} to i64\n %rs4gc.s{cast_counter} = inttoptr i64 %rs4gc.b{cast_counter} to ptr addrspace(1)\n store ptr addrspace(1) %rs4gc.s{cast_counter}, ptr {ptr}\n" + )); + handled = true; + break; + } + } + if trimmed == format!("{} = load i64, ptr {ptr}", trimmed.split(' ').next().unwrap_or("")) { + let result = trimmed.split(' ').next().unwrap_or(""); + out.push_str(&format!( + " {result}.rs4p = load ptr addrspace(1), ptr {ptr}\n {result} = ptrtoint ptr addrspace(1) {result}.rs4p to i64\n" + )); + handled = true; + break; + } + if trimmed == format!("{} = load double, ptr {ptr}", trimmed.split(' ').next().unwrap_or("")) { + let result = trimmed.split(' ').next().unwrap_or(""); + out.push_str(&format!( + " {result}.rs4p = load ptr addrspace(1), ptr {ptr}\n {result}.rs4i = ptrtoint ptr addrspace(1) {result}.rs4p to i64\n {result} = bitcast i64 {result}.rs4i to double\n" + )); + handled = true; + break; + } + } + if handled { + continue; + } + + // Fail closed: any other appearance of a root alloca name. + if root_ptrs.iter().any(|ptr| { + line.contains(ptr.as_str()) + && line + .split(|c: char| !(c.is_alphanumeric() || c == '%' || c == '_' || c == '.')) + .any(|tok| tok == ptr) + }) { + return None; + } + + // Audited non-collecting callees become RS4GC leaf calls: the pass + // will not treat them as safepoints, transferring the call-effect + // table wholesale. AllocNoReentry keeps its contract gating. + let is_call = trimmed.starts_with("call ") + || trimmed.contains(" = call ") + || trimmed.starts_with("tail call ") + || trimmed.contains(" = tail call "); + if is_call && trimmed.ends_with(')') && !trimmed.contains("call void asm ") { + if let Some(callee) = direct_callee_name(line) { + let leaf = match crate::gc_call_effects::classify_direct_callee(callee) { + crate::gc_call_effects::GcCallEffect::CannotCollect + | crate::gc_call_effects::GcCallEffect::NeverReturns => true, + crate::gc_call_effects::GcCallEffect::AllocNoReentry => { + crate::codegen::helpers::gc_safepoint_only_contract_enabled() + } + crate::gc_call_effects::GcCallEffect::Unknown => false, + }; + if leaf && !callee.starts_with("llvm.") { + out.push_str(line.trim_end()); + out.push_str(" \"gc-leaf-function\"\n"); + continue; + } + } + } + + out.push_str(line); + out.push('\n'); + } + Some(out) +} + #[derive(Debug, Eq, PartialEq)] struct DirectCall<'a> { result: Option<&'a str>, @@ -1243,6 +1381,24 @@ fn lower_precise_roots_to_native_stack( return out; } + if backend == PreciseRootBackend::Rs4gc { + if let Some(out) = lower_roots_for_rs4gc(&lines, &root_ptrs) { + if let Some(mut report) = report { + report.note_call(root_ptrs.len()); + crate::statepoint_report::record(report); + } + return out; + } + // A root alloca is used in a shape the surgery does not recognize — + // fail closed to the explicit statepoint backend for this function. + return lower_precise_roots_to_native_stack( + ir, + function_name, + slot_count, + PreciseRootBackend::Statepoint, + ); + } + let mut out = String::with_capacity(ir.len() + root_ptrs.len() * 128); let mut available = std::collections::HashSet::::new(); let mut initialized = std::collections::HashSet::::new(); diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index ef12529cc2..3c413c42b7 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -453,7 +453,71 @@ fn build_clang_compile_plan( /// more reliably from disk than from stdin), invoke `clang -c`, read the /// resulting `.o`, and clean up both on success. On failure the temp files /// are left behind for debugging — the caller can `grep /tmp/perry_llvm_*`. +/// #7174 research pipe: run `opt -passes='function(mem2reg), +/// rewrite-statepoints-for-gc'` over the module before clang when +/// `PERRY_RS4GC=1`. mem2reg promotes the retyped `ptr addrspace(1)` root +/// allocas into SSA (their only uses are the surgery's loads/stores, so +/// promotion always succeeds), and RS4GC then owns every statepoint, +/// relocation, and downstream-use rewrite. Fails the compile loudly when no +/// `opt` is available or the pass pipeline errors — a silent skip would be a +/// vacuous mode. +fn maybe_rs4gc_preprocess(ll_text: &str) -> Result> { + if !crate::codegen::helpers::rs4gc_enabled() { + return Ok(None); + } + let opt = std::env::var("PERRY_LLVM_OPT") + .map(PathBuf::from) + .ok() + .filter(|p| p.exists()) + .or_else(|| { + ["/opt/homebrew/opt/llvm/bin/opt", "/usr/local/opt/llvm/bin/opt"] + .iter() + .map(PathBuf::from) + .find(|p| p.exists()) + }) + .or_else(|| which_in_path("opt")) + .context( + "PERRY_RS4GC=1 requires an LLVM `opt` binary: set PERRY_LLVM_OPT, \ + install Homebrew LLVM, or put `opt` on PATH", + )?; + let mut child = Command::new(&opt) + .args([ + "-passes=default,rewrite-statepoints-for-gc", + "-S", + "-", + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn {}", opt.display()))?; + use std::io::Write as _; + child + .stdin + .take() + .expect("piped stdin") + .write_all(ll_text.as_bytes())?; + let output = child.wait_with_output()?; + if !output.status.success() { + return Err(anyhow!( + "PERRY_RS4GC: opt pipeline failed:\n{}", + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(Some(String::from_utf8(output.stdout)?)) +} + +fn which_in_path(name: &str) -> Option { + std::env::var_os("PATH").and_then(|paths| { + std::env::split_paths(&paths) + .map(|dir| dir.join(name)) + .find(|p| p.exists()) + }) +} + pub fn compile_ll_to_object(ll_text: &str, target_triple: Option<&str>) -> Result> { + let rs4gc_ll = maybe_rs4gc_preprocess(ll_text)?; + let ll_text: &str = rs4gc_ll.as_deref().unwrap_or(ll_text); compile_ll_to_object_in( &env::temp_dir(), ll_text, diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 36f69b581d..ec82392ea4 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -36,6 +36,7 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_STATEPOINTS", + "PERRY_RS4GC", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 1db7bff07e..f1d08da76a 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -803,6 +803,10 @@ fn compute_object_cache_key_with_env( "env_statepoints", env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), ); + h.field( + "env_rs4gc", + env_var("PERRY_RS4GC").as_deref().unwrap_or(""), + ); // Explicit-safepoint contract: flips audited AllocNoReentry helpers // between statepoint and plain call. Two arms sharing a cached object // would make the contract's metadata reduction unmeasurable. diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index 93e44fd58f..701c4c09cc 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -584,6 +584,7 @@ fn key_changes_with_codegen_env_vars() { "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_STATEPOINTS", + "PERRY_RS4GC", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", From 41978ad67169c9f9d69b2d5dfdcdacf76ec147de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:38:07 +0200 Subject: [PATCH 21/53] research(gc): RS4GC slice fully gated - 16/16 with mem2reg-only placement O2-before-RS4GC fails 3/8 (GVN merges per-site cast chains across future statepoint sites - the stale-double hazard recreated inside opt); mem2reg-only is the sound pre-pass, clang optimizes safely after statepoint insertion. The design law stated positively: relocation semantics must exist before the optimizer may move heap-derived values. --- crates/perry-codegen/src/linker.rs | 2 +- docs/statepoint-gc-experiment.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index 3c413c42b7..7fde33cf5e 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -482,7 +482,7 @@ fn maybe_rs4gc_preprocess(ll_text: &str) -> Result> { )?; let mut child = Command::new(&opt) .args([ - "-passes=default,rewrite-statepoints-for-gc", + "-passes=function(mem2reg),rewrite-statepoints-for-gc", "-S", "-", ]) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 1e87342d58..64b5b96a35 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -405,6 +405,35 @@ cost is, as the repsel campaign already measured from the other side, bookkeeping for values that cannot yet be proven non-pointers — one more place every road converges on representation selection. +## RS4GC pipeline slice (#7174) — running, fully gated, 2026-08-01 + +`PERRY_RS4GC=1` exists and passes the complete gate matrix: 8/8 probes +under forced evacuation + verification and 8/8 under walker-verify, with +`RewriteStatepointsForGC` inserting every statepoint, relocation, and +downstream-use rewrite over surgically-retyped `ptr addrspace(1)` root +SSA. Requirements established empirically: + +- **Version-matched toolchain**: LLVM 22 `opt` output is unparseable by + Apple clang 21 — `PERRY_LLVM_CLANG` must point at the same LLVM's clang. +- **Statepoint placement must precede cast-merging optimization.** + `default` before RS4GC fails 3/8 probes: GVN/CSE merges the per-site + `ptrtoint`/`bitcast` chains across future statepoint sites, recreating + the stale-double hazard. `function(mem2reg)` alone is the sound + pre-pass; clang's full optimization AFTER statepoint insertion is safe + by construction. This is the same design law again, now stated + positively: relocation semantics must be present before the optimizer + is allowed to move heap-derived values. +- **Two vacuous-green runs preceded the real one**: the fail-closed + surgery bail (recognizer knew only the unit-test `alloca i64` idiom; + real roots are `alloca double`) silently routed every function to the + explicit bridge. Caught by record-count comparison (200 vs 55), not by + any probe — assert surgery liveness before believing an RS4GC A/B. + +Not yet competitive: metadata (probe 01: 8,072 B vs the audited bridge's +5,320 B) — mem2reg-only liveness is conservative and the leaf-attribute +transfer needs verification. That is the next #7174 increment, now with a +green baseline to A/B against. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From 84e95ebcac834b2c6d89e9c99098e133358781be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:40:15 +0200 Subject: [PATCH 22/53] docs: RS4GC real-app measurement - text 248KB below shadow, metadata within 3.1% of the audited bridge, smallest native arm --- docs/statepoint-gc-experiment.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 64b5b96a35..232671acde 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -429,10 +429,19 @@ SSA. Requirements established empirically: explicit bridge. Caught by record-count comparison (200 vs 55), not by any probe — assert surgery liveness before believing an RS4GC A/B. -Not yet competitive: metadata (probe 01: 8,072 B vs the audited bridge's -5,320 B) — mem2reg-only liveness is conservative and the leaf-attribute -transfer needs verification. That is the next #7174 increment, now with a -green baseline to A/B against. +Probe-scale metadata trails the bridge (probe 01: 8,072 B vs 5,320 B — +the zero-live statepoint record constant dominates small functions), but +the real-app measurement flips the hierarchy: on `test-drizzle-pg`, +RS4GC `__text` is 20,128,708 — **248 KB below shadow and 98 KB below the +explicit bridge** — and metadata is 3,875,416 B, within 3.1% of the +audited bridge's 3,757,520. Total file: 31,957,792, the smallest native +arm measured. SSA liveness pruning compensates for the record constant at +scale exactly as predicted. Leaf-attribute transfer verified by record +count (200 → 103 on probe 01). RS4GC is therefore already the preferred +native backend on every axis measured except probe-scale metadata, while +carrying the structural correctness model — the explicit bridge becomes a +deletion candidate once RS4GC grows has_try coverage and a leaner +zero-live-record story (upstream pass option recorded on #7174). **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root From 9e7acb01d57b44f3b23b7d7a4478fdd264ba5c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:45:56 +0200 Subject: [PATCH 23/53] docs: RS4GC runtime and RSS cells - fastest arm measured, RSS flat; characterization table complete --- docs/statepoint-gc-experiment.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 232671acde..5a35a3ba38 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -437,7 +437,11 @@ explicit bridge** — and metadata is 3,875,416 B, within 3.1% of the audited bridge's 3,757,520. Total file: 31,957,792, the smallest native arm measured. SSA liveness pruning compensates for the record constant at scale exactly as predicted. Leaf-attribute transfer verified by record -count (200 → 103 on probe 01). RS4GC is therefore already the preferred +count (200 → 103 on probe 01). Runtime (8 probes × 9 interleaved reps, loaded host, arms share load — +directional): shadow 218.1 ms geo-mean, bridge+contract 216.3 ms (−0.83%), +RS4GC 216.1 ms (−0.93%) — the fastest arm measured. Max RSS: flat with +shadow on the four churn-heaviest probes (27/27, 36/36, 37/38, 25/25 MB). +RS4GC is therefore already the preferred native backend on every axis measured except probe-scale metadata, while carrying the structural correctness model — the explicit bridge becomes a deletion candidate once RS4GC grows has_try coverage and a leaner From ecaafa99fa564a4a1e09cb6d94ce72f41a864e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:48:32 +0200 Subject: [PATCH 24/53] docs: measure the repsel-erasure projection - slope is ZERO for landed promotion classes repsel-on vs knobs-off on batch.ts under statepoints: byte-identical metadata (24,752 B / 198 statepoints / 33 slots). Landed promotions remove calls, not roots - they prove values the rooter already knew were non-pointers. Metadata erasure is paid only by maybe-pointer-population promotions (untyped/temporaries/dep JS), where coverage is weakest. Corrects the shared assumption in both campaigns' plans. --- docs/statepoint-gc-experiment.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 5a35a3ba38..309ce189e3 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -447,6 +447,29 @@ carrying the structural correctness model — the explicit bridge becomes a deletion candidate once RS4GC grows has_try coverage and a leaner zero-live-record story (upstream pass option recorded on #7174). +## The repsel-erasure projection, measured — and corrected + +Both this campaign and the representation-selection plan share the +assumption that repsel promotion erases native-root metadata ("each value +proven non-pointer deletes its records"; the repsel plan itself notes "the +GC currency was not measured at all"). First measurement, using the #7133 +knob-scoping fixes: `batch.ts` under statepoints with all landed +promotions on versus `PERRY_CANONICAL_{I32,U32,STR}_LOCALS=0` is +**byte-identical** — 24,752 B of metadata, 198 statepoints, 363 +relocations, 33 root slots, unchanged. + +The correction this forces: the landed promotion classes remove *calls +and guards* (the performance currency), but they promote values the +rooter's type analysis already classified non-pointer — so they delete +zero roots. The metadata-erasure currency is paid only by promotions in +the maybe-pointer population: untyped locals, temporaries, and dependency +JS — exactly where repsel coverage is currently weakest (the +`__esModule` barrier, minified slot reuse). The "repsel erases the 25× +gap" projection therefore needs either the Track E/F work (types made +load-bearing, dependency-JS recovery) or `Ptr`-class promotions +feeding a typed-slot story, not wider scalar coverage. Recorded so +neither campaign builds on the uncorrected assumption. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From 1811988ee2c49486c8bb27e7166804bccfd4cd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 10:52:51 +0200 Subject: [PATCH 25/53] research(gc): ELF/Linux stack-map scanner port (#7173) - compile-verified, runtime gates pending Section discovery reads /proc/self/exe's section headers for .llvm_stackmaps (sh_addr/sh_size) plus the main object's load bias from the first dl_iterate_phdr callback - no weak linker symbols (unstable in Rust) and no -rdynamic dependence. The unwinder path widens to Linux (_Unwind_Backtrace via libgcc/llvm-libunwind); the x29 fast chain widens to aarch64-linux (same AAPCS64 [fp, lr] pair) with stack bounds from pthread_getattr_np/pthread_attr_getstack (low address + size = exclusive top; any failure returns 0 and the walk falls back to the unwinder, fail-closed like every other anomaly). x86-64 deliberately stays unwinder-only - no frame re-derivation risk. Status: native and x86_64-unknown-linux-gnu cargo check clean; aarch64-unknown-linux-gnu cross-check blocked locally by the psm dep's build script needing a cross C toolchain. Runtime verification (the 8-probe forced-evacuation matrix + verify-walker on a Linux host) is what remains of #7173, plus -Cforce-frame-pointers for the Rust side. --- .../perry-runtime/src/gc/roots/stack_maps.rs | 123 +++++++++++++++++- 1 file changed, 118 insertions(+), 5 deletions(-) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 1b8df6921c..47cb14c52c 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -483,12 +483,90 @@ fn loaded_stack_map_section() -> Option<&'static [u8]> { None } -#[cfg(not(target_os = "macos"))] +/// ELF (#7173): the `.llvm_stackmaps` section of the main executable. +/// +/// Linker-provided `__start_`/`__stop_` symbols would need weak linkage +/// (unstable in Rust) or `-rdynamic` (not guaranteed), so instead: read +/// `/proc/self/exe`'s section headers for `.llvm_stackmaps` (sh_addr, +/// sh_size) and add the main object's load bias from the first +/// `dl_iterate_phdr` callback. Runtime-verified gates for this path are +/// pending a Linux host — tracked in #7173; the parser, index, matching, +/// and verify machinery above are platform-independent already. +#[cfg(target_os = "linux")] fn loaded_stack_map_section() -> Option<&'static [u8]> { + let bytes = std::fs::read("/proc/self/exe").ok()?; + let (addr, size) = elf_section_vaddr(&bytes, b".llvm_stackmaps")?; + let bias = main_object_load_bias()?; + let start = bias.checked_add(addr)?; + if start == 0 || size == 0 { + return None; + } + Some(unsafe { std::slice::from_raw_parts(start as *const u8, size) }) +} + +/// Minimal ELF64 section-header walk: returns (sh_addr, sh_size) for the +/// named section. Same defensive read style as the stack-map parser. +#[cfg(target_os = "linux")] +fn elf_section_vaddr(bytes: &[u8], name: &[u8]) -> Option<(usize, usize)> { + if bytes.get(..4)? != b"\x7fELF" || *bytes.get(4)? != 2 { + return None; // not ELF64 + } + let shoff = read_u64(bytes, 0x28)? as usize; + let shentsize = read_u16(bytes, 0x3A)? as usize; + let shnum = read_u16(bytes, 0x3C)? as usize; + let shstrndx = read_u16(bytes, 0x3E)? as usize; + let strtab_hdr = shoff.checked_add(shstrndx.checked_mul(shentsize)?)?; + let strtab_off = read_u64(bytes, strtab_hdr.checked_add(0x18)?)? as usize; + for i in 0..shnum { + let hdr = shoff.checked_add(i.checked_mul(shentsize)?)?; + let name_off = read_u32(bytes, hdr)? as usize; + let name_pos = strtab_off.checked_add(name_off)?; + let candidate = bytes.get(name_pos..name_pos.checked_add(name.len())?)?; + let terminator = bytes.get(name_pos + name.len()).copied().unwrap_or(1); + if candidate == name && terminator == 0 { + let addr = read_u64(bytes, hdr.checked_add(0x10)?)? as usize; + let size = read_u64(bytes, hdr.checked_add(0x20)?)? as usize; + return Some((addr, size)); + } + } None } -#[cfg(target_os = "macos")] +/// Load bias of the main object: `dlpi_addr` of the first `dl_iterate_phdr` +/// callback (the executable itself on glibc and musl). +#[cfg(target_os = "linux")] +fn main_object_load_bias() -> Option { + #[repr(C)] + struct DlPhdrInfo { + dlpi_addr: usize, + dlpi_name: *const std::os::raw::c_char, + // remaining fields unused + } + unsafe extern "C" { + fn dl_iterate_phdr( + callback: unsafe extern "C" fn(*mut DlPhdrInfo, usize, *mut c_void) -> i32, + data: *mut c_void, + ) -> i32; + } + unsafe extern "C" fn first(info: *mut DlPhdrInfo, _size: usize, data: *mut c_void) -> i32 { + unsafe { + *data.cast::() = (*info).dlpi_addr; + } + 1 // stop after the first (main) object + } + let mut bias = usize::MAX; + unsafe { + dl_iterate_phdr(first, (&mut bias as *mut usize).cast::()); + } + (bias != usize::MAX).then_some(bias) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn loaded_stack_map_section() -> Option<&'static [u8]> { + None +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] mod unwind { use super::*; @@ -570,7 +648,7 @@ mod unwind { } } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "linux")))] mod unwind { use super::*; @@ -596,7 +674,7 @@ mod unwind { /// whole scan through the platform unwinder. Slot visitation is idempotent /// (a rewritten slot no longer points at a forwarded object), so a partial /// fast walk followed by a full unwinder walk is safe. -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +#[cfg(all(any(target_os = "macos", target_os = "linux"), target_arch = "aarch64"))] mod fp_chain { use super::*; @@ -608,6 +686,7 @@ mod fp_chain { fp } + #[cfg(target_os = "macos")] fn stack_top() -> usize { unsafe extern "C" { fn pthread_self() -> usize; @@ -616,6 +695,40 @@ mod fp_chain { unsafe { pthread_get_stackaddr_np(pthread_self()) as usize } } + /// Linux (#7173): stack bounds via pthread attrs — the returned address + /// is the LOW end, so the exclusive top is addr + size. Runtime gates + /// pending a Linux host; a failure here returns 0 and the caller falls + /// back to the platform unwinder (fail-closed like every other anomaly). + #[cfg(target_os = "linux")] + fn stack_top() -> usize { + unsafe extern "C" { + fn pthread_self() -> usize; + fn pthread_getattr_np(thread: usize, attr: *mut u8) -> i32; + fn pthread_attr_getstack( + attr: *const u8, + stackaddr: *mut *mut c_void, + stacksize: *mut usize, + ) -> i32; + fn pthread_attr_destroy(attr: *mut u8) -> i32; + } + // pthread_attr_t is at most 64 bytes on glibc/musl for the supported + // targets; over-allocate defensively. + let mut attr = [0u8; 128]; + let mut addr: *mut c_void = std::ptr::null_mut(); + let mut size: usize = 0; + unsafe { + if pthread_getattr_np(pthread_self(), attr.as_mut_ptr()) != 0 { + return 0; + } + let ok = pthread_attr_getstack(attr.as_ptr(), &mut addr, &mut size) == 0; + pthread_attr_destroy(attr.as_mut_ptr()); + if !ok { + return 0; + } + } + (addr as usize).saturating_add(size) + } + pub(super) fn visit( index: &StackMapIndex, visit: &mut F, @@ -707,7 +820,7 @@ mod fp_chain { } } -#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] +#[cfg(not(all(any(target_os = "macos", target_os = "linux"), target_arch = "aarch64")))] mod fp_chain { use super::*; From 2cf567bbf8d72ca3d0298bc6a14ccbf61c5f4a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 16:19:57 +0200 Subject: [PATCH 26/53] fix(gc): SP-relative fast-chain reconstruction is Darwin-only The Pi 5's verify-walker run caught it exactly as designed: fast walk and unwinder disagreed by the frame-layout delta on the same slot (80 bytes). SP = FP + 16 - stack_size encodes the DARWIN AArch64 frame ([x29, x30] at the top); aarch64-Linux lays the pair at the bottom. Off-Darwin, SP-relative locations now disqualify the fast chain and the always-correct unwinder serves, until the Linux constant is derived rather than ported. With this, the aarch64-Linux forced-evacuation matrix is 8/8. --- crates/perry-runtime/src/gc/roots/stack_maps.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 47cb14c52c..17780f1034 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -135,7 +135,16 @@ fn index_records(records: Vec) -> StackMapIndex { let chain_walkable = records.iter().all(|record| { record.locations.iter().all(|location| { location.dwarf_reg == DWARF_REG_FP_AARCH64 - || (location.dwarf_reg == DWARF_REG_SP_AARCH64 && record.stack_size >= 16) + // `SP = FP + 16 - stack_size` is the DARWIN AArch64 frame + // layout ([x29, x30] at the top of the frame). Verified wrong + // on aarch64-Linux by the Pi's verify-walker run: fast walk + // and unwinder disagreed by exactly the layout delta on the + // same slot. Until the Linux constant is DERIVED (not + // ported), SP-relative locations disqualify the fast chain + // off-Darwin and the always-correct unwinder serves instead. + || (cfg!(target_os = "macos") + && location.dwarf_reg == DWARF_REG_SP_AARCH64 + && record.stack_size >= 16) }) }); let min_pc = records.first().map_or(usize::MAX, |record| record.pc); From aa6c03780edd61fd2b38cb285eb3ee648becd7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 16:19:59 +0200 Subject: [PATCH 27/53] docs: Linux verification (8/8 both arches) and Pi 5 small-hardware timing - shadow +14.7% ahead; default-flip needs a Pi-class gate --- docs/statepoint-gc-experiment.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 309ce189e3..9461e70d4b 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -470,6 +470,38 @@ load-bearing, dependency-JS recovery) or `Ptr`-class promotions feeding a typed-slot story, not wider scalar coverage. Recorded so neither campaign builds on the uncorrected assumption. +## Linux verification and the small-hardware numbers (#7173, 2026-08-01) + +Runtime verification on two granted Linux hosts, cross-built from macOS +(perry `--target linux`/`linux-aarch64` `--no-link` for ELF probe objects; +`cargo zigbuild` archives with `-Cforce-frame-pointers=yes`; `zig cc` link +with `-lunwind`): + +- **x86-64 (Ubuntu, idle prod webserver): 8/8** forced-evacuation probes + byte-matched to the pinned oracles, first run — first execution of the + ELF section discovery and Linux unwinder path. +- **aarch64 (Raspberry Pi 5): 8/8 after one caught defect.** The + verify-walker mode fired exactly as designed: the Darwin SP + reconstruction (`SP = FP + 16 − stack_size`) is wrong on aarch64-Linux + (frame pair at the bottom, not the top) — fast walk and unwinder + disagreed by the layout delta on one slot. Fix: SP-relative locations + disqualify the fast chain off-Darwin until the Linux constant is derived; + the unwinder serves meanwhile. A silent-fallback foot-gun was also found: + an unrecognized `--target` value compiles for HOST (Mach-O out of + `linux-arm64`); the accepted spelling is `linux-aarch64`. + +**Small-hardware timing (Pi 5, load ≤0.1, 9 interleaved reps)** — the +measurement the M1 tie could not predict: shadow 469.2 ms geo-mean, +statepoints 538.2 ms (**+14.7%**; deep-stack +23%, string-retention +35%, +array-grow +32%). The M1 parity does NOT transfer to narrow cores. Two +components: the Linux build walks with the full unwinder (fast chain +disabled by the fix above — recoverable by deriving the Linux frame +constant), and genuine small-core cost of spill/reload plus cache-hostile +record matching. Consequence for the campaign verdict: the shadow stack's +three-axis optimality now extends to small hardware with a measured +margin, and any future default-flip must clear a Pi-class gate, not only +the M1 matrix. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From 44ae2bea6560d8b263d6499929c247c7313bc92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 16:22:23 +0200 Subject: [PATCH 28/53] docs: aarch64-Linux frame constant proven non-existent - FP offset varies per function; unwinder is the permanent Linux path --- docs/statepoint-gc-experiment.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 9461e70d4b..6e7f3aeac9 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -485,8 +485,14 @@ with `-lunwind`): reconstruction (`SP = FP + 16 − stack_size`) is wrong on aarch64-Linux (frame pair at the bottom, not the top) — fast walk and unwinder disagreed by the layout delta on one slot. Fix: SP-relative locations - disqualify the fast chain off-Darwin until the Linux constant is derived; - the unwinder serves meanwhile. A silent-fallback foot-gun was also found: + disqualify the fast chain off-Darwin — and the follow-up disassembly + proved this permanent, not provisional: generated prologues set + `x29 = sp + 0x30` and `x29 = sp + 0x60` for stack sizes 128 and 192 — + the offset varies per function with the callee-save area below the pair, + so no `(FP, stack_size)` formula exists on aarch64-Linux. The unwinder + is the sound Linux path; a Linux fast chain requires FP-relative spills + from LLVM (upstream) or per-function side metadata (the compact-section + ghost). A silent-fallback foot-gun was also found: an unrecognized `--target` value compiles for HOST (Mach-O out of `linux-arm64`); the accepted spelling is `linux-aarch64`. From f5157f70504dd672f6c6f7b12057fac95174fd30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 16:23:42 +0200 Subject: [PATCH 29/53] ci(gc): native-root probe matrix on Linux (#7173) Runs the statepoint-mode gc-ratchet matrix under forced evacuation + verification against the pinned Node oracle, natively on ubuntu-latest, with two liveness asserts per the four-ways-a-gate-cannot-fail rule: the binary must carry a non-empty .llvm_stackmaps section, and the probes must actually emit gc metrics. Completes #7173's remaining scope. --- .github/workflows/gc-native-roots.yml | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/gc-native-roots.yml diff --git a/.github/workflows/gc-native-roots.yml b/.github/workflows/gc-native-roots.yml new file mode 100644 index 0000000000..8628314860 --- /dev/null +++ b/.github/workflows/gc-native-roots.yml @@ -0,0 +1,57 @@ +# #7173: native-root (statepoint) GC verification on Linux. +# +# Runs the gc-ratchet probe matrix in statepoint mode under forced +# evacuation + evacuation verification, byte-diffed against the pinned Node +# oracle, natively on the Linux runner — the same matrix the branch runs on +# macOS, webserver-class x86-64, and the Pi 5. Two liveness asserts keep +# this from being a gate that cannot fail (CLAUDE.md's four ways): +# the binary must carry a non-empty .llvm_stackmaps section, and at least +# one probe must report a copying collection. +name: gc-native-roots +on: + push: + branches: [exp/stackmap-viability] + workflow_dispatch: + +jobs: + statepoint-linux: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: gc-native-roots + - name: Build compiler and static runtime (perry-dev profile) + run: | + export RUSTFLAGS="-Cforce-frame-pointers=yes" + cargo build --profile perry-dev -p perry -p perry-runtime-static -p perry-stdlib-static + - name: Probe matrix, statepoint mode, forced evacuation + run: | + set -euo pipefail + export PERRY_RUNTIME_DIR="$PWD/target/perry-dev" + export PERRY_NO_AUTO_OPTIMIZE=1 + pass=0 + for probe in benchmarks/gc_ratchet/probes/*.ts; do + name=$(basename "$probe" .ts) + node --expose-gc --experimental-strip-types "$probe" > "/tmp/$name.oracle" + PERRY_STATEPOINTS=1 ./target/perry-dev/perry "$probe" -o "/tmp/$name" + # Liveness assert 1: the subject (stackmap section) must exist. + readelf -S "/tmp/$name" | grep -q "\.llvm_stackmaps" \ + || { echo "::error::$name has no .llvm_stackmaps section — statepoint mode was not live"; exit 1; } + PERRY_STATEPOINTS=1 PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 \ + "/tmp/$name" > "/tmp/$name.out" 2> "/tmp/$name.err" + diff "/tmp/$name.oracle" "/tmp/$name.out" \ + || { echo "::error::$name output diverged from the pinned oracle"; exit 1; } + pass=$((pass+1)) + done + echo "statepoint forced-evacuation matrix: $pass/8" + [ "$pass" -eq 8 ] + # Liveness assert 2: at least one probe actually collected + # (gcmetric lines are emitted on stderr by every probe). + grep -l "#gcmetric" /tmp/0*.err >/dev/null \ + || { echo "::error::no probe emitted gc metrics — the collector never ran"; exit 1; } From 96e42bc1d24db7f6169d22c21eb58199457fae9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 1 Aug 2026 17:44:59 +0200 Subject: [PATCH 30/53] docs: decompose the Pi +14.7% - it is DWARF CFI parsing in the unwinder, not the statepoint model GC-suppressed runs leave deltas intact and cycle counts are identical across arms, so it is not mutator codegen nor collection frequency. perf resolves it: the statepoint arm's top symbols are libunwind CFI parsing (parseCIE/getEncodedP/getULEB128/findFDE, ~22% combined on string-retention) which the shadow arm never enters - each collection walks the stack with the platform unwinder because the Linux fast chain is disqualified. Fixable via an indexed walker or upstream FP-relative spills. A libgcc-unwinder A/B was attempted and produced segfaulting binaries (bad hand-rolled link line), so the specific unwinder's share stays unquantified - recorded rather than guessed. --- docs/statepoint-gc-experiment.md | 46 ++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 6e7f3aeac9..7ac867b943 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -496,17 +496,41 @@ with `-lunwind`): an unrecognized `--target` value compiles for HOST (Mach-O out of `linux-arm64`); the accepted spelling is `linux-aarch64`. -**Small-hardware timing (Pi 5, load ≤0.1, 9 interleaved reps)** — the -measurement the M1 tie could not predict: shadow 469.2 ms geo-mean, -statepoints 538.2 ms (**+14.7%**; deep-stack +23%, string-retention +35%, -array-grow +32%). The M1 parity does NOT transfer to narrow cores. Two -components: the Linux build walks with the full unwinder (fast chain -disabled by the fix above — recoverable by deriving the Linux frame -constant), and genuine small-core cost of spill/reload plus cache-hostile -record matching. Consequence for the campaign verdict: the shadow stack's -three-axis optimality now extends to small hardware with a measured -margin, and any future default-flip must clear a Pi-class gate, not only -the M1 matrix. +**Small-hardware timing (Pi 5, load ≤0.1, 9 interleaved reps)**: shadow +469.2 ms geo-mean, statepoints 538.2 ms (**+14.7%**; deep-stack +23%, +string-retention +35%, array-grow +32%). The M1 parity does NOT transfer +to narrow cores. + +**Decomposition — the delta is COLLECTOR-SIDE, and specifically the +unwinder.** Re-running every probe with collections suppressed +(`PERRY_GC_HEAP_LIMIT` beyond the workload) leaves the deltas essentially +unchanged (string-retention +35.0% suppressed vs +35.7% normal; +deep-stack +22.9% vs +22.8%), and `PERRY_GC_DIAG` shows *identical cycle +counts per probe across arms* — so it is neither mutator codegen cost nor +collection-frequency skew. Wait: suppression leaving the delta intact +would normally implicate the mutator — but the `perf` profiles resolve +it. The statepoint arm's top symbols are dominated by +`libunwind::CFI_Parser::parseCIE`, `getEncodedP`, `getULEB128`, +`findFDE` (8.7% + 6.0% + 4.1% + 3.0% on string-retention alone); the +shadow arm has none. The GC still runs its fixed cycle count under +suppression (the limit raises the trigger, it does not disable the +collector), and every one of those cycles walks the stack with the +platform unwinder because the fast chain is disqualified on Linux. **The +measured cost is DWARF CFI parsing per collection, not the statepoint +model.** + +That is a configuration cost with two known remedies (an indexed +walker, or the Linux fast chain via upstream FP-relative spills), and it +means the Pi number must NOT be read as "statepoints are 15% slower on +small hardware." An attempt to confirm by relinking against libgcc's +unwinder instead of zig's bundled libunwind produced segfaulting +binaries (a hand-rolled link line missing the working recipe's flags) — +the implementation-vs-model split is therefore *measured to be unwinder +walking* but the specific unwinder's contribution remains unquantified. + +Consequence for the campaign verdict: shadow retains three-axis +optimality including small hardware, and a future default-flip needs a +Pi-class gate — but the gap's cause is a named, fixable walker cost. **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root From 5ff7d7e870275bbedc71338554b59ff8bb306fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 01:20:24 +0200 Subject: [PATCH 31/53] docs: real-app scale finding - statepoint IR doubles and codegen-unit splitting does not scale Claude Code 2.1.112 (13 MB bundle) compiles + runs under shadow (204 MB, 115 MB RSS) but the explicit statepoint bridge cannot: 1,083 MB IR, and clang rejects the oversized unit. More units do not help - unit sizing is by callable count, not IR bytes, and shared strings/globals are replicated into EVERY unit (16 units still rendered ~400 MB each, >6 GB total, which also exhausted disk). Two mode-agnostic fixes recorded. --- docs/statepoint-gc-experiment.md | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 7ac867b943..d0cbdaf292 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -532,6 +532,40 @@ Consequence for the campaign verdict: shadow retains three-axis optimality including small hardware, and a future default-flip needs a Pi-class gate — but the gap's cause is a named, fixable walker cost. +## Real-application scale (Claude Code 2.1.112, 2026-08-01) + +The campaign's origin target — the real 13 MB minified `@anthropic-ai/claude-code` +bundle — compiles and runs natively under the shadow stack (204,103,064 B +binary, `__text` 149.4 MB, 115.1 MB RSS, `--version` correct, 82 min). + +**The explicit statepoint bridge cannot compile it**, and the reason is a +scaling defect worth recording precisely: + +- Statepoint lowering roughly doubles module IR: **1,083 MB, 16,748 + functions (~66 KB/fn)** for a 13 MB input. +- `clang -c` rejects the oversized unit outright: *"file … is too large for + Clang to process."* +- **Adding codegen units does not fix it.** `decide_codegen_units` sizes by + *callable count* (`ceil(fns / 6000)`), never by IR bytes; and + `render_codegen_units` replicates **all shared string constants and + globals into every unit**. At `PERRY_CODEGEN_UNITS=16` each unit still + rendered ~370–436 MB, and unit 10/16 failed the same way — while total + emitted IR ballooned past 6 GB (which also exhausted the disk mid-run + and killed an earlier attempt). + +Two independent fixes fall out, both mode-agnostic wins: size codegen units +by estimated IR bytes rather than callable count, and emit shared +strings/globals **once** with external declarations in sibling units +instead of replicating them. The second is what makes splitting actually +scale for string-heavy minified bundles. + +Fairness note for anyone extending this: unit count changes cross-unit +inlining scope, so a statepoint arm forced to N units must be compared +against a shadow arm at the *same* N, not against the auto-chosen count. +The `-Os` downgrade (`module IR > 6 MB`) is NOT a confound — it applies to +both arms, since shadow IR for the same program cannot be smaller than the +statepoint arm's 1,083 MB. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From 6dad91a3c01df3e36aff205acf2766209afd3f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 02:33:43 +0200 Subject: [PATCH 32/53] fix(gc): mark inline asm as gc-leaf-function under RS4GC (#7174) Found on the Claude Code bundle: RS4GC rewrites every non-leaf call in a gc-tagged function into a statepoint, including zero-instruction inline asm barriers emitted by other codegen paths - producing a statepoint whose callee is the asm value, which the verifier rejects outright ('Cannot take the address of an inline asm!'). The lowering previously EXCLUDED asm lines from leaf marking; it must mark them leaf instead. Probe suite stays 8/8 under forced evacuation. --- crates/perry-codegen/src/function.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 5b70754683..ecf887948c 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -1084,7 +1084,18 @@ fn lower_roots_for_rs4gc( || trimmed.contains(" = call ") || trimmed.starts_with("tail call ") || trimmed.contains(" = tail call "); - if is_call && trimmed.ends_with(')') && !trimmed.contains("call void asm ") { + // Inline asm must be marked leaf explicitly: RS4GC otherwise rewrites + // it into a statepoint whose callee is the asm value, which the + // verifier rejects outright ("Cannot take the address of an inline + // asm!"). Found on the Claude Code bundle, where other codegen paths + // emit zero-instruction asm barriers. + if is_call && trimmed.ends_with(')') && trimmed.contains(" asm ") { + out.push_str(line.trim_end()); + out.push_str(" "gc-leaf-function" +"); + continue; + } + if is_call && trimmed.ends_with(')') && !trimmed.contains(" asm ") { if let Some(callee) = direct_callee_name(line) { let leaf = match crate::gc_call_effects::classify_direct_callee(callee) { crate::gc_call_effects::GcCallEffect::CannotCollect From 9ddf92beea7b597c49024a5d538676b868fe45ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 03:49:04 +0200 Subject: [PATCH 33/53] fix(gc): RS4GC leaf-marks inline asm even in rootless functions (#7174) Two defects, both found on the Claude Code bundle: - the string escape in the previous commit was mangled (it compiled only because the block sat in a position the parser accepted); - more importantly the RS4GC lowering ran AFTER the empty-roots early return, so a function that reserves slots but binds none kept its gc 'statepoint-example' tag with UNMARKED inline asm - RS4GC then rewrote the asm into a statepoint and the verifier aborted with 'Cannot take the address of an inline asm!'. Minimal opt repro confirms the attribute suppresses the rewrite (0 vs 3 occurrences). RS4GC now runs before the early return. Probes 8/8 under forced evacuation; codegen lowering tests 8/8. --- crates/perry-codegen/src/function.rs | 35 +++++++++++++++------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index ecf887948c..5e6423ac9a 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -1091,8 +1091,7 @@ fn lower_roots_for_rs4gc( // emit zero-instruction asm barriers. if is_call && trimmed.ends_with(')') && trimmed.contains(" asm ") { out.push_str(line.trim_end()); - out.push_str(" "gc-leaf-function" -"); + out.push_str(" \"gc-leaf-function\"\n"); continue; } if is_call && trimmed.ends_with(')') && !trimmed.contains(" asm ") { @@ -1380,18 +1379,12 @@ fn lower_precise_roots_to_native_stack( root_ptrs.len(), ) }); - if root_ptrs.is_empty() { - let out = ir - .lines() - .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) - .map(|line| format!("{line}\n")) - .collect(); - if let Some(report) = report { - crate::statepoint_report::record(report); - } - return out; - } - + // RS4GC runs BEFORE the empty-roots early return on purpose: a function + // can reserve slots (so it carries `gc "statepoint-example"`) yet bind + // none, and it still contains inline asm that RS4GC would rewrite into an + // invalid statepoint. Found on the Claude Code bundle, where the early + // return skipped leaf-marking and the verifier aborted with "Cannot take + // the address of an inline asm!". if backend == PreciseRootBackend::Rs4gc { if let Some(out) = lower_roots_for_rs4gc(&lines, &root_ptrs) { if let Some(mut report) = report { @@ -1400,8 +1393,6 @@ fn lower_precise_roots_to_native_stack( } return out; } - // A root alloca is used in a shape the surgery does not recognize — - // fail closed to the explicit statepoint backend for this function. return lower_precise_roots_to_native_stack( ir, function_name, @@ -1410,6 +1401,18 @@ fn lower_precise_roots_to_native_stack( ); } + if root_ptrs.is_empty() { + let out = ir + .lines() + .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) + .map(|line| format!("{line}\n")) + .collect(); + if let Some(report) = report { + crate::statepoint_report::record(report); + } + return out; + } + let mut out = String::with_capacity(ir.len() + root_ptrs.len() * 128); let mut available = std::collections::HashSet::::new(); let mut initialized = std::collections::HashSet::::new(); From 73824c6e13501a5f10786cb213daf4924ef1e6e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 05:22:40 +0200 Subject: [PATCH 34/53] perf(codegen): emit each global into the units that reference it, not all of them Codegen-unit splitting replicated EVERY string constant and global into EVERY unit, so per-unit IR grew with the unit COUNT: on the 13 MB Claude Code bundle each of 16 units still rendered ~400 MB (>6 GB total) and clang refused the translation unit outright ('ran out of source locations' / 'too large to process'), no matter how finely it was split. Splitting could not fix a floor that splitting itself multiplied. Now each bucket's function text is rendered first, its @symbol references collected, and a global is emitted only into units that reference it (unreferenced ones keep a home in unit 0). Definitions stay linkonce_odr so the linker folds the rare multi-unit case. An earlier variant emitted one definition plus declarations elsewhere; that is subtly wrong under -dead_strip, where the sole definition can be discarded with its unit's atoms while a live reference survives in another object - it showed up as an undefined _perry_null_guard_zero linking probe 07 at 4 units. Reference-scoped emission avoids the linkage question entirely. gc-ratchet probes 8/8 at 1, 4 and 8 units; codegen suite 418/418. --- crates/perry-codegen/src/module.rs | 128 +++++++++++++++++++++++++---- 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 24f7f88abc..dcfa42b895 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -44,6 +44,39 @@ fn strip_leading_linkage(s: &str) -> &str { /// single copy when the same global is emitted into multiple units. `external` /// declarations (no initializer) are returned unchanged — duplicating a /// declaration is harmless. + +/// Symbol name of a global/string definition line (`@name = ...`). +fn global_symbol_name(line: &str) -> Option<&str> { + let line = line.trim_start(); + if !line.starts_with('@') { + return None; + } + let end = line.find(" = ")?; + Some(&line[..end]) +} + +/// Collect every `@symbol` referenced in a chunk of IR text. +fn collect_symbol_refs(text: &str, out: &mut HashSet) { + let b = text.as_bytes(); + let mut i = 0usize; + while i < b.len() { + if b[i] == b'@' { + let start = i; + i += 1; + while i < b.len() + && (b[i].is_ascii_alphanumeric() || matches!(b[i], b'_' | b'.' | b'$' | b'-')) + { + i += 1; + } + if i > start + 1 { + out.insert(text[start..i].to_string()); + } + } else { + i += 1; + } + } +} + fn promote_global_for_units(line: &str) -> String { if line.contains(" = external ") { return line.to_string(); @@ -668,8 +701,52 @@ impl LlModule { .or_insert_with(|| declare_line_for(f)); } + // #7174 (real-app scaling): render each bucket's functions first, then + // give every global/string exactly ONE defining unit and hand the rest + // an `external` declaration. Replicating all definitions into every + // unit made per-unit IR grow with unit COUNT — on the 13 MB Claude Code + // bundle that meant ~400 MB units and `clang: translation unit is too + // large ... ran out of source locations`, no matter how finely it was + // split. Definitions are already `linkonce_odr` (visible), so an + // external declaration resolves to the same symbol at link time. + let bucket_texts: Vec = buckets + .iter() + .map(|bucket| { + let mut t = String::new(); + for func in bucket { + t.push_str(&render_fn_external(func)); + t.push('\n'); + } + t + }) + .collect(); + let bucket_refs: Vec> = bucket_texts + .iter() + .map(|t| { + let mut refs = HashSet::new(); + collect_symbol_refs(t, &mut refs); + refs + }) + .collect(); + + // A global is emitted into every unit that REFERENCES it — normally + // exactly one, and `linkonce_odr` lets the linker fold the rare + // multi-unit case. Definition-in-one-unit + `external` elsewhere was + // tried first and is subtly wrong under `-dead_strip`: the sole + // definition can be discarded with its unit's atoms while a live + // reference survives in another object. + let all_globals: Vec<&String> = + shared_strings.iter().chain(shared_globals.iter()).collect(); + let referenced_anywhere: Vec = all_globals + .iter() + .map(|def| { + global_symbol_name(def) + .is_some_and(|nm| bucket_refs.iter().any(|refs| refs.contains(nm))) + }) + .collect(); + let mut units = Vec::with_capacity(n); - for bucket in &buckets { + for (bi, bucket) in buckets.iter().enumerate() { let defined: HashSet<&str> = bucket.iter().map(|f| f.name.as_str()).collect(); let mut ir = String::new(); ir.push_str("; Generated by perry-codegen (codegen unit)\n"); @@ -680,14 +757,15 @@ impl LlModule { ir.push_str("module asm \".no_dead_strip __LLVM_StackMaps\"\n\n"); } - for sc in &shared_strings { - ir.push_str(sc); - ir.push('\n'); - } - ir.push('\n'); - for g in &shared_globals { - ir.push_str(g); - ir.push('\n'); + for (gi, def) in all_globals.iter().enumerate() { + let referenced = + global_symbol_name(def).is_some_and(|nm| bucket_refs[bi].contains(nm)); + // Unreferenced globals (anchors, `llvm.*`, appending lists) + // keep a home in unit 0 so nothing is lost. + if referenced || (!referenced_anywhere[gi] && bi == 0) { + ir.push_str(def); + ir.push('\n'); + } } ir.push('\n'); @@ -707,10 +785,7 @@ impl LlModule { } ir.push('\n'); - for func in bucket { - ir.push_str(&render_fn_external(func)); - ir.push('\n'); - } + ir.push_str(&bucket_texts[bi]); self.push_attrs_and_metadata(&mut ir); units.push(ir); @@ -766,10 +841,31 @@ mod tests { .unwrap(); assert!(u_with_f.contains("declare double @perry_fn_m__g()")); - // Shared globals appear in BOTH units, promoted to linkonce_odr. + // #7174: each shared global is DEFINED exactly once across units; + // units that reference it get an `external` declaration instead of a + // copy. Replicating definitions made per-unit IR grow with the unit + // count and broke clang's translation-unit limit on real bundles. + let global_defs = units + .iter() + .filter(|u| u.contains("@perry_global_x = linkonce_odr global double 0.0")) + .count(); + assert_eq!(global_defs, 1, "global must be defined in exactly one unit"); + let str_defs = units + .iter() + .filter(|u| u.contains("@.str.0 = linkonce_odr unnamed_addr constant")) + .count(); + assert_eq!(str_defs, 1, "string must be defined in exactly one unit"); + + // Every unit that mentions the symbol either defines it or declares it + // external — never neither. for u in &units { - assert!(u.contains("@perry_global_x = linkonce_odr global double 0.0")); - assert!(u.contains("@.str.0 = linkonce_odr unnamed_addr constant")); + if u.contains("@perry_global_x") { + assert!( + u.contains("@perry_global_x = linkonce_odr global double 0.0") + || u.contains("@perry_global_x = external global double"), + "referencing unit must define or externally declare the global" + ); + } assert!(u.contains("declare void @js_console_log_number(double)")); assert!(u.contains("target triple = \"arm64-apple-macosx15.0.0\"")); } From b6f024fd29cebf9e1d1a8e2514e610fd7e38e5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 05:51:08 +0200 Subject: [PATCH 35/53] style: cargo fmt --- crates/perry-codegen/src/function.rs | 21 ++++++++++++------- crates/perry-codegen/src/linker.rs | 11 ++++++---- crates/perry-runtime/src/gc/policy.rs | 16 +++++++------- .../perry-runtime/src/gc/roots/stack_maps.rs | 21 ++++++++++--------- .../src/commands/compile/object_cache.rs | 5 +---- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 5e6423ac9a..61290847de 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -983,15 +983,11 @@ impl PreciseRootBackend { } } - /// RS4GC surgery (#7174): retype root allocas to `ptr addrspace(1)` and cast /// at every recognized load/store site. Returns `None` when any root alloca /// appears in an unrecognized shape (the caller falls back to the explicit /// statepoint backend for the whole function). -fn lower_roots_for_rs4gc( - lines: &[&str], - root_ptrs: &[String], -) -> Option { +fn lower_roots_for_rs4gc(lines: &[&str], root_ptrs: &[String]) -> Option { let roots: std::collections::HashSet<&str> = root_ptrs.iter().map(String::as_str).collect(); let mut out = String::with_capacity(lines.len() * 48 + root_ptrs.len() * 96); let mut cast_counter = 0usize; @@ -1046,7 +1042,12 @@ fn lower_roots_for_rs4gc( break; } } - if trimmed == format!("{} = load i64, ptr {ptr}", trimmed.split(' ').next().unwrap_or("")) { + if trimmed + == format!( + "{} = load i64, ptr {ptr}", + trimmed.split(' ').next().unwrap_or("") + ) + { let result = trimmed.split(' ').next().unwrap_or(""); out.push_str(&format!( " {result}.rs4p = load ptr addrspace(1), ptr {ptr}\n {result} = ptrtoint ptr addrspace(1) {result}.rs4p to i64\n" @@ -1054,7 +1055,12 @@ fn lower_roots_for_rs4gc( handled = true; break; } - if trimmed == format!("{} = load double, ptr {ptr}", trimmed.split(' ').next().unwrap_or("")) { + if trimmed + == format!( + "{} = load double, ptr {ptr}", + trimmed.split(' ').next().unwrap_or("") + ) + { let result = trimmed.split(' ').next().unwrap_or(""); out.push_str(&format!( " {result}.rs4p = load ptr addrspace(1), ptr {ptr}\n {result}.rs4i = ptrtoint ptr addrspace(1) {result}.rs4p to i64\n {result} = bitcast i64 {result}.rs4i to double\n" @@ -1458,7 +1464,6 @@ fn lower_precise_roots_to_native_stack( } } - // Insert before calls, not after. Rebuild the tail when the line just // appended is a call so the intrinsic's instruction offset is the // actual call-site offset in the final machine function. diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index 7fde33cf5e..c8af2f4318 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -470,10 +470,13 @@ fn maybe_rs4gc_preprocess(ll_text: &str) -> Result> { .ok() .filter(|p| p.exists()) .or_else(|| { - ["/opt/homebrew/opt/llvm/bin/opt", "/usr/local/opt/llvm/bin/opt"] - .iter() - .map(PathBuf::from) - .find(|p| p.exists()) + [ + "/opt/homebrew/opt/llvm/bin/opt", + "/usr/local/opt/llvm/bin/opt", + ] + .iter() + .map(PathBuf::from) + .find(|p| p.exists()) }) .or_else(|| which_in_path("opt")) .context( diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 39f63d0e6b..5ad676ee96 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -715,11 +715,13 @@ pub(super) enum SafepointOnlyContract { pub(super) fn gc_safepoint_only_contract() -> SafepointOnlyContract { use std::sync::OnceLock; static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(|| match std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref() { - Ok("1") | Ok("on") | Ok("true") => SafepointOnlyContract::Heal, - Ok("strict") => SafepointOnlyContract::Strict, - _ => SafepointOnlyContract::Off, - }) + *CACHED.get_or_init( + || match std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref() { + Ok("1") | Ok("on") | Ok("true") => SafepointOnlyContract::Heal, + Ok("strict") => SafepointOnlyContract::Strict, + _ => SafepointOnlyContract::Off, + }, + ) } /// Contract enforcement chokepoint, called once at every synchronous @@ -736,9 +738,7 @@ pub(super) fn contract_scan_heal_guard() -> Option = OnceLock::new(); const DWARF_REG_FP_AARCH64: u16 = 29; @@ -188,8 +187,12 @@ impl StackMapIndex { if ip.abs_diff(candidate_pc) > MAX_SAFEPOINT_RETURN_DELTA { return &[]; } - let first = self.records.partition_point(|record| record.pc < candidate_pc); - let last = self.records.partition_point(|record| record.pc <= candidate_pc); + let first = self + .records + .partition_point(|record| record.pc < candidate_pc); + let last = self + .records + .partition_point(|record| record.pc <= candidate_pc); &self.records[first..last] } } @@ -246,7 +249,8 @@ fn verify_visit( unwind_addresses.sort_unstable(); unwind_addresses.dedup(); assert_eq!( - fast_addresses, unwind_addresses, + fast_addresses, + unwind_addresses, "PERRY_STACKMAP_WALKER=verify: fast walk visited {} unique slots, \ unwinder visited {}", fast_addresses.len(), @@ -777,8 +781,7 @@ mod fp_chain { if caller_fp == 0 { return None; } - stats.records_matched = - stats.records_matched.saturating_add(matched.len()); + stats.records_matched = stats.records_matched.saturating_add(matched.len()); for record in matched { // LLVM's AArch64 frame keeps the [x29, x30] pair // at the top of the frame, so the caller's body @@ -788,8 +791,7 @@ mod fp_chain { .checked_add(16) .and_then(|top| top.checked_sub(record.stack_size as usize)); for location in &record.locations { - stats.locations_visited = - stats.locations_visited.saturating_add(1); + stats.locations_visited = stats.locations_visited.saturating_add(1); let base = if location.dwarf_reg == DWARF_REG_FP_AARCH64 { Some(caller_fp) } else { @@ -806,8 +808,7 @@ mod fp_chain { let Some(address) = address else { continue; }; - if address == 0 - || address & (std::mem::align_of::() - 1) != 0 + if address == 0 || address & (std::mem::align_of::() - 1) != 0 { continue; } diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index f1d08da76a..ddd7f595a7 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -803,10 +803,7 @@ fn compute_object_cache_key_with_env( "env_statepoints", env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), ); - h.field( - "env_rs4gc", - env_var("PERRY_RS4GC").as_deref().unwrap_or(""), - ); + h.field("env_rs4gc", env_var("PERRY_RS4GC").as_deref().unwrap_or("")); // Explicit-safepoint contract: flips audited AllocNoReentry helpers // between statepoint and plain call. Two arms sharing a cached object // would make the contract's metadata reduction unmeasurable. From ea980ad6d9e12950823052efbc7ce7ec994af8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 06:10:48 +0200 Subject: [PATCH 36/53] perf(gc): decode the prologue to recover SP, re-enabling the fast walker on Linux (#7173) The Pi 5's +14.7% was DWARF CFI parsing: every collection walked the stack with the platform unwinder because SP-relative statepoint spills were unrecoverable off Darwin. Disassembly had shown x29 = sp + K with K VARYING per function (0x30, 0x60 in adjacent functions), which killed the constant-formula approach - but K is not unknowable, it is encoded in the prologue's own 'add x29, sp, #imm', and the stack-map header already gives every record its function's start address. The walker now decodes that instruction (mask 0xFFC003FF, pattern 0x910003FD, immediate in bits 21:10; encoding verified against both observed prologues) and takes the body SP as fp - imm. Bounded prologue scan, stops at 'ret', fails closed to the platform unwinder when the pattern is absent. Decoding happens per FRAME in the walker, never at index time: deciding chain-walkability up front would dereference every function address at startup, which segfaults on records whose addresses are not live code. macOS statepoint probes 8/8 run and 8/8 under PERRY_STACKMAP_WALKER=verify (prologue-decoded SP agrees with the unwinder on every slot). --- .../perry-runtime/src/gc/roots/stack_maps.rs | 140 ++++++++++-------- 1 file changed, 81 insertions(+), 59 deletions(-) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 7b4ab54f39..1b61db7645 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -31,10 +31,11 @@ struct StackMapLocation { #[derive(Clone, Debug, Eq, PartialEq)] struct StackMapRecord { pc: usize, + /// Start address of the containing function, from the stack-map header. + /// Used to decode that function's prologue when an SP-relative location + /// needs the FP-to-SP offset (see `fp_to_sp_offset`). + function_address: usize, /// The containing function's total frame size from the stack-map header. - /// LLVM's AArch64 frame places the `[x29, x30]` pair at the top of the - /// frame, so a chain walker can reconstruct the body SP as - /// `fp + 16 - stack_size` for SP-relative locations. stack_size: u64, locations: Vec, } @@ -131,19 +132,18 @@ fn stack_maps() -> &'static StackMapIndex { } fn index_records(records: Vec) -> StackMapIndex { + // SP-relative locations are admitted here and resolved per FRAME in the + // walker, which decodes the owning function's `add x29, sp, #imm` + // prologue to get the body SP (#7173). Deciding it here would mean + // dereferencing every function address at startup — unsafe for records + // whose addresses are not live code, and unnecessary because the walker + // already fails closed to the platform unwinder on any anomaly. let chain_walkable = records.iter().all(|record| { record.locations.iter().all(|location| { - location.dwarf_reg == DWARF_REG_FP_AARCH64 - // `SP = FP + 16 - stack_size` is the DARWIN AArch64 frame - // layout ([x29, x30] at the top of the frame). Verified wrong - // on aarch64-Linux by the Pi's verify-walker run: fast walk - // and unwinder disagreed by exactly the layout delta on the - // same slot. Until the Linux constant is DERIVED (not - // ported), SP-relative locations disqualify the fast chain - // off-Darwin and the always-correct unwinder serves instead. - || (cfg!(target_os = "macos") - && location.dwarf_reg == DWARF_REG_SP_AARCH64 - && record.stack_size >= 16) + matches!( + location.dwarf_reg, + DWARF_REG_FP_AARCH64 | DWARF_REG_SP_AARCH64 + ) }) }); let min_pc = records.first().map_or(usize::MAX, |record| record.pc); @@ -156,6 +156,48 @@ fn index_records(records: Vec) -> StackMapIndex { } } +/// Recover a function's frame-pointer-to-stack-pointer offset by decoding its +/// prologue (#7173). +/// +/// AArch64 prologues set the frame pointer with a single +/// `add x29, sp, #imm` after saving the `[x29, x30]` pair, so the body SP is +/// `fp - imm`. On Darwin that offset is a constant (the pair sits at the top +/// of the frame) but on Linux it varies per function with the callee-save +/// area laid out below the pair — measured 0x30 and 0x60 in adjacent +/// generated functions, which is why no `(fp, stack_size)` formula works +/// there and the fast chain previously fell back to the DWARF unwinder for +/// every collection (~22% of samples on a Pi 5). +/// +/// Instruction encoding: ADD (immediate, 64-bit, shift 0) with Rn = 31 (sp) +/// and Rd = 29 (fp) — `word & 0xFFC0_03FF == 0x9100_03FD`, immediate in bits +/// [21:10]. Scans a bounded prologue window and fails closed (`None`) if the +/// pattern is absent, in which case the caller uses the platform unwinder. +#[cfg(target_arch = "aarch64")] +fn fp_to_sp_offset(function_address: usize) -> Option { + const ADD_FP_SP_MASK: u32 = 0xFFC0_03FF; + const ADD_FP_SP_PATTERN: u32 = 0x9100_03FD; + const PROLOGUE_WINDOW_INSNS: usize = 24; + if function_address == 0 || function_address & 0x3 != 0 { + return None; + } + for i in 0..PROLOGUE_WINDOW_INSNS { + let word = unsafe { std::ptr::read((function_address + i * 4) as *const u32) }; + if word & ADD_FP_SP_MASK == ADD_FP_SP_PATTERN { + return Some(((word >> 10) & 0xFFF) as usize); + } + // `ret` ends the prologue window for a leaf that never sets up fp. + if word == 0xD65F_03C0 { + break; + } + } + None +} + +#[cfg(not(target_arch = "aarch64"))] +fn fp_to_sp_offset(_function_address: usize) -> Option { + None +} + fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { let insertion = maps.partition_point(|record| record.pc < ip); let before = insertion @@ -351,6 +393,7 @@ fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { out.push(StackMapRecord { pc: function_address.checked_add(instruction_offset)?, + function_address, stack_size: function_stack_size, locations, }); @@ -783,13 +826,11 @@ mod fp_chain { } stats.records_matched = stats.records_matched.saturating_add(matched.len()); for record in matched { - // LLVM's AArch64 frame keeps the [x29, x30] pair - // at the top of the frame, so the caller's body - // SP is its fp + 16 - stack_size. `chain_walkable` - // guaranteed stack_size >= 16 for SP records. - let sp = caller_fp - .checked_add(16) - .and_then(|top| top.checked_sub(record.stack_size as usize)); + // Body SP = fp - (prologue's `add x29, sp, #imm`). + // `chain_walkable` proved this decodes for every + // SP-relative record in the image (#7173). + let sp = fp_to_sp_offset(record.function_address) + .and_then(|off| caller_fp.checked_sub(off)); for location in &record.locations { stats.locations_visited = stats.locations_visited.saturating_add(1); let base = if location.dwarf_reg == DWARF_REG_FP_AARCH64 { @@ -896,6 +937,7 @@ mod tests { records, vec![StackMapRecord { pc: 0x1010, + function_address: 0x1000, stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, @@ -929,6 +971,7 @@ mod tests { records, vec![StackMapRecord { pc: 0x1020, + function_address: 0x1000, stack_size: 32, locations: vec![StackMapLocation { dwarf_reg: 29, @@ -950,52 +993,29 @@ mod tests { } #[test] - fn chain_walkable_index_accepts_fp_and_sized_sp_locations_only() { - let fp_record = StackMapRecord { - pc: 0x1000, - stack_size: 0, - locations: vec![StackMapLocation { - dwarf_reg: DWARF_REG_FP_AARCH64, - offset: -8, - }], - }; - let sp_record = StackMapRecord { - pc: 0x2000, + fn chain_walkable_index_accepts_fp_and_sp_locations_only() { + let rec = |pc: usize, reg: u16| StackMapRecord { + pc, + function_address: pc, stack_size: 160, locations: vec![StackMapLocation { - dwarf_reg: DWARF_REG_SP_AARCH64, - offset: 16, - }], - }; - let frameless_sp_record = StackMapRecord { - pc: 0x3000, - stack_size: 0, - locations: vec![StackMapLocation { - dwarf_reg: DWARF_REG_SP_AARCH64, - offset: 8, - }], - }; - let other_reg_record = StackMapRecord { - pc: 0x4000, - stack_size: 160, - locations: vec![StackMapLocation { - dwarf_reg: 1, - offset: 0, + dwarf_reg: reg, + offset: -8, }], }; - - let walkable = index_records(vec![fp_record.clone(), sp_record.clone()]); + // FP and SP are both walkable: SP resolves per frame by decoding the + // owning function's prologue (#7173). + let walkable = index_records(vec![ + rec(0x1000, DWARF_REG_FP_AARCH64), + rec(0x2000, DWARF_REG_SP_AARCH64), + ]); assert!(walkable.chain_walkable); assert_eq!(walkable.min_pc, 0x1000); assert_eq!(walkable.max_pc, 0x2000); - - assert!( - !index_records(vec![fp_record.clone(), frameless_sp_record]).chain_walkable, - "an SP location without a usable frame size must disable the fast walk" - ); + // Any other register disqualifies the whole image. assert!( - !index_records(vec![fp_record, other_reg_record]).chain_walkable, - "any non-FP/SP register must disable the fast walk" + !index_records(vec![rec(0x1000, DWARF_REG_FP_AARCH64), rec(0x3000, 1)]).chain_walkable, + "a non-FP/SP register must disable the fast walk" ); } @@ -1004,11 +1024,13 @@ mod tests { let maps = vec![ StackMapRecord { pc: 0x1000, + function_address: 0x1000, stack_size: 32, locations: Vec::new(), }, StackMapRecord { pc: 0x1020, + function_address: 0x1020, stack_size: 32, locations: Vec::new(), }, From f7940a294d93188c14188681f6dd8d52875f947a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 06:54:12 +0200 Subject: [PATCH 37/53] fix(codegen): close global-to-global references transitively when splitting units MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A global's initializer can name another global — a string header pointing at its payload, a closure record naming its thunk. Scoping emission to function-text references alone therefore under-approximated what a unit needs, and the 13 MB bundle failed with 'use of undefined value @..._.str.10138.bytes'. Each unit's reference set is now closed transitively over global initializers before deciding what to emit. Also declares the safepoint-contract heal as its own ConservativeScanSite (#7148's census enumerates every conservative-scan site; main added the argument during the rebase). Probes 8/8 at 1, 4 and 8 units; codegen suite 526/526. --- crates/perry-codegen/src/module.rs | 44 +++++++++++++++++--- crates/perry-runtime/src/gc/policy.rs | 4 +- crates/perry-runtime/src/gc/scan_fallback.rs | 17 ++++++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index dcfa42b895..7a98319bba 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -737,13 +737,48 @@ impl LlModule { // reference survives in another object. let all_globals: Vec<&String> = shared_strings.iter().chain(shared_globals.iter()).collect(); - let referenced_anywhere: Vec = all_globals + // Globals reference OTHER globals in their initializers (a string + // header pointing at its `.bytes` payload, a closure record naming its + // thunk). Function-text references alone therefore under-approximate + // what a unit needs — the first cut emitted `@....str.N.bytes` nowhere + // and clang rejected the unit with "use of undefined value". Close the + // reference set transitively per unit before deciding what to emit. + let global_index: std::collections::HashMap<&str, usize> = all_globals + .iter() + .enumerate() + .filter_map(|(i, def)| global_symbol_name(def).map(|nm| (nm, i))) + .collect(); + let global_refs: Vec> = all_globals .iter() .map(|def| { - global_symbol_name(def) - .is_some_and(|nm| bucket_refs.iter().any(|refs| refs.contains(nm))) + let mut refs = HashSet::new(); + collect_symbol_refs(def, &mut refs); + refs }) .collect(); + let bucket_needs: Vec> = bucket_refs + .iter() + .map(|refs| { + let mut need: HashSet = refs + .iter() + .filter_map(|nm| global_index.get(nm.as_str()).copied()) + .collect(); + let mut work: Vec = need.iter().copied().collect(); + while let Some(gi) = work.pop() { + for nm in &global_refs[gi] { + if let Some(&next) = global_index.get(nm.as_str()) { + if need.insert(next) { + work.push(next); + } + } + } + } + need + }) + .collect(); + let referenced_anywhere: Vec = (0..all_globals.len()) + .map(|gi| bucket_needs.iter().any(|need| need.contains(&gi))) + .collect(); let mut units = Vec::with_capacity(n); for (bi, bucket) in buckets.iter().enumerate() { @@ -758,8 +793,7 @@ impl LlModule { } for (gi, def) in all_globals.iter().enumerate() { - let referenced = - global_symbol_name(def).is_some_and(|nm| bucket_refs[bi].contains(nm)); + let referenced = bucket_needs[bi].contains(&gi); // Unreferenced globals (anchors, `llvm.*`, appending lists) // keep a home in unit 0 so nothing is lost. if referenced || (!referenced_anywhere[gi] && bi == 0) { diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 5ad676ee96..43456e6d29 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -753,7 +753,9 @@ pub(super) fn contract_scan_heal_guard() -> Option usize { match self { @@ -92,6 +98,7 @@ impl ConservativeScanSite { Self::EmergencyReclaim => 2, Self::ManualCollect => 3, Self::ManualMinor => 4, + Self::SafepointContractHeal => 5, } } @@ -102,6 +109,7 @@ impl ConservativeScanSite { Self::EmergencyReclaim => "emergency_reclaim", Self::ManualCollect => "manual_collect", Self::ManualMinor => "manual_minor", + Self::SafepointContractHeal => "safepoint_contract_heal", } } @@ -110,9 +118,10 @@ impl ConservativeScanSite { /// collections a program pays for without asking for them. pub(crate) const fn is_automatic(self) -> bool { match self { - Self::OldReclaimAllocPoint | Self::NurseryChurnSlackValve | Self::EmergencyReclaim => { - true - } + Self::OldReclaimAllocPoint + | Self::NurseryChurnSlackValve + | Self::EmergencyReclaim + | Self::SafepointContractHeal => true, Self::ManualCollect | Self::ManualMinor => false, } } From c054599ce66757d78ccc8f1f4b2bf4e0a9bf3fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 08:49:42 +0200 Subject: [PATCH 38/53] perf(codegen): compile codegen units concurrently, bounded The split existed for peak memory (#5391) but the clang phase ran one unit at a time: the 13 MB Claude Code bundle measured 4,939 s wall against 4,672 s user - essentially single-threaded on a 10-core host, with the dominant phase serialized. Units are independent clang invocations, so they now run on a bounded worker pool (std::thread::scope, no new dependency). Bounded rather than one-thread-per-unit because each job parses a multi-hundred-megabyte translation unit; unbounded fan-out would trade wall time for an OOM and undo the peak-memory win the split was introduced for. Default is a quarter of available parallelism clamped to [1, 4]; PERRY_CODEGEN_UNIT_JOBS overrides. Codegen suite 526/526. --- crates/perry-codegen/src/linker.rs | 53 ++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index c8af2f4318..e8a171ac5f 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -729,9 +729,58 @@ pub fn compile_units_to_object(units: &[String], target_triple: Option<&str>) -> let pid = std::process::id(); let nonce = TEMP_NONCE_COUNTER.fetch_add(1, Ordering::Relaxed); + // Units are independent clang invocations, so compile them concurrently. + // Measured before this: the 13 MB Claude Code bundle spent 4,939 s wall + // against 4,672 s user — the split existed for memory (#5391) but the + // clang phase, which dominates, ran one unit at a time. + // + // Concurrency is BOUNDED rather than one-thread-per-unit: each job parses + // a multi-hundred-megabyte translation unit, so unbounded fan-out trades + // wall time for an OOM (and would undo the peak-memory win the split was + // introduced for). Default is a quarter of the machine's parallelism, + // clamped to [1, 4]; `PERRY_CODEGEN_UNIT_JOBS` overrides. + let jobs = std::env::var("PERRY_CODEGEN_UNIT_JOBS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|p| (p.get() / 4).clamp(1, 4)) + .unwrap_or(1) + }) + .min(units.len()); + + let mut compiled: Vec>>> = (0..units.len()).map(|_| None).collect(); + if jobs <= 1 { + for (i, unit) in units.iter().enumerate() { + compiled[i] = Some(compile_ll_to_object(unit, target_triple)); + } + } else { + let slots: Vec>>>> = (0..units.len()) + .map(|_| std::sync::Mutex::new(None)) + .collect(); + let next = std::sync::atomic::AtomicUsize::new(0); + std::thread::scope(|scope| { + for _ in 0..jobs { + scope.spawn(|| loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= units.len() { + break; + } + let out = compile_ll_to_object(&units[i], target_triple); + *slots[i].lock().expect("codegen-unit slot poisoned") = Some(out); + }); + } + }); + for (i, slot) in slots.into_iter().enumerate() { + compiled[i] = slot.into_inner().expect("codegen-unit slot poisoned"); + } + } + let mut obj_paths: Vec = Vec::with_capacity(units.len()); - for (i, unit) in units.iter().enumerate() { - let bytes = compile_ll_to_object(unit, target_triple) + for (i, result) in compiled.into_iter().enumerate() { + let bytes = result + .expect("every codegen unit is compiled") .with_context(|| format!("codegen unit {}/{} failed to compile", i + 1, units.len()))?; let p = tmp_dir.join(format!("perry_cgu_{}_{}_{}.o", pid, nonce, i)); fs::write(&p, &bytes) From 6f9939d49beac23712b9a3103969b59d13ca5b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 14:00:43 +0200 Subject: [PATCH 39/53] perf(codegen): scope each unit's declarations to what it references Splitting a module MULTIPLIED total IR instead of dividing it, because every unit carried the whole module's declaration list. Measured on benchmarks/app-patterns/kernels/batch.ts: one unit = 431 KB, four units = 885 KB (2.05x), with 2,972 declares (149 KB) per unit against 4-7 actual definitions. On the 13 MB Claude Code bundle each unit carried ~16,700 declares, which is why per-unit IR stayed above a gigabyte and clang rejected it with 'translation unit is too large ... ran out of source locations' (its SourceManager tops out near 2^31 bytes) at 6 units AND at 16 - more units could not fix a floor that more units also multiplied. Units now emit only the declarations they reference, reusing the same reference sets computed for the globals scoping, including names reached through the initializers of the globals a unit emits. Result on the same benchmark: four units = 299 KB (0.69x of a single unit, down from 2.05x), 31-71 declares per unit. Splitting now shrinks total work. TRAP for anyone extending this: collect_symbol_refs yields '@name' while decl_by_name is keyed on the bare name; comparing them directly filters EVERY declare and the build fails loudly (it did). gc-ratchet probes 8/8 shadow at 1, 4 and 8 units, and 8/8 statepoint at 4 units under forced evacuation + verification; codegen suite 526/526. --- crates/perry-codegen/src/module.rs | 42 +++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 7a98319bba..b64131fd79 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -803,9 +803,29 @@ impl LlModule { } ir.push('\n'); - // Declares for everything this unit references but does not define. + // Declares for everything this unit REFERENCES but does not + // define. Emitting the whole module's declaration list into every + // unit left a per-unit floor that splitting cannot reduce: a + // 24-function benchmark carried 2,972 declares (149 KB) per unit, + // and the 13 MB Claude Code bundle carried ~16,700 — which is how + // units stayed above a gigabyte and hit clang's 2^31 source-location + // ceiling ("translation unit is too large ... ran out of source + // locations") regardless of unit count. Referenced names include + // those reached through the initializers of the globals this unit + // emits, so the closure computed above feeds this filter too. + // `collect_symbol_refs` yields `@name`; `decl_by_name` is keyed on + // the bare name, so strip the sigil or nothing ever matches. + let mut needed: HashSet<&str> = bucket_refs[bi] + .iter() + .map(|nm| nm.trim_start_matches('@')) + .collect(); + for gi in &bucket_needs[bi] { + for nm in &global_refs[*gi] { + needed.insert(nm.trim_start_matches('@')); + } + } for (name, decl) in &decl_by_name { - if defined.contains(name) { + if defined.contains(name) || !needed.contains(*name) { continue; } ir.push_str(decl); @@ -900,7 +920,16 @@ mod tests { "referencing unit must define or externally declare the global" ); } - assert!(u.contains("declare void @js_console_log_number(double)")); + // Declares are now scoped to what a unit references (the + // whole-module declaration list was a per-unit floor that + // splitting could not reduce). A unit that calls the helper must + // still declare it. + if u.contains("call void @js_console_log_number") { + assert!( + u.contains("declare void @js_console_log_number(double)"), + "a unit calling the helper must declare it" + ); + } assert!(u.contains("target triple = \"arm64-apple-macosx15.0.0\"")); } } @@ -1052,7 +1081,12 @@ mod tests { m.declare_function("js_is_truthy", I32, &[DOUBLE]); for k in 0..2 { let f = m.define_function(format!("perry_fn_m__f{k}"), DOUBLE, vec![]); - f.create_block("entry").ret(DOUBLE, "0.0"); + let b = f.create_block("entry"); + // Reference the helper so the declare is genuinely needed: declares + // are scoped per unit now, and a test whose units never call the + // helper would assert nothing about its attribute group. + b.call(I32, "js_is_truthy", &[(DOUBLE, "0.0")]); + b.ret(DOUBLE, "0.0"); } let units = m.render_codegen_units(2); assert_eq!(units.len(), 2); From 65e6430898ba0da6ba7265adad4e3993334ef0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 2 Aug 2026 17:11:00 +0200 Subject: [PATCH 40/53] docs: Pi small-hardware gap closed and inverted (+14.72% -> -1.74%), with honest attribution Statepoints now beat shadow on every probe on the Pi 5, geo -1.74%. Correctness 8/8 forced-evac and 8/8 verify-walker (prologue-decoded SP agrees with the DWARF unwinder on aarch64-Linux). Attribution is NOT the walker: a same-binary fast-vs-unwind A/B on the two worst probes is a dead heat, so the DWARF parsing perf measured at ~22% is no longer hot. The other variable is the rebase onto main's GC work (#7192/#7196/#7148); shadow itself improved 469->429ms, which fits that explanation and not 'the walker fixed it'. Also records an instrument failure: a cycle-count grep reported 0 cycles for both arms after main changed the diag format; raw output shows 81.9 MB freed. A count that cannot fail is not evidence. --- docs/statepoint-gc-experiment.md | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index d0cbdaf292..c855158be8 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -566,6 +566,52 @@ The `-Os` downgrade (`module IR > 6 MB`) is NOT a confound — it applies to both arms, since shadow IR for the same program cannot be smaller than the statepoint arm's 1,083 MB. +## Pi 5 re-measurement after the prologue-decode walker + main rebase (2026-08-02) + +The small-hardware regression is **closed and inverted**. Same host (Pi 5, +aarch64 Linux, load <1), same method (9 interleaved reps, per-probe +medians), both arms cross-built from one tree and one runtime archive: + +| Probe | before: shadow / statepoint | after: shadow / statepoint | +|---|---:|---:| +| Nursery churn | 385.7 / 402.0 (+4.2%) | 392.4 / 391.9 (−0.1%) | +| Survivor promotion | 448.4 / 446.7 (−0.4%) | 286.1 / 277.1 (−3.1%) | +| Cross-gen writes | 428.4 / 461.0 (+7.6%) | 374.8 / 367.4 (−2.0%) | +| Dead after deep stack | 932.1 / 1145.0 (+22.8%) | 1039.2 / 1017.6 (−2.1%) | +| Closure capture | 334.6 / 361.6 (+8.0%) | 449.8 / 436.3 (−3.0%) | +| String retention | 275.7 / 374.1 (+35.7%) | 240.9 / 240.5 (−0.2%) | +| Array grow/evacuate | 362.8 / 483.1 (+33.1%) | 226.9 / 225.6 (−0.6%) | +| Map/set side tables | 978.9 / 1139.9 (+16.4%) | 1070.3 / 1040.8 (−2.8%) | +| **geometric mean** | **+14.72%** | **−1.74%** | + +Correctness first, as always: 8/8 under forced evacuation + verification +and 8/8 under `PERRY_STACKMAP_WALKER=verify` (fast x29 walk and the +platform unwinder visit the identical slot set) — the prologue-decoded SP +is right on the architecture where the constant-based approach was proven +impossible. + +**Attribution, stated honestly: the walker is NOT the cause of the +improvement.** A direct A/B on the two worst probes — same binary, fast +chain versus `PERRY_STACKMAP_WALKER=unwind` — is a dead heat (0.24 s vs +0.24 s; 1.01 s vs 1.01 s). The DWARF CFI parsing that `perf` measured at +~22% of samples is simply no longer hot. The other variable between the +two runs is the rebase onto main's 64 commits of GC work (root-store +dominance #7192, from-space protection and zeal #7196, and #7148's precise +safepoint drains replacing conservative-scan fallbacks), which plausibly +reduced how often the native stack is walked at all. Shadow itself got +faster on the same probes (469.2 → 429.2 ms geo), which is consistent with +that explanation and inconsistent with "the walker fixed it". + +So: the prologue decode is *correct and verified* and removes a real +fallback, but the measured win belongs to main's GC hardening. Both are +recorded rather than conflated. + +★ One instrument failure worth recording: a first attempt to count GC +cycles reported **0 cycles for both arms**, which would have made the whole +comparison vacuous. It was a grep pattern that no longer matched main's +changed diagnostic format — raw output shows 81.9 MB freed across 79 arena +blocks. Never trust a count without looking at what produced it. + **Conclusion, stated as the design law this branch keeps re-deriving:** *with an optimizing compiler between the source and the safepoint, root metadata without relocation semantics is unsound — per-call plain maps From 50c5229471383e3ce3ac2700fe4a918e9ec53c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 12:13:30 +0200 Subject: [PATCH 41/53] gc: compact the stack map, closing the statepoint file-size gap The statepoint backend's only losing axis was file size, and it was not generated code: on test-drizzle-pg the RS4GC arm's __text is 248 KB SMALLER than shadow's. The entire 3.5 MB loss is the __llvm_stackmaps section. Measured composition of that section (scripts/stackmap_anatomy.py, which asserts it parsed 100% of the bytes): 40.6% Constant location slots -- exactly 3 per record, gc.statepoint's CC / Flags / NumDeopt preamble 13.3% duplicate base/derived slots (Perry has no interior pointers) 18.0% record headers, incl. an 8-byte patchpoint ID nothing patches 11.3% inter-record padding The runtime already discarded the constants and collapsed the base/derived pair at parse time, so over half the section was shipped in the binary and thrown away at startup. LLVM's stack map is a JIT-patching wire format; an AOT collector needs {dwarf_reg, offset} per distinct root and nothing else. Compaction measured on drizzle (4,214,384 B, 124 concatenated maps, 1,717 functions, 33,406 records, 154,020 distinct roots): flat varint 387,199 B 10.9x + roots sorted and delta-encoded 286,258 B 14.7x + "same live set as previous record" flag 132,418 B 31.8x The last step is a fact about real programs rather than a coding trick: 77% of records have exactly the live set of the record before them, because consecutive safepoints in a function share their roots. The decoder points repeats at one copy instead of materialising 154k entries, so it shrinks the in-memory index too. Projected onto the measured RS4GC arm: ~28.20 MB against shadow's 28.47 MB, a ~271 KB win where there was a 3.5 MB loss. Statepoints then lead on all three axes -- wall-clock -0.93%, RSS flat, size -271 KB. The rewrite happens on assembly because that is where LLVM prints the map's function addresses as symbol NAMES (.quad _main). One text parser replaces Mach-O and ELF relocation parsing, llvm-objcopy, and a second link pass. Two facts settled that empirically: the address fields are external symbol relocations (otool -r: extern 1), so a separately assembled table resolves at link; and -S costs the same 0.04s as -c, because codegen is the cost and printing text is free. Only the statepoint backends emit a stack map, so only they pay for it. A module with no block, or one that does not parse, is assembled unchanged: falling back costs bytes, never roots. --- crates/perry-codegen/src/gc_map.rs | 560 ++++++++++++++++++ crates/perry-codegen/src/lib.rs | 1 + crates/perry-codegen/src/linker.rs | 104 +++- .../perry-runtime/src/gc/roots/stack_maps.rs | 498 +++++++++------- docs/statepoint-gc-experiment.md | 94 ++- scripts/stackmap_anatomy.py | 271 +++++++++ 6 files changed, 1297 insertions(+), 231 deletions(-) create mode 100644 crates/perry-codegen/src/gc_map.rs create mode 100644 scripts/stackmap_anatomy.py diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs new file mode 100644 index 0000000000..871182c763 --- /dev/null +++ b/crates/perry-codegen/src/gc_map.rs @@ -0,0 +1,560 @@ +//! Re-encode LLVM's stack-map section into Perry's compact GC map. +//! +//! # Why this exists +//! +//! `gc.statepoint` metadata is the statepoint backend's *only* losing axis +//! against the shadow stack. Measured on `test-drizzle-pg`: generated `__text` +//! is 248 KB **smaller** under statepoints, but `__llvm_stackmaps` adds 3.9 MB, +//! so the binary loses by 3.5 MB overall. +//! +//! Almost none of those bytes carry information an AOT collector can use. +//! Measured composition of that section: +//! +//! * **60% of all location slots are `Constant`** — exactly three per record, +//! `gc.statepoint`'s calling-convention / flags / num-deopt preamble. +//! * every root is recorded as a **(base, derived) pair**, and Perry has no +//! interior pointers, so half of the remainder is the same slot twice; +//! * each record carries a 16-byte header whose 8-byte **patchpoint ID** only +//! matters to a JIT that patches call sites, plus inter-record padding. +//! +//! The runtime already threw all of that away at startup (see +//! `perry-runtime/src/gc/roots/stack_maps.rs`): it kept `{dwarf_reg, offset}` +//! per distinct root and nothing else. This module simply stops shipping what +//! was always discarded. +//! +//! # Where the remaining win comes from +//! +//! Dropping the dead weight alone is ~11x. Two further facts about real +//! programs take it to ~32x: +//! +//! * roots within a record cluster in the frame, so **sorting by frame offset +//! and delta-encoding** them makes most roots a single byte; +//! * **77% of records have exactly the live set of the record before them** — +//! consecutive safepoints in a function usually share their roots — so a +//! repeat flag replaces the whole payload. +//! +//! On drizzle: 4,214,384 B -> 131,402 B, which turns the 3.5 MB file-size loss +//! into a ~271 KB win and lets the statepoint backend lead on size, speed and +//! RSS simultaneously. +//! +//! # Why the rewrite happens on assembly rather than the object +//! +//! `clang -S` prints the stack map as ordinary directives with the function +//! addresses as **symbol names in plain text** (`.quad _main`). Rewriting there +//! needs one text parser. Rewriting the object instead would need Mach-O *and* +//! ELF relocation parsing (the addresses are external relocations), plus +//! `llvm-objcopy` to drop the old section, plus a second link pass. +//! +//! It costs almost nothing: `-S` takes the same time as `-c` (the codegen is +//! the cost; printing text is free), and assembling the result is ~0.02s. + +use std::collections::HashMap; + +/// Magic at the start of every emitted blob. +pub const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; +/// Format version. Bump on any layout change — the runtime rejects others. +pub const GC_MAP_VERSION: u8 = 1; +/// Section the compact map is emitted into, and the label it is given. +const GC_MAP_LABEL: &str = "_perry_gc_map"; +const MACHO_SECTION: &str = "__PERRY_GCMAP,__perry_gcmap"; +const ELF_SECTION: &str = ".perry_gcmap,\"a\",@progbits"; + +/// LLVM stack-map v3 location kinds. Only these two describe a frame slot; +/// `Constant`/`ConstIndex` carry the statepoint preamble and `Register` cannot +/// be recovered at collection time (which is what made plain stack maps +/// unsound — see the experiment write-up). +const LOCATION_DIRECT: u8 = 2; +const LOCATION_INDIRECT: u8 = 3; + +/// One safepoint: where it is in its function, and which frame slots are live. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Record { + instruction_offset: u32, + /// `(dwarf_reg, frame_offset)`, deduplicated and sorted by frame offset. + roots: Vec<(u16, i32)>, +} + +/// One function's safepoints, keyed by the symbol the linker will relocate. +#[derive(Debug, Clone)] +struct FunctionMap { + symbol: String, + stack_size: u64, + records: Vec, +} + +/// Byte width contributed by each data directive LLVM emits in the block. +fn directive_width(directive: &str) -> Option { + match directive { + ".byte" => Some(1), + ".short" | ".value" | ".hword" => Some(2), + ".long" | ".word" => Some(4), + ".quad" | ".xword" => Some(8), + _ => None, + } +} + +/// The assembled bytes of the stack-map block, plus the byte offsets at which +/// a `.quad` referenced a symbol instead of a literal. +struct RawBlock { + start_line: usize, + end_line: usize, + bytes: Vec, + symbols: HashMap, +} + +fn parse_block(lines: &[&str]) -> Option { + let start_line = lines.iter().position(|line| { + let t = line.trim_start(); + t.starts_with(".section") && (t.contains("__LLVM_STACKMAPS") || t.contains(".llvm_stackmaps")) + })?; + + let mut bytes: Vec = Vec::new(); + let mut symbols: HashMap = HashMap::new(); + let mut end_line = lines.len(); + + for (index, raw) in lines.iter().enumerate().skip(start_line + 1) { + let line = raw.trim(); + // The block runs to the next section or to the Mach-O epilogue. + if line.starts_with(".section") || line.starts_with(".subsections_via_symbols") { + end_line = index; + break; + } + if line.is_empty() || line.starts_with('#') || line.starts_with("//") || line.ends_with(':') + { + continue; + } + let mut parts = line.splitn(2, char::is_whitespace); + let directive = parts.next().unwrap_or_default(); + let operand = parts.next().unwrap_or_default(); + let operand = operand + .split('#') + .next() + .unwrap_or_default() + .split("//") + .next() + .unwrap_or_default() + .trim(); + + // Alignment is real content: LLVM aligns every record, and skipping + // the padding desynchronises every offset that follows it. + if directive == ".p2align" || directive == ".align" { + let first = operand.split(',').next().unwrap_or_default().trim(); + let value: u32 = first.parse().ok()?; + let align = if directive == ".p2align" { + 1usize << value + } else { + value as usize + }; + while align > 1 && bytes.len() % align != 0 { + bytes.push(0); + } + continue; + } + + let Some(width) = directive_width(directive) else { + continue; + }; + match parse_int(operand) { + Some(value) => bytes.extend_from_slice(&value.to_le_bytes()[..width]), + None => { + // A symbolic `.quad`: the function address. Remember the name + // and reserve the slot so later offsets stay correct. + if width != 8 { + return None; + } + symbols.insert(bytes.len(), operand.to_string()); + bytes.extend_from_slice(&0u64.to_le_bytes()); + } + } + } + + Some(RawBlock { + start_line, + end_line, + bytes, + symbols, + }) +} + +fn parse_int(text: &str) -> Option { + let text = text.trim(); + if let Some(hex) = text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + return u64::from_str_radix(hex, 16).ok(); + } + if let Some(negative) = text.strip_prefix('-') { + return negative.parse::().ok().map(|v| (v as i64).wrapping_neg() as u64); + } + text.parse::().ok() +} + +fn read_u16(bytes: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(bytes.get(at..at + 2)?.try_into().ok()?)) +} + +fn read_u32(bytes: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) +} + +fn read_u64(bytes: &[u8], at: usize) -> Option { + Some(u64::from_le_bytes(bytes.get(at..at + 8)?.try_into().ok()?)) +} + +fn align_up(value: usize, alignment: usize) -> usize { + value.div_ceil(alignment) * alignment +} + +/// Decode every concatenated v3 map in the block. +/// +/// The section is a *sequence* of maps, one per object the linker saw — a +/// decoder that reads only the first header silently drops the rest, so this +/// walks until the bytes are consumed. +fn decode_v3(block: &RawBlock) -> Option> { + let bytes = &block.bytes; + let mut out: Vec = Vec::new(); + let mut pos = 0usize; + + while pos + 16 <= bytes.len() { + if bytes[pos] != 3 { + // Inter-map alignment padding. + pos += 1; + continue; + } + let function_count = read_u32(bytes, pos + 4)? as usize; + let constant_count = read_u32(bytes, pos + 8)? as usize; + let record_count = read_u32(bytes, pos + 12)? as usize; + pos += 16; + + let mut heads = Vec::with_capacity(function_count); + let mut expected = 0usize; + for _ in 0..function_count { + let symbol = block.symbols.get(&pos)?.clone(); + let stack_size = read_u64(bytes, pos + 8)?; + let records = read_u64(bytes, pos + 16)? as usize; + expected = expected.checked_add(records)?; + heads.push((symbol, stack_size, records)); + pos += 24; + } + if expected != record_count { + return None; + } + pos = pos.checked_add(constant_count.checked_mul(8)?)?; + + for (symbol, stack_size, count) in heads { + let mut records = Vec::with_capacity(count); + for _ in 0..count { + let record_start = pos; + let instruction_offset = read_u32(bytes, pos + 8)?; + let location_count = read_u16(bytes, pos + 14)? as usize; + pos += 16; + + let mut roots: Vec<(u16, i32)> = Vec::new(); + for _ in 0..location_count { + let kind = *bytes.get(pos)?; + let size = read_u16(bytes, pos + 2)?; + let dwarf_reg = read_u16(bytes, pos + 4)?; + let offset = read_u32(bytes, pos + 8)? as i32; + // Keep exactly what the collector keeps: 8-byte frame + // slots, with the base/derived pair collapsed to one. + if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) + && size == 8 + && !roots.contains(&(dwarf_reg, offset)) + { + roots.push((dwarf_reg, offset)); + } + pos += 12; + } + + pos = align_up(pos - record_start, 8) + record_start; + let live_out_count = read_u16(bytes, pos + 2)? as usize; + pos = pos + .checked_add(4)? + .checked_add(live_out_count.checked_mul(4)?)?; + pos = align_up(pos - record_start, 8) + record_start; + if pos > bytes.len() { + return None; + } + + roots.sort_unstable_by_key(|(_, offset)| *offset); + records.push(Record { + instruction_offset, + roots, + }); + } + out.push(FunctionMap { + symbol, + stack_size, + records, + }); + } + } + + if out.is_empty() { + None + } else { + Some(out) + } +} + +fn push_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8 & 0x7F) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +fn zigzag(value: i32) -> u64 { + ((value << 1) ^ (value >> 31)) as u32 as u64 +} + +/// DWARF register number for the stack pointer on aarch64; every other base +/// this backend emits is the frame pointer. +const DWARF_REG_SP_AARCH64: u16 = 31; + +fn encode_stream(functions: &[FunctionMap]) -> Vec { + let mut stream = Vec::new(); + for function in functions { + let mut previous_offset = 0u32; + let mut previous_roots: Option<&Vec<(u16, i32)>> = None; + for record in &function.records { + push_varint( + &mut stream, + u64::from(record.instruction_offset.wrapping_sub(previous_offset)), + ); + previous_offset = record.instruction_offset; + + if previous_roots == Some(&record.roots) { + // Repeat flag: the live set is the previous record's. + push_varint(&mut stream, 1); + continue; + } + push_varint(&mut stream, (record.roots.len() as u64) << 1); + + // Deltas are zigzagged rather than emitted raw. `decode_v3` sorts + // roots so they are non-negative in practice, but a raw negative + // delta sign-extends into a 10-byte varint and silently bloats the + // map — the format must not depend on an ordering invariant held + // somewhere else. + let mut previous: Option = None; + for (reg, offset) in &record.roots { + let base_bit = u64::from(*reg == DWARF_REG_SP_AARCH64); + let delta = match previous { + None => *offset, + Some(prev) => offset.wrapping_sub(prev), + }; + push_varint(&mut stream, (zigzag(delta) << 1) | base_bit); + previous = Some(*offset); + } + previous_roots = Some(&record.roots); + } + } + stream +} + +/// Assemble the emitted directives for one compact blob. +/// +/// Layout (little-endian), mirrored by the runtime decoder: +/// +/// ```text +/// 0 "PGCM" +/// 4 u8 version, u8 reserved, u16 reserved +/// 8 u32 function_count +/// 12 u32 total_len -- lets the runtime walk concatenated blobs +/// 16 function_count x { u64 address, u32 stack_size, u32 record_count } +/// varint stream (see `encode_stream`) +/// ``` +/// +/// The function table starts at 16 so every relocated address is 8-byte +/// aligned. +fn emit_asm(functions: &[FunctionMap], stream: &[u8], elf: bool) -> String { + let total_len = 16 + functions.len() * 16 + stream.len(); + let mut out = String::new(); + if elf { + out.push_str(&format!("\t.section\t{ELF_SECTION}\n")); + } else { + out.push_str(&format!("\t.section\t{MACHO_SECTION}\n")); + } + out.push_str("\t.p2align\t3\n"); + out.push_str(&format!("{GC_MAP_LABEL}:\n")); + out.push_str("\t.ascii\t\"PGCM\"\n"); + out.push_str(&format!("\t.byte\t{GC_MAP_VERSION}\n")); + out.push_str("\t.byte\t0\n"); + out.push_str("\t.short\t0\n"); + out.push_str(&format!("\t.long\t{}\n", functions.len())); + out.push_str(&format!("\t.long\t{total_len}\n")); + for function in functions { + out.push_str(&format!("\t.quad\t{}\n", function.symbol)); + out.push_str(&format!("\t.long\t{}\n", function.stack_size as u32)); + out.push_str(&format!("\t.long\t{}\n", function.records.len())); + } + for chunk in stream.chunks(32) { + let bytes: Vec = chunk.iter().map(|b| b.to_string()).collect(); + out.push_str(&format!("\t.byte\t{}\n", bytes.join(","))); + } + out +} + +/// Statistics for the caller to log — a compaction that silently did nothing +/// must be distinguishable from one that ran. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GcMapStats { + pub original_bytes: usize, + pub compact_bytes: usize, + pub functions: usize, + pub records: usize, + pub roots: usize, +} + +/// Rewrite the LLVM stack-map block in `asm` into the compact map. +/// +/// Returns `None` when there is no stack-map block to rewrite (the common case +/// for a module without safepoints) or when the block does not parse — a +/// module whose metadata we do not fully understand keeps LLVM's section +/// rather than shipping a map that might be missing roots. +pub fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats)> { + let lines: Vec<&str> = asm.lines().collect(); + let block = parse_block(&lines)?; + let functions = decode_v3(&block)?; + let stream = encode_stream(&functions); + + let stats = GcMapStats { + original_bytes: block.bytes.len(), + compact_bytes: 16 + functions.len() * 16 + stream.len(), + functions: functions.len(), + records: functions.iter().map(|f| f.records.len()).sum(), + roots: functions + .iter() + .flat_map(|f| f.records.iter()) + .map(|r| r.roots.len()) + .sum(), + }; + + let replacement = emit_asm(&functions, &stream, elf); + let mut out = String::with_capacity(asm.len()); + for line in &lines[..block.start_line] { + // `.no_dead_strip` names the block's label from outside it. It is also + // the only thing keeping a section nothing references from being + // discarded, so retarget it instead of dropping it — without it the + // map is stripped and the collector finds no roots at all. + if line.contains(".no_dead_strip") && line.contains("__LLVM_StackMaps") { + out.push_str(&format!("\t.no_dead_strip\t{GC_MAP_LABEL}\n")); + } else { + out.push_str(line); + out.push('\n'); + } + } + out.push_str(&replacement); + for line in &lines[block.end_line..] { + if line.contains("__LLVM_StackMaps") { + continue; + } + out.push_str(line); + out.push('\n'); + } + Some((out, stats)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a minimal but structurally real v3 block: one function, one + /// record, two roots of which the second is the base/derived duplicate. + fn sample_asm() -> String { + let mut asm = String::new(); + asm.push_str("\t.no_dead_strip\t__LLVM_StackMaps\n"); + asm.push_str("\t.section\t__LLVM_STACKMAPS,__llvm_stackmaps\n"); + asm.push_str("__LLVM_StackMaps:\n"); + asm.push_str("\t.byte\t3\n\t.byte\t0\n\t.short\t0\n"); + asm.push_str("\t.long\t1\n"); // functions + asm.push_str("\t.long\t0\n"); // constants + asm.push_str("\t.long\t1\n"); // records + asm.push_str("\t.quad\t_probe_fn\n"); + asm.push_str("\t.quad\t144\n"); // stack size + asm.push_str("\t.quad\t1\n"); // record count + // record: id, instruction offset, reserved, location count + asm.push_str("\t.quad\t0\n"); + asm.push_str("\t.long\t64\n"); + asm.push_str("\t.short\t0\n"); + asm.push_str("\t.short\t4\n"); + // three statepoint preamble constants, then base/derived pair + for _ in 0..3 { + asm.push_str("\t.byte\t4\n\t.byte\t0\n\t.short\t8\n\t.short\t0\n\t.short\t0\n\t.long\t0\n"); + } + asm.push_str("\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n\t.short\t0\n\t.long\t4294967272\n"); + asm.push_str("\t.p2align\t3\n"); + asm.push_str("\t.short\t0\n\t.short\t0\n"); // live-out header + asm.push_str("\t.p2align\t3\n"); + asm.push_str("\t.subsections_via_symbols\n"); + asm + } + + #[test] + fn compacts_and_keeps_only_real_roots() { + let (out, stats) = compact_stack_map_asm(&sample_asm(), false).expect("block rewritten"); + assert_eq!(stats.functions, 1); + assert_eq!(stats.records, 1); + // Four locations in, one root out: three constants dropped. + assert_eq!(stats.roots, 1); + assert!( + stats.compact_bytes < stats.original_bytes, + "compact {} should beat original {}", + stats.compact_bytes, + stats.original_bytes + ); + assert!(out.contains("_perry_gc_map:")); + assert!(out.contains(".quad\t_probe_fn")); + // The old section must be gone, and nothing may still name its label. + assert!(!out.contains("__llvm_stackmaps")); + assert!(!out.contains("__LLVM_StackMaps")); + // The dead-strip guard must survive, retargeted. + assert!(out.contains(".no_dead_strip\t_perry_gc_map")); + } + + #[test] + fn repeated_live_sets_cost_one_byte() { + let shared = vec![(29u16, -24i32), (29, -32)]; + let functions = vec![FunctionMap { + symbol: "_f".to_string(), + stack_size: 64, + records: vec![ + Record { + instruction_offset: 0, + roots: shared.clone(), + }, + Record { + instruction_offset: 8, + roots: shared.clone(), + }, + Record { + instruction_offset: 16, + roots: shared, + }, + ], + }]; + let one_record = vec![FunctionMap { + symbol: functions[0].symbol.clone(), + stack_size: functions[0].stack_size, + records: functions[0].records[..1].to_vec(), + }]; + // The two extra records cost a delta byte plus a repeat byte each, + // regardless of how many roots the shared live set holds. + assert_eq!( + encode_stream(&functions).len(), + encode_stream(&one_record).len() + 4 + ); + } + + #[test] + fn no_stack_map_block_is_left_alone() { + assert!(compact_stack_map_asm("\t.section\t__TEXT,__text\n\tret\n", false).is_none()); + } + + #[test] + fn unparsable_block_keeps_llvm_section() { + // Truncated header: better to ship LLVM's section than a map that may + // be missing roots. + let asm = "\t.section\t__LLVM_STACKMAPS,__llvm_stackmaps\n\t.byte\t3\n"; + assert!(compact_stack_map_asm(asm, false).is_none()); + } +} diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index 45a9a626e9..866a9e2610 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -12,6 +12,7 @@ pub mod expr; pub mod ext_registry; pub mod function; pub(crate) mod gc_call_effects; +pub mod gc_map; pub mod linker; pub(crate) mod loop_purity; pub(crate) mod lower_array_method; diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index e8a171ac5f..343f0545e8 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -223,6 +223,10 @@ struct ClangCompilePlan { ll_path: PathBuf, obj_path: PathBuf, stderr_remarks_path: PathBuf, + /// Set when the stack map is being compacted: clang emits assembly here + /// instead of an object, the stack-map block is rewritten (see + /// `crate::gc_map`), and the result is assembled to `obj_path`. + asm_path: Option, } fn native_tuning_arg_for_host() -> &'static str { @@ -395,7 +399,21 @@ fn build_clang_compile_plan( "-O3" }; - let mut clang_args = vec!["-c".to_string(), opt_flag.to_string()]; + // Compacting the stack map means going through assembly, because that is + // where LLVM prints the map's function addresses as symbol *names* — the + // one form that needs neither relocation parsing nor a second link. Only + // the statepoint backends emit a stack map, so only they pay for it, and + // the cost is small: `-S` takes the same time as `-c` (codegen is the + // cost, printing text is free) and assembling is ~0.02s per module. + let compact_gc_map = crate::codegen::helpers::statepoints_enabled() + || crate::codegen::helpers::rs4gc_enabled(); + let asm_path = + compact_gc_map.then(|| PathBuf::from(format!("{}.s", obj_path.display()))); + + let mut clang_args = vec![ + if compact_gc_map { "-S" } else { "-c" }.to_string(), + opt_flag.to_string(), + ]; // A parameter rather than an env probe so a test can pin what `-g` does // and does not reach — measured in #7144: on a Perry `.ll` it produces a // byte-identical object with no `.debug_*` sections, because Perry's @@ -423,7 +441,13 @@ fn build_clang_compile_plan( } clang_args.push(ll_path.display().to_string()); clang_args.push("-o".to_string()); - clang_args.push(obj_path.display().to_string()); + clang_args.push( + asm_path + .as_ref() + .unwrap_or(&obj_path) + .display() + .to_string(), + ); clang_args.push("-target".to_string()); clang_args.push(effective_target.clone()); @@ -443,6 +467,7 @@ fn build_clang_compile_plan( ll_path, obj_path, stderr_remarks_path, + asm_path, } } @@ -510,6 +535,77 @@ fn maybe_rs4gc_preprocess(ll_text: &str) -> Result> { Ok(Some(String::from_utf8(output.stdout)?)) } +/// Rewrite the stack map in `asm_path` into Perry's compact form, then +/// assemble it to `obj_path`. +/// +/// A module with no stack-map block, or one whose block does not parse, is +/// assembled unchanged — LLVM's section is correct, merely large, so falling +/// back costs bytes rather than roots. +fn compact_gc_map_and_assemble( + plan: &ClangCompilePlan, + asm_path: &Path, + obj_path: &Path, +) -> Result<()> { + let asm = fs::read_to_string(asm_path) + .with_context(|| format!("Failed to read assembly at {}", asm_path.display()))?; + + let elf = !plan.effective_target.contains("apple") + && !plan.effective_target.contains("darwin") + && !plan.effective_target.contains("windows"); + if let Some((rewritten, stats)) = crate::gc_map::compact_stack_map_asm(&asm, elf) { + fs::write(asm_path, rewritten).with_context(|| { + format!("Failed to write compacted assembly at {}", asm_path.display()) + })?; + GC_MAP_ORIGINAL_BYTES.fetch_add(stats.original_bytes as u64, Ordering::Relaxed); + GC_MAP_COMPACT_BYTES.fetch_add(stats.compact_bytes as u64, Ordering::Relaxed); + log::debug!( + "perry-codegen: gc map {} -> {} bytes ({} functions, {} records, {} roots)", + stats.original_bytes, + stats.compact_bytes, + stats.functions, + stats.records, + stats.roots, + ); + } + + let output = Command::new(&plan.clang) + .arg("-c") + .arg(asm_path) + .arg("-o") + .arg(obj_path) + .arg("-target") + .arg(&plan.effective_target) + .output() + .with_context(|| format!("Failed to invoke {}", plan.clang.display()))?; + if !output.status.success() { + return Err(anyhow!( + "assembling the compacted stack map failed (status={}).\n\ + assembly left at: {}\n\ + \n\ + stderr:\n{}", + output.status, + asm_path.display(), + String::from_utf8_lossy(&output.stderr) + )); + } + let _ = fs::remove_file(asm_path); + Ok(()) +} + +/// Totals for the whole process, so a build can report what compaction did. +/// A run where these stay zero did not compact anything — the distinction a +/// gate needs in order to be able to fail. +static GC_MAP_ORIGINAL_BYTES: AtomicU64 = AtomicU64::new(0); +static GC_MAP_COMPACT_BYTES: AtomicU64 = AtomicU64::new(0); + +/// `(llvm_bytes, compact_bytes)` summed across every module compiled so far. +pub fn gc_map_compaction_totals() -> (u64, u64) { + ( + GC_MAP_ORIGINAL_BYTES.load(Ordering::Relaxed), + GC_MAP_COMPACT_BYTES.load(Ordering::Relaxed), + ) +} + fn which_in_path(name: &str) -> Option { std::env::var_os("PATH").and_then(|paths| { std::env::split_paths(&paths) @@ -671,6 +767,10 @@ fn compile_ll_to_object_in( )); } + if let Some(asm_path) = &plan.asm_path { + compact_gc_map_and_assemble(&plan, asm_path, &obj_path)?; + } + let bytes = fs::read(&obj_path) .with_context(|| format!("Failed to read clang output at {}", obj_path.display()))?; diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 1b61db7645..ecc2309348 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -7,7 +7,7 @@ //! in the emitted stack-map section. //! //! This first implementation deliberately targets macOS, where the experiment -//! is being measured. It discovers the concatenated `__LLVM_STACKMAPS` section +//! is being measured. It discovers the concatenated `__PERRY_GCMAP` section //! in the main Mach-O image and uses the platform unwinder to recover the //! frame-register value for each active generated frame. Unsupported targets //! return no roots; neither native-stack experiment may be used for correctness @@ -18,9 +18,13 @@ use crate::gc::telemetry::RootSourcesTraceStats; use std::ffi::c_void; use std::sync::OnceLock; -const STACK_MAP_VERSION: u8 = 3; -const LOCATION_DIRECT: u8 = 2; -const LOCATION_INDIRECT: u8 = 3; +/// Magic and version of the compact map the compiler emits +/// (`perry-codegen/src/gc_map.rs`). LLVM's own stack-map section is rewritten +/// at assembly time and never reaches the binary: >50% of it was the +/// statepoint constant preamble and base/derived duplicates that this parser +/// discarded anyway, and shipping it cost 3.9 MB on a real application. +const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; +const GC_MAP_VERSION: u8 = 1; const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct StackMapLocation { @@ -31,13 +35,20 @@ struct StackMapLocation { #[derive(Clone, Debug, Eq, PartialEq)] struct StackMapRecord { pc: usize, - /// Start address of the containing function, from the stack-map header. - /// Used to decode that function's prologue when an SP-relative location - /// needs the FP-to-SP offset (see `fp_to_sp_offset`). + /// Start address of the containing function, from the map's function + /// table. Used to decode that function's prologue when an SP-relative + /// location needs the FP-to-SP offset (see `fp_to_sp_offset`). function_address: usize, - /// The containing function's total frame size from the stack-map header. + /// The containing function's total frame size from the function table. stack_size: u64, - locations: Vec, + /// Half-open range into `StackMapIndex::roots`. + /// + /// A range rather than an owned `Vec` because **77% of records have the + /// identical live set as the record before them** — consecutive safepoints + /// in a function usually share their roots — so the decoder points the + /// repeats at one copy instead of duplicating 154k entries. + roots_start: u32, + roots_len: u32, } /// Parsed section plus the facts the fast walker's preconditions need. @@ -50,11 +61,22 @@ struct StackMapRecord { #[derive(Debug, Default)] struct StackMapIndex { records: Vec, + /// Every root slot, referenced by `StackMapRecord`'s range. Shared between + /// records whose live sets are identical. + roots: Vec, chain_walkable: bool, min_pc: usize, max_pc: usize, } +impl StackMapIndex { + fn locations(&self, record: &StackMapRecord) -> &[StackMapLocation] { + let start = record.roots_start as usize; + let end = start + record.roots_len as usize; + self.roots.get(start..end).unwrap_or(&[]) + } +} + static STACK_MAPS: OnceLock = OnceLock::new(); const DWARF_REG_FP_AARCH64: u16 = 29; @@ -125,31 +147,36 @@ fn stack_maps() -> &'static StackMapIndex { let Some(section) = loaded_stack_map_section() else { return StackMapIndex::default(); }; - let mut records = parse_concatenated_stack_maps(section).unwrap_or_default(); + let Some((mut records, roots)) = parse_gc_map(section) else { + return StackMapIndex::default(); + }; records.sort_unstable_by_key(|record| record.pc); - index_records(records) + index_records(records, roots) }) } -fn index_records(records: Vec) -> StackMapIndex { +fn index_records(records: Vec, roots: Vec) -> StackMapIndex { // SP-relative locations are admitted here and resolved per FRAME in the // walker, which decodes the owning function's `add x29, sp, #imm` // prologue to get the body SP (#7173). Deciding it here would mean // dereferencing every function address at startup — unsafe for records // whose addresses are not live code, and unnecessary because the walker // already fails closed to the platform unwinder on any anomaly. - let chain_walkable = records.iter().all(|record| { - record.locations.iter().all(|location| { - matches!( - location.dwarf_reg, - DWARF_REG_FP_AARCH64 | DWARF_REG_SP_AARCH64 - ) - }) + // The decoder only ever produces these two bases, but keep the check: it + // is what decides the fast walker is usable at all, and a format change + // that introduced a third base must disable the chain walk, not be + // trusted by it. + let chain_walkable = roots.iter().all(|location| { + matches!( + location.dwarf_reg, + DWARF_REG_FP_AARCH64 | DWARF_REG_SP_AARCH64 + ) }); let min_pc = records.first().map_or(usize::MAX, |record| record.pc); let max_pc = records.last().map_or(0, |record| record.pc); StackMapIndex { records, + roots, chain_walkable, min_pc, max_pc, @@ -302,104 +329,118 @@ fn verify_visit( stats } -fn parse_concatenated_stack_maps(bytes: &[u8]) -> Option> { - let mut all = Vec::new(); +/// Decode every concatenated compact map in the section. +/// +/// The linker concatenates one blob per object file, so this walks blob by +/// blob using each header's `total_len` rather than assuming a single map — +/// a decoder that reads only the first header silently drops every other +/// object's roots, which is invisible until a collection frees a live object. +fn parse_gc_map(bytes: &[u8]) -> Option<(Vec, Vec)> { + let mut records = Vec::new(); + let mut roots: Vec = Vec::new(); let mut base = 0usize; - while base < bytes.len() { - // Linkers preserve the input section's 8-byte alignment. Ignore a - // zero-filled tail, but do not search through malformed non-zero data. - if bytes[base..].iter().all(|byte| *byte == 0) { - break; + + while base + 16 <= bytes.len() { + if bytes.get(base..base + 4)? != GC_MAP_MAGIC { + // Linkers pad between input sections; a zero tail is the end. + if bytes[base..].iter().all(|byte| *byte == 0) { + break; + } + base += 1; + continue; } - let (mut records, consumed) = parse_one_stack_map(&bytes[base..])?; - if consumed == 0 { + if read_u8(bytes, base + 4)? != GC_MAP_VERSION { + return None; + } + let function_count = read_u32(bytes, base + 8)? as usize; + let total_len = read_u32(bytes, base + 12)? as usize; + let table = base.checked_add(16)?; + let stream_start = table.checked_add(function_count.checked_mul(16)?)?; + let blob_end = base.checked_add(total_len)?; + if blob_end > bytes.len() || stream_start > blob_end { return None; } - all.append(&mut records); - base = base.checked_add(consumed)?; - } - Some(all) -} - -fn parse_one_stack_map(bytes: &[u8]) -> Option<(Vec, usize)> { - if read_u8(bytes, 0)? != STACK_MAP_VERSION { - return None; - } - let function_count = read_u32(bytes, 4)? as usize; - let constant_count = read_u32(bytes, 8)? as usize; - let record_count = read_u32(bytes, 12)? as usize; - let mut offset = 16usize; - - let mut functions = Vec::with_capacity(function_count); - let mut expected_records = 0usize; - for _ in 0..function_count { - let address = read_u64(bytes, offset)? as usize; - let stack_size = read_u64(bytes, offset + 8)?; - let records = read_u64(bytes, offset + 16)? as usize; - functions.push((address, stack_size, records)); - expected_records = expected_records.checked_add(records)?; - offset = offset.checked_add(24)?; - } - if expected_records != record_count { - return None; - } - offset = offset.checked_add(constant_count.checked_mul(8)?)?; - if offset > bytes.len() { - return None; - } - let mut out = Vec::with_capacity(record_count); - for (function_address, function_stack_size, function_record_count) in functions { - for _ in 0..function_record_count { - let instruction_offset = read_u32(bytes, offset + 8)? as usize; - let location_count = read_u16(bytes, offset + 14)? as usize; - offset = offset.checked_add(16)?; - - let mut locations = Vec::new(); - for _ in 0..location_count { - let kind = read_u8(bytes, offset)?; - let size = read_u16(bytes, offset + 2)?; - let dwarf_reg = read_u16(bytes, offset + 4)?; - let location_offset = read_i32(bytes, offset + 8)?; - if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { - let location = StackMapLocation { - dwarf_reg, - offset: location_offset, - }; - // A statepoint records a base/derived pair for every - // relocation. Perry currently uses the same value for - // both, so LLVM commonly emits the exact same spill slot - // twice. Visit that physical word once. - if !locations.contains(&location) { - locations.push(location); + let mut cursor = stream_start; + for index in 0..function_count { + let entry = table + index * 16; + let function_address = read_u64(bytes, entry)? as usize; + let stack_size = u64::from(read_u32(bytes, entry + 8)?); + let record_count = read_u32(bytes, entry + 12)? as usize; + + let mut instruction_offset = 0u32; + let mut previous: Option<(u32, u32)> = None; + for _ in 0..record_count { + let (delta, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + instruction_offset = instruction_offset.wrapping_add(delta as u32); + + let (header, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + let range = if header & 1 == 1 { + // Repeat: this safepoint's live set is the previous one's. + previous? + } else { + let count = (header >> 1) as usize; + let start = u32::try_from(roots.len()).ok()?; + let mut last: Option = None; + for _ in 0..count { + let (value, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + let dwarf_reg = if value & 1 == 1 { + DWARF_REG_SP_AARCH64 + } else { + DWARF_REG_FP_AARCH64 + }; + let delta = unzigzag((value >> 1) as u32); + let offset = match last { + None => delta, + Some(previous_offset) => previous_offset.wrapping_add(delta), + }; + last = Some(offset); + roots.push(StackMapLocation { dwarf_reg, offset }); } - } - offset = offset.checked_add(12)?; + (start, u32::try_from(count).ok()?) + }; + previous = Some(range); + + records.push(StackMapRecord { + pc: function_address.checked_add(instruction_offset as usize)?, + function_address, + stack_size, + roots_start: range.0, + roots_len: range.1, + }); } + } - // LLVM aligns the live-out header independently from the whole - // record. This first padding is observable whenever the location - // count is odd (one Direct root is a common case). - offset = align_up(offset, 8)?; - // Two reserved bytes followed by the live-out count. - let live_out_count = read_u16(bytes, offset + 2)? as usize; - offset = offset - .checked_add(4)? - .checked_add(live_out_count.checked_mul(4)?)?; - offset = align_up(offset, 8)?; - if offset > bytes.len() { - return None; - } + base = align_up(blob_end, 8)?; + } - out.push(StackMapRecord { - pc: function_address.checked_add(instruction_offset)?, - function_address, - stack_size: function_stack_size, - locations, - }); + Some((records, roots)) +} + +/// LEB128 read bounded by the blob it belongs to, so a corrupt length cannot +/// walk into the next blob or off the section. +fn read_varint(bytes: &[u8], mut at: usize, end: usize) -> Option<(u64, usize)> { + let mut value = 0u64; + let mut shift = 0u32; + loop { + if at >= end || shift > 63 { + return None; } + let byte = *bytes.get(at)?; + at += 1; + value |= u64::from(byte & 0x7F) << shift; + if byte & 0x80 == 0 { + return Some((value, at)); + } + shift += 7; } - Some((out, offset)) +} + +fn unzigzag(value: u32) -> i32 { + ((value >> 1) as i32) ^ -((value & 1) as i32) } fn align_up(value: usize, alignment: usize) -> Option { @@ -412,24 +453,12 @@ fn read_u8(bytes: &[u8], offset: usize) -> Option { bytes.get(offset).copied() } -fn read_u16(bytes: &[u8], offset: usize) -> Option { - Some(u16::from_le_bytes( - bytes.get(offset..offset + 2)?.try_into().ok()?, - )) -} - fn read_u32(bytes: &[u8], offset: usize) -> Option { Some(u32::from_le_bytes( bytes.get(offset..offset + 4)?.try_into().ok()?, )) } -fn read_i32(bytes: &[u8], offset: usize) -> Option { - Some(i32::from_le_bytes( - bytes.get(offset..offset + 4)?.try_into().ok()?, - )) -} - fn read_u64(bytes: &[u8], offset: usize) -> Option { Some(u64::from_le_bytes( bytes.get(offset..offset + 8)?.try_into().ok()?, @@ -520,8 +549,8 @@ fn loaded_stack_map_section() -> Option<&'static [u8]> { let mut section_ptr = command_ptr.add(std::mem::size_of::()); for _ in 0..segment.section_count { let section = std::ptr::read_unaligned(section_ptr.cast::()); - if fixed_name_matches(§ion.segment_name, b"__LLVM_STACKMAPS") - && fixed_name_matches(§ion.section_name, b"__llvm_stackmaps") + if fixed_name_matches(§ion.segment_name, b"__PERRY_GCMAP") + && fixed_name_matches(§ion.section_name, b"__perry_gcmap") { let address = (section.address as isize).checked_add(slide)? as usize; let size = usize::try_from(section.size).ok()?; @@ -539,11 +568,11 @@ fn loaded_stack_map_section() -> Option<&'static [u8]> { None } -/// ELF (#7173): the `.llvm_stackmaps` section of the main executable. +/// ELF (#7173): the `.perry_gcmap` section of the main executable. /// /// Linker-provided `__start_`/`__stop_` symbols would need weak linkage /// (unstable in Rust) or `-rdynamic` (not guaranteed), so instead: read -/// `/proc/self/exe`'s section headers for `.llvm_stackmaps` (sh_addr, +/// `/proc/self/exe`'s section headers for `.perry_gcmap` (sh_addr, /// sh_size) and add the main object's load bias from the first /// `dl_iterate_phdr` callback. Runtime-verified gates for this path are /// pending a Linux host — tracked in #7173; the parser, index, matching, @@ -551,7 +580,7 @@ fn loaded_stack_map_section() -> Option<&'static [u8]> { #[cfg(target_os = "linux")] fn loaded_stack_map_section() -> Option<&'static [u8]> { let bytes = std::fs::read("/proc/self/exe").ok()?; - let (addr, size) = elf_section_vaddr(&bytes, b".llvm_stackmaps")?; + let (addr, size) = elf_section_vaddr(&bytes, b".perry_gcmap")?; let bias = main_object_load_bias()?; let start = bias.checked_add(addr)?; if start == 0 || size == 0 { @@ -680,7 +709,7 @@ mod unwind { } state.stats.records_matched = state.stats.records_matched.saturating_add(matched.len()); for record in matched { - for location in &record.locations { + for location in state.index.locations(record) { state.stats.locations_visited = state.stats.locations_visited.saturating_add(1); let base = _Unwind_GetGR(context, i32::from(location.dwarf_reg)); let address = if location.offset < 0 { @@ -831,7 +860,7 @@ mod fp_chain { // SP-relative record in the image (#7173). let sp = fp_to_sp_offset(record.function_address) .and_then(|off| caller_fp.checked_sub(off)); - for location in &record.locations { + for location in index.locations(record) { stats.locations_visited = stats.locations_visited.saturating_add(1); let base = if location.dwarf_reg == DWARF_REG_FP_AARCH64 { Some(caller_fp) @@ -887,154 +916,183 @@ mod fp_chain { mod tests { use super::*; - fn one_map_with_locations( - function: u64, - id: u64, - offset: u32, - locations: &[(u8, i32)], - ) -> Vec { + fn push_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8 & 0x7F) | 0x80); + value >>= 7; + } + out.push(value as u8); + } + + fn zigzag(value: i32) -> u64 { + ((value << 1) ^ (value >> 31)) as u32 as u64 + } + + /// Build one compact blob, mirroring `perry-codegen/src/gc_map.rs`. + /// `records` is `(instruction_offset, roots)`, roots as `(dwarf_reg, offset)`; + /// an empty root slice with `repeat` set encodes the repeat flag. + fn one_map(function: u64, records: &[(u32, Vec<(u16, i32)>, bool)]) -> Vec { + let mut stream = Vec::new(); + let mut previous_offset = 0u32; + for (instruction_offset, roots, repeat) in records { + push_varint(&mut stream, u64::from(instruction_offset.wrapping_sub(previous_offset))); + previous_offset = *instruction_offset; + if *repeat { + push_varint(&mut stream, 1); + continue; + } + push_varint(&mut stream, (roots.len() as u64) << 1); + let mut last: Option = None; + for (reg, offset) in roots { + let bit = u64::from(*reg == DWARF_REG_SP_AARCH64); + let delta = match last { + None => *offset, + Some(previous) => offset.wrapping_sub(previous), + }; + push_varint(&mut stream, (zigzag(delta) << 1) | bit); + last = Some(*offset); + } + } + + let total_len = 16 + 16 + stream.len(); let mut bytes = Vec::new(); - bytes.extend_from_slice(&[STACK_MAP_VERSION, 0, 0, 0]); - bytes.extend_from_slice(&1u32.to_le_bytes()); - bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(GC_MAP_MAGIC); + bytes.push(GC_MAP_VERSION); + bytes.extend_from_slice(&[0, 0, 0]); bytes.extend_from_slice(&1u32.to_le_bytes()); + bytes.extend_from_slice(&(total_len as u32).to_le_bytes()); bytes.extend_from_slice(&function.to_le_bytes()); - bytes.extend_from_slice(&32u64.to_le_bytes()); - bytes.extend_from_slice(&1u64.to_le_bytes()); - bytes.extend_from_slice(&id.to_le_bytes()); - bytes.extend_from_slice(&offset.to_le_bytes()); - bytes.extend_from_slice(&0u16.to_le_bytes()); - bytes.extend_from_slice(&(locations.len() as u16).to_le_bytes()); - for (kind, frame_offset) in locations { - bytes.push(*kind); - bytes.push(0); - bytes.extend_from_slice(&8u16.to_le_bytes()); - bytes.extend_from_slice(&29u16.to_le_bytes()); - bytes.extend_from_slice(&0u16.to_le_bytes()); - bytes.extend_from_slice(&frame_offset.to_le_bytes()); - } - while bytes.len() % 8 != 0 { - bytes.push(0); - } - bytes.extend_from_slice(&0u16.to_le_bytes()); - bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.extend_from_slice(&32u32.to_le_bytes()); + bytes.extend_from_slice(&(records.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&stream); while bytes.len() % 8 != 0 { bytes.push(0); } bytes } - fn one_map(function: u64, id: u64, offset: u32, frame_offset: i32) -> Vec { - one_map_with_locations(function, id, offset, &[(LOCATION_DIRECT, frame_offset)]) + fn simple(function: u64, offset: u32, frame_offset: i32) -> Vec { + one_map(function, &[(offset, vec![(29, frame_offset)], false)]) } #[test] - fn parses_direct_mutable_frame_location() { - let bytes = one_map(0x1000, 42, 0x10, -8); - let (records, consumed) = parse_one_stack_map(&bytes).expect("valid stack map"); - assert_eq!(consumed, bytes.len()); + fn decodes_frame_location() { + let bytes = simple(0x1000, 0x10, -8); + let (records, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].pc, 0x1010); + assert_eq!(records[0].function_address, 0x1000); + assert_eq!(records[0].stack_size, 32); assert_eq!( - records, - vec![StackMapRecord { - pc: 0x1010, - function_address: 0x1000, - stack_size: 32, - locations: vec![StackMapLocation { - dwarf_reg: 29, - offset: -8, - }], + roots, + vec![StackMapLocation { + dwarf_reg: 29, + offset: -8, }] ); } #[test] - fn parses_linker_concatenated_input_sections() { - let mut bytes = one_map(0x1000, 42, 0x10, -8); - bytes.extend_from_slice(&one_map(0x2000, 43, 0x20, -16)); - let records = parse_concatenated_stack_maps(&bytes).expect("concatenated maps"); + fn decodes_linker_concatenated_input_sections() { + let mut bytes = simple(0x1000, 0x10, -8); + bytes.extend_from_slice(&simple(0x2000, 0x20, -16)); + let (records, _) = parse_gc_map(&bytes).expect("concatenated maps"); assert_eq!(records.len(), 2); assert_eq!(records[0].pc, 0x1010); assert_eq!(records[1].pc, 0x2020); } #[test] - fn parses_and_deduplicates_statepoint_spill_locations() { - let bytes = one_map_with_locations( + fn repeated_live_sets_share_one_copy() { + // Three safepoints, the last two repeating the first's live set: the + // whole point of the format, and the reason the in-memory index does + // not hold 154k duplicated entries on a real application. + let bytes = one_map( 0x1000, - 7, - 0x20, - &[(LOCATION_INDIRECT, -16), (LOCATION_INDIRECT, -16)], + &[ + (0x10, vec![(29, -8), (29, -16)], false), + (0x20, vec![], true), + (0x30, vec![], true), + ], ); - let (records, consumed) = parse_one_stack_map(&bytes).expect("valid statepoint map"); - assert_eq!(consumed, bytes.len()); + let (records, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!(records.len(), 3); + assert_eq!(roots.len(), 2, "the repeats must not append new roots"); + for record in &records { + assert_eq!(record.roots_start, 0); + assert_eq!(record.roots_len, 2); + } + } + + #[test] + fn decodes_negative_and_ascending_root_offsets() { + let bytes = one_map(0x1000, &[(0, vec![(29, -64), (29, -8), (31, 24)], false)]); + let (_, roots) = parse_gc_map(&bytes).expect("valid map"); assert_eq!( - records, - vec![StackMapRecord { - pc: 0x1020, - function_address: 0x1000, - stack_size: 32, - locations: vec![StackMapLocation { - dwarf_reg: 29, - offset: -16, - }], - }] + roots, + vec![ + StackMapLocation { dwarf_reg: 29, offset: -64 }, + StackMapLocation { dwarf_reg: 29, offset: -8 }, + StackMapLocation { dwarf_reg: 31, offset: 24 }, + ] ); } #[test] fn rejects_truncated_or_wrong_version_sections() { - assert!(parse_one_stack_map(&[]).is_none()); - let mut bytes = one_map(0x1000, 42, 0x10, -8); - bytes[0] = 2; - assert!(parse_one_stack_map(&bytes).is_none()); - bytes[0] = STACK_MAP_VERSION; - bytes.truncate(bytes.len() - 1); - assert!(parse_one_stack_map(&bytes).is_none()); + assert!(parse_gc_map(&[]).is_none() || parse_gc_map(&[]).unwrap().0.is_empty()); + let mut bytes = simple(0x1000, 0x10, -8); + bytes[4] = GC_MAP_VERSION + 1; + assert!(parse_gc_map(&bytes).is_none(), "an unknown version must not be guessed at"); + // A total_len that runs past the section must fail rather than read on. + let mut bytes = simple(0x1000, 0x10, -8); + let len = bytes.len(); + bytes[12..16].copy_from_slice(&((len as u32) + 64).to_le_bytes()); + assert!(parse_gc_map(&bytes).is_none()); } #[test] fn chain_walkable_index_accepts_fp_and_sp_locations_only() { - let rec = |pc: usize, reg: u16| StackMapRecord { + let rec = |pc: usize| StackMapRecord { pc, function_address: pc, stack_size: 160, - locations: vec![StackMapLocation { - dwarf_reg: reg, - offset: -8, - }], + roots_start: 0, + roots_len: 1, }; // FP and SP are both walkable: SP resolves per frame by decoding the // owning function's prologue (#7173). - let walkable = index_records(vec![ - rec(0x1000, DWARF_REG_FP_AARCH64), - rec(0x2000, DWARF_REG_SP_AARCH64), - ]); + let walkable = index_records( + vec![rec(0x1000), rec(0x2000)], + vec![ + StackMapLocation { dwarf_reg: DWARF_REG_FP_AARCH64, offset: -8 }, + StackMapLocation { dwarf_reg: DWARF_REG_SP_AARCH64, offset: -8 }, + ], + ); assert!(walkable.chain_walkable); assert_eq!(walkable.min_pc, 0x1000); assert_eq!(walkable.max_pc, 0x2000); // Any other register disqualifies the whole image. assert!( - !index_records(vec![rec(0x1000, DWARF_REG_FP_AARCH64), rec(0x3000, 1)]).chain_walkable, + !index_records( + vec![rec(0x1000)], + vec![StackMapLocation { dwarf_reg: 1, offset: -8 }], + ) + .chain_walkable, "a non-FP/SP register must disable the fast walk" ); } #[test] fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { - let maps = vec![ - StackMapRecord { - pc: 0x1000, - function_address: 0x1000, - stack_size: 32, - locations: Vec::new(), - }, - StackMapRecord { - pc: 0x1020, - function_address: 0x1020, - stack_size: 32, - locations: Vec::new(), - }, - ]; + let rec = |pc: usize| StackMapRecord { + pc, + function_address: pc, + stack_size: 32, + roots_start: 0, + roots_len: 0, + }; + let maps = vec![rec(0x1000), rec(0x1020)]; assert_eq!(closest_record_pc(&maps, 0x1004), Some(0x1000)); assert_eq!(closest_record_pc(&maps, 0x101c), Some(0x1020)); assert_eq!(closest_record_pc(&maps, 0x1020), Some(0x1020)); diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index c855158be8..533bba6f16 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -379,15 +379,91 @@ imbalance that no audited elision closes.** The contract's real-app effect is −4.8% metadata (probe-scale was −8.8%; dependency code has proportionally fewer audited-helper sites). -**Standing verdict, now measured on every axis:** the shadow stack is the -three-axis optimum shipping today — wall-clock tied within timer -quantization, RSS tied, file-size won by 3.5 MB on a real application. -The statepoint backend is correctness-superior (the forgot-to-root class -is structurally impossible), speed-competitive, and 59% leaner in metadata -than its own first prototype — and its remaining 25× size gap is proven -(not projected) to close only through repsel promotion shrinking the -maybe-pointer root set, or RS4GC managed-pointer SSA. Both are tracked; -neither is this branch's to deliver. +**Verdict as of 2026-08-01 (superseded below):** the shadow stack was the +three-axis optimum — wall-clock tied within timer quantization, RSS tied, +file-size won by 3.5 MB on a real application. The statepoint backend was +correctness-superior (the forgot-to-root class is structurally impossible), +speed-competitive, and 59% leaner in metadata than its own first prototype, +but carried a 25× metadata imbalance that no audited elision closed. + +That conclusion assumed the metadata's *content* was the cost. It was not. + +## The file-size axis was the wire format, not the roots (2026-08-03) + +Re-examined with `scripts/stackmap_anatomy.py`, which breaks the section +down by structural component and asserts it parsed 100% of the bytes. + +**First correction: generated code is already smaller under statepoints.** +On `test-drizzle-pg` the RS4GC arm's `__text` is 20,128,708 against shadow's +20,376,748 — **248 KB better**. The entire loss is `__llvm_stackmaps`. + +**Second correction: most of that section is provably dead weight.** + +| component | share of section | +|---|---:| +| `Constant` location slots (3 per record: CC / Flags / NumDeopt) | 40.6% | +| duplicate base/derived location slots | 13.3% | +| record headers (incl. an 8-byte patchpoint ID never patched) | 18.0% | +| inter-record padding | 11.3% | + +`stack_maps.rs` **already discarded the constants and collapsed the +base/derived pair at parse time**. Over half the section was shipped in the +binary and thrown away at startup — redundancy, not a tradeoff. LLVM's +stack map is a JIT-patching wire format; an AOT collector needs +`{dwarf_reg, offset}` per distinct root and nothing else. + +**Compaction measured on drizzle** (4,214,384 B, 124 concatenated maps, +1,717 functions, 33,406 records, 154,020 distinct roots): + +| encoding | bytes | ratio | +|---|---:|---:| +| flat varint (drop constants + duplicate pairs) | 387,199 | 10.9× | +| + roots sorted and delta-encoded | 286,258 | 14.7× | +| + "same live set as previous record" flag | **132,418** | **31.8×** | + +The last row is the big one and it is a fact about real programs, not a +coding trick: **77% of records have exactly the live set of the record +before them**, because consecutive safepoints in a function share their +roots. That same fact shrinks the in-memory index — the decoder points +repeats at one copy instead of materialising 154k entries — so it is an RSS +win as well as a file-size one. + +**Projected onto the measured RS4GC arm:** 3,875,416 B of metadata becomes +~121 KB, taking the binary from 31,957,792 to ~28.20 MB against shadow's +28,474,576 — a **~271 KB win**, versus a 3.5 MB loss before. Statepoints +then lead on **all three axes** (wall-clock −0.93%, RSS flat, size −271 KB). + +### Why the rewrite happens on assembly + +`clang -S` prints the stack map as ordinary directives with the function +addresses as **symbol names in plain text** (`.quad _main`), so one text +parser replaces Mach-O *and* ELF relocation parsing, `llvm-objcopy`, and a +second link pass. Two facts settled this empirically rather than by taste: + +- the stack map's function-address fields are **external symbol + relocations** (`otool -r`: `extern 1` at offsets 16 and 40), so a + separately assembled table can reference them by name and the linker + resolves it — no relink needed; +- `-S` costs nothing: it takes the **same 0.04s as `-c`** (codegen is the + cost, printing text is free), and `llvm-mc` assembles in 0.02s. Use + `llvm-mc`, not `clang -c file.s` — the latter is 0.13s of driver overhead. + +### Two traps that cost time here + +- **On-disk stack-map addresses look like garbage** (`0x00300000000008b0`) + because they are **dyld chained-fixup entries**, not addresses: bits 0-35 + are the target, bits 51-62 the chain delta to the next fixup. dyld + resolves them at load. Do not conclude records are mismatching from an + on-disk read. +- **The section is a concatenation of one map per object file**, not a + single map. A parser that reads only the first header silently + under-counts, and nothing downstream notices. Assert that parse coverage + equals the section size. +- **`.no_dead_strip` names the block's label from outside the block.** + Removing the label without retargeting that directive leaves an undefined + symbol — and the directive is also the only thing keeping a section + nothing references from being stripped, which would leave the collector + with no roots at all. ### The transfer question, also measured: can the audit shrink the SHADOW stack? diff --git a/scripts/stackmap_anatomy.py b/scripts/stackmap_anatomy.py new file mode 100644 index 0000000000..d882cd1588 --- /dev/null +++ b/scripts/stackmap_anatomy.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Break an LLVM stackmap v3 section down into where its bytes actually go. + +Answers the only question that matters for file size: which structural +component dominates, and how much of it is redundant for a collector that +has no interior pointers. + +Usage: stackmap_anatomy.py # finds the section itself + stackmap_anatomy.py --raw +""" +import struct +import subprocess +import sys + +LOC_KIND = {1: "Register", 2: "Direct", 3: "Indirect", 4: "Constant", 5: "ConstIndex"} + + +def extract_section(path): + """Pull __LLVM_STACKMAPS (Mach-O) or .llvm_stackmaps (ELF) out of a binary.""" + with open(path, "rb") as fh: + head = fh.read(4) + if head[:4] == b"\x7fELF": + out = subprocess.run( + ["readelf", "-x", ".llvm_stackmaps", path], + capture_output=True, text=True).stdout + data = bytearray() + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0].startswith("0x"): + for word in parts[1:5]: + if all(c in "0123456789abcdefABCDEF" for c in word) and len(word) % 2 == 0: + data += bytes.fromhex(word) + return bytes(data) + # Mach-O: locate section by (segment, section) and slice the file. + out = subprocess.run(["otool", "-l", path], capture_output=True, text=True).stdout + lines = out.splitlines() + for i, line in enumerate(lines): + if "__llvm_stackmaps" in line: + off = size = None + for probe in lines[i:i + 14]: + p = probe.split() + if len(p) == 2 and p[0] == "offset": + off = int(p[1], 0) + if len(p) == 2 and p[0] == "size": + size = int(p[1], 0) + if off is not None and size is not None: + with open(path, "rb") as fh: + fh.seek(off) + return fh.read(size) + raise SystemExit(f"no stackmap section found in {path}") + + +def analyze(buf): + ver, _, _ = struct.unpack_from("12,} {100.0 * val / total:5.1f}%") + print() + print(" location kinds:") + for key, val in sorted(kinds.items(), key=lambda kv: -kv[1]): + print(f" {str(key):<16} {val:>12,} {100.0 * val / max(n_locs,1):5.1f}%") + print() + print(f" duplicate location slots within a record: {dup_locs:,} " + f"({100.0 * dup_locs / max(n_locs,1):.1f}% of locations, " + f"{dup_locs * 12:,} B = {100.0 * dup_locs * 12 / total:.1f}% of section)") + print(f" constant-kind locations: {const_locs:,} " + f"({const_locs * 12:,} B = {100.0 * const_locs * 12 / total:.1f}% of section)") + + return kinds + + +def varint_len(value): + n = 1 + while value >= 0x80: + value >>= 7 + n += 1 + return n + + +def compact_size(buf): + """Exact byte count of the encoding the runtime would actually consume. + + The runtime already discards Constant locations and dedups the base/derived + pair at parse time (`stack_maps.rs`), so it keeps only {dwarf_reg, offset} + per distinct root. This measures shipping precisely that: + + header 16 B + per function 16 B (u64 relocated address, u32 stack size, u32 records) + per record varint(delta instruction offset) + varint(root count) + per root varint(zigzag(offset) << 1 | reg_is_sp) + """ + _ver, _, _ = struct.unpack_from("> 31) + size += varint_len((zig << 1) | (1 if reg == 31 else 0)) + roots_kept += len(seen) + if (pos - rec_start) % 8: + pos += 8 - ((pos - rec_start) % 8) + (nlive,) = struct.unpack_from(" len(buf): + break + maps.append(buf[pos:pos + extent]) + pos += extent + while pos % 8: + pos += 1 + return maps + + +if __name__ == "__main__": + args = sys.argv[1:] + raw = open(args[1], "rb").read() if args and args[0] == "--raw" \ + else extract_section(args[0]) + maps = split_maps(raw) + print(f"section {len(raw):,} B = {len(maps)} concatenated map(s)\n") + merged = {"total": 0, "compact": 0, "roots": 0} + for chunk in maps: + merged["total"] += len(chunk) + csize, croots = compact_size(chunk) + # Only the first map pays a format header in the merged encoding. + merged["compact"] += csize + merged["roots"] += croots + if len(maps) == 1: + analyze(maps[0]) + else: + # Aggregate composition across every map. + agg = {} + for chunk in maps: + nfunc, nconst, nrec = struct.unpack_from(" {len(raw) / max(merged['compact'],1):.1f}x smaller, " + f"saves {len(raw) - merged['compact']:,} B") From e76897877db6f888231b83c2e5d964bb9bf63cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 12:46:29 +0200 Subject: [PATCH 42/53] gc: ship the compact map, measured -131 KB against the shadow stack Completes the previous commit with the constraint that changed its design, and replaces the projection with a measurement. At -O3, LLVM does NOT emit a record's instruction offset as a literal: it emits a label difference (`.long Ltmp9-_main`) that only the assembler can evaluate. Those offsets therefore cannot be delta-varint-encoded at rewrite time, and now live in a fixed-width u32 array (~4 B/record). That is 18.7x compaction rather than 31.8x. Recovering the difference would mean assembling twice -- once to learn the numbers the assembler just computed, once to emit them -- which is more machinery than 92 KB is worth. This was worth catching for a second reason: a prototype that treated any non-integer operand as a symbol appeared to work while silently decoding every such offset as ZERO. Literal offsets do appear without -O3, so a hand-compiled probe hides the whole problem. Measured on test-drizzle-pg, one compiler, identical flags, clean object cache per arm (a clean-cache rebuild reproduced the cached shadow figure to within 8 bytes, so this is not a stale-artifact reading): shadow (default) 28,737,536 __text 20,646,900 map 0 statepoint + compact 28,688,464 __text 20,497,296 map 227,275 -49,072 RS4GC + compact 28,605,912 __text 20,409,232 map 224,126 -131,624 Metadata 4,214,384 -> 227,275 B (18.5x), within 1% of what the encoder model predicted. The file-size axis is flipped: the statepoint backend now leads on ALL THREE axes -- wall-clock -0.93%, RSS flat, size -131,624 B -- where it previously lost size by 3.5 MB. Both arms pass the full gate: 8/8 probes byte-match the pinned Node oracle normally and under PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 PERRY_STACKMAP_WALKER=verify. That last one is the check that can fail if the format decoded to a smaller root set -- lost roots corrupt the heap under forced evacuation rather than merely printing something different. The gate also asserts its subject was live (__llvm_stackmaps absent AND __perry_gcmap non-empty) before comparing any output; its first run correctly reported 0/8 because the rewrite had not run at all. Compile time: 11.95s vs 10.43s for the whole application (+14.6%), covering statepoint lowering plus the assembly round trip. Also fixes a pre-existing bug on this branch: ConservativeScanSite::ALL was missing SafepointContractHeal while COUNT already counted it, so that scan site could never be enumerated -- and the mismatch broke every perry-runtime test build. The compaction driver lives in gc_map.rs rather than linker.rs, which keeps linker.rs under the 2000-line lint cap. --- crates/perry-codegen/src/gc_map.rs | 165 +++++++++++++++--- crates/perry-codegen/src/linker.rs | 93 ++-------- .../perry-runtime/src/gc/roots/stack_maps.rs | 66 +++++-- crates/perry-runtime/src/gc/scan_fallback.rs | 1 + docs/statepoint-gc-experiment.md | 52 +++++- 5 files changed, 242 insertions(+), 135 deletions(-) diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index 871182c763..60a65712e9 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -49,11 +49,17 @@ //! the cost; printing text is free), and assembling the result is ~0.02s. use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::{anyhow, Context, Result}; /// Magic at the start of every emitted blob. pub const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; /// Format version. Bump on any layout change — the runtime rejects others. -pub const GC_MAP_VERSION: u8 = 1; +pub const GC_MAP_VERSION: u8 = 2; /// Section the compact map is emitted into, and the label it is given. const GC_MAP_LABEL: &str = "_perry_gc_map"; const MACHO_SECTION: &str = "__PERRY_GCMAP,__perry_gcmap"; @@ -67,9 +73,14 @@ const LOCATION_DIRECT: u8 = 2; const LOCATION_INDIRECT: u8 = 3; /// One safepoint: where it is in its function, and which frame slots are live. +/// +/// `instruction_offset` is the **assembly expression**, not a number: at `-O3` +/// LLVM emits it as a label difference (`Ltmp9-_main`) that only the assembler +/// can evaluate. That is why the emitted map stores offsets in a fixed-width +/// `u32` array rather than folding them into the varint stream. #[derive(Debug, Clone, PartialEq, Eq)] struct Record { - instruction_offset: u32, + instruction_offset: String, /// `(dwarf_reg, frame_offset)`, deduplicated and sorted by frame offset. roots: Vec<(u16, i32)>, } @@ -105,7 +116,8 @@ struct RawBlock { fn parse_block(lines: &[&str]) -> Option { let start_line = lines.iter().position(|line| { let t = line.trim_start(); - t.starts_with(".section") && (t.contains("__LLVM_STACKMAPS") || t.contains(".llvm_stackmaps")) + t.starts_with(".section") + && (t.contains("__LLVM_STACKMAPS") || t.contains(".llvm_stackmaps")) })?; let mut bytes: Vec = Vec::new(); @@ -157,13 +169,12 @@ fn parse_block(lines: &[&str]) -> Option { match parse_int(operand) { Some(value) => bytes.extend_from_slice(&value.to_le_bytes()[..width]), None => { - // A symbolic `.quad`: the function address. Remember the name - // and reserve the slot so later offsets stay correct. - if width != 8 { - return None; - } + // A symbolic operand. Two kinds appear: the `.quad` function + // address, and — at `-O3` — the `.long` instruction offset as + // a label difference. Remember the expression and reserve the + // slot so every later structural offset stays correct. symbols.insert(bytes.len(), operand.to_string()); - bytes.extend_from_slice(&0u64.to_le_bytes()); + bytes.extend_from_slice(&0u64.to_le_bytes()[..width]); } } } @@ -182,7 +193,10 @@ fn parse_int(text: &str) -> Option { return u64::from_str_radix(hex, 16).ok(); } if let Some(negative) = text.strip_prefix('-') { - return negative.parse::().ok().map(|v| (v as i64).wrapping_neg() as u64); + return negative + .parse::() + .ok() + .map(|v| (v as i64).wrapping_neg() as u64); } text.parse::().ok() } @@ -243,7 +257,11 @@ fn decode_v3(block: &RawBlock) -> Option> { let mut records = Vec::with_capacity(count); for _ in 0..count { let record_start = pos; - let instruction_offset = read_u32(bytes, pos + 8)?; + let instruction_offset = block + .symbols + .get(&(pos + 8)) + .cloned() + .unwrap_or_else(|| read_u32(bytes, pos + 8).unwrap_or(0).to_string()); let location_count = read_u16(bytes, pos + 14)? as usize; pos += 16; @@ -314,15 +332,8 @@ const DWARF_REG_SP_AARCH64: u16 = 31; fn encode_stream(functions: &[FunctionMap]) -> Vec { let mut stream = Vec::new(); for function in functions { - let mut previous_offset = 0u32; let mut previous_roots: Option<&Vec<(u16, i32)>> = None; for record in &function.records { - push_varint( - &mut stream, - u64::from(record.instruction_offset.wrapping_sub(previous_offset)), - ); - previous_offset = record.instruction_offset; - if previous_roots == Some(&record.roots) { // Repeat flag: the live set is the previous record's. push_varint(&mut stream, 1); @@ -361,13 +372,22 @@ fn encode_stream(functions: &[FunctionMap]) -> Vec { /// 8 u32 function_count /// 12 u32 total_len -- lets the runtime walk concatenated blobs /// 16 function_count x { u64 address, u32 stack_size, u32 record_count } -/// varint stream (see `encode_stream`) +/// record_count_total x u32 instruction_offset +/// varint root stream (see `encode_stream`) /// ``` /// /// The function table starts at 16 so every relocated address is 8-byte -/// aligned. +/// aligned, and the offset array that follows it is 4-byte aligned. +/// +/// Instruction offsets are a fixed-width array rather than part of the varint +/// stream because at `-O3` they are **label differences the assembler +/// evaluates** (`Ltmp9-_main`), so their values do not exist at rewrite time. +/// That costs ~4 bytes per record — 18.7x compaction instead of 31.8x — and +/// buys not having to assemble twice just to learn numbers the assembler is +/// about to compute anyway. fn emit_asm(functions: &[FunctionMap], stream: &[u8], elf: bool) -> String { - let total_len = 16 + functions.len() * 16 + stream.len(); + let record_total: usize = functions.iter().map(|f| f.records.len()).sum(); + let total_len = 16 + functions.len() * 16 + record_total * 4 + stream.len(); let mut out = String::new(); if elf { out.push_str(&format!("\t.section\t{ELF_SECTION}\n")); @@ -387,6 +407,11 @@ fn emit_asm(functions: &[FunctionMap], stream: &[u8], elf: bool) -> String { out.push_str(&format!("\t.long\t{}\n", function.stack_size as u32)); out.push_str(&format!("\t.long\t{}\n", function.records.len())); } + for function in functions { + for record in &function.records { + out.push_str(&format!("\t.long\t{}\n", record.instruction_offset)); + } + } for chunk in stream.chunks(32) { let bytes: Vec = chunk.iter().map(|b| b.to_string()).collect(); out.push_str(&format!("\t.byte\t{}\n", bytes.join(","))); @@ -419,7 +444,10 @@ pub fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats let stats = GcMapStats { original_bytes: block.bytes.len(), - compact_bytes: 16 + functions.len() * 16 + stream.len(), + compact_bytes: 16 + + functions.len() * 16 + + functions.iter().map(|f| f.records.len()).sum::() * 4 + + stream.len(), functions: functions.len(), records: functions.iter().map(|f| f.records.len()).sum(), roots: functions @@ -454,6 +482,80 @@ pub fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats Some((out, stats)) } +/// Rewrite the stack map in `asm_path` into Perry's compact form, then +/// assemble it to `obj_path`. +/// +/// A module with no stack-map block, or one whose block does not parse, is +/// assembled unchanged — LLVM's section is correct, merely large, so falling +/// back costs bytes rather than roots. +pub fn compact_and_assemble( + clang: &Path, + target: &str, + asm_path: &Path, + obj_path: &Path, +) -> Result<()> { + let asm = fs::read_to_string(asm_path) + .with_context(|| format!("Failed to read assembly at {}", asm_path.display()))?; + + let elf = + !target.contains("apple") && !target.contains("darwin") && !target.contains("windows"); + if let Some((rewritten, stats)) = compact_stack_map_asm(&asm, elf) { + fs::write(asm_path, rewritten).with_context(|| { + format!( + "Failed to write compacted assembly at {}", + asm_path.display() + ) + })?; + GC_MAP_ORIGINAL_BYTES.fetch_add(stats.original_bytes as u64, Ordering::Relaxed); + GC_MAP_COMPACT_BYTES.fetch_add(stats.compact_bytes as u64, Ordering::Relaxed); + log::debug!( + "perry-codegen: gc map {} -> {} bytes ({} functions, {} records, {} roots)", + stats.original_bytes, + stats.compact_bytes, + stats.functions, + stats.records, + stats.roots, + ); + } + + let output = Command::new(clang) + .arg("-c") + .arg(asm_path) + .arg("-o") + .arg(obj_path) + .arg("-target") + .arg(target) + .output() + .with_context(|| format!("Failed to invoke {}", clang.display()))?; + if !output.status.success() { + return Err(anyhow!( + "assembling the compacted stack map failed (status={}).\n\ + assembly left at: {}\n\ + \n\ + stderr:\n{}", + output.status, + asm_path.display(), + String::from_utf8_lossy(&output.stderr) + )); + } + let _ = fs::remove_file(asm_path); + Ok(()) +} + +/// Totals for the whole process, so a build can report what compaction did. +/// A run where these stay zero did not compact anything — the distinction a +/// gate needs in order to be able to fail. +static GC_MAP_ORIGINAL_BYTES: AtomicU64 = AtomicU64::new(0); +static GC_MAP_COMPACT_BYTES: AtomicU64 = AtomicU64::new(0); + +/// `(llvm_bytes, compact_bytes)` summed across every module compiled so far. +pub fn gc_map_compaction_totals() -> (u64, u64) { + ( + GC_MAP_ORIGINAL_BYTES.load(Ordering::Relaxed), + GC_MAP_COMPACT_BYTES.load(Ordering::Relaxed), + ) +} + #[cfg(test)] mod tests { use super::*; @@ -479,9 +581,13 @@ mod tests { asm.push_str("\t.short\t4\n"); // three statepoint preamble constants, then base/derived pair for _ in 0..3 { - asm.push_str("\t.byte\t4\n\t.byte\t0\n\t.short\t8\n\t.short\t0\n\t.short\t0\n\t.long\t0\n"); + asm.push_str( + "\t.byte\t4\n\t.byte\t0\n\t.short\t8\n\t.short\t0\n\t.short\t0\n\t.long\t0\n", + ); } - asm.push_str("\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n\t.short\t0\n\t.long\t4294967272\n"); + asm.push_str( + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n\t.short\t0\n\t.long\t4294967272\n", + ); asm.push_str("\t.p2align\t3\n"); asm.push_str("\t.short\t0\n\t.short\t0\n"); // live-out header asm.push_str("\t.p2align\t3\n"); @@ -519,15 +625,15 @@ mod tests { stack_size: 64, records: vec![ Record { - instruction_offset: 0, + instruction_offset: "0".to_string(), roots: shared.clone(), }, Record { - instruction_offset: 8, + instruction_offset: "8".to_string(), roots: shared.clone(), }, Record { - instruction_offset: 16, + instruction_offset: "16".to_string(), roots: shared, }, ], @@ -537,11 +643,12 @@ mod tests { stack_size: functions[0].stack_size, records: functions[0].records[..1].to_vec(), }]; - // The two extra records cost a delta byte plus a repeat byte each, + // Offsets live in their own fixed-width array now, so in the varint + // stream the two extra records cost exactly one repeat byte each, // regardless of how many roots the shared live set holds. assert_eq!( encode_stream(&functions).len(), - encode_stream(&one_record).len() + 4 + encode_stream(&one_record).len() + 2 ); } diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index 343f0545e8..611d9d0661 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -405,10 +405,9 @@ fn build_clang_compile_plan( // the statepoint backends emit a stack map, so only they pay for it, and // the cost is small: `-S` takes the same time as `-c` (codegen is the // cost, printing text is free) and assembling is ~0.02s per module. - let compact_gc_map = crate::codegen::helpers::statepoints_enabled() - || crate::codegen::helpers::rs4gc_enabled(); - let asm_path = - compact_gc_map.then(|| PathBuf::from(format!("{}.s", obj_path.display()))); + let compact_gc_map = + crate::codegen::helpers::statepoints_enabled() || crate::codegen::helpers::rs4gc_enabled(); + let asm_path = compact_gc_map.then(|| PathBuf::from(format!("{}.s", obj_path.display()))); let mut clang_args = vec![ if compact_gc_map { "-S" } else { "-c" }.to_string(), @@ -441,13 +440,7 @@ fn build_clang_compile_plan( } clang_args.push(ll_path.display().to_string()); clang_args.push("-o".to_string()); - clang_args.push( - asm_path - .as_ref() - .unwrap_or(&obj_path) - .display() - .to_string(), - ); + clang_args.push(asm_path.as_ref().unwrap_or(&obj_path).display().to_string()); clang_args.push("-target".to_string()); clang_args.push(effective_target.clone()); @@ -535,77 +528,6 @@ fn maybe_rs4gc_preprocess(ll_text: &str) -> Result> { Ok(Some(String::from_utf8(output.stdout)?)) } -/// Rewrite the stack map in `asm_path` into Perry's compact form, then -/// assemble it to `obj_path`. -/// -/// A module with no stack-map block, or one whose block does not parse, is -/// assembled unchanged — LLVM's section is correct, merely large, so falling -/// back costs bytes rather than roots. -fn compact_gc_map_and_assemble( - plan: &ClangCompilePlan, - asm_path: &Path, - obj_path: &Path, -) -> Result<()> { - let asm = fs::read_to_string(asm_path) - .with_context(|| format!("Failed to read assembly at {}", asm_path.display()))?; - - let elf = !plan.effective_target.contains("apple") - && !plan.effective_target.contains("darwin") - && !plan.effective_target.contains("windows"); - if let Some((rewritten, stats)) = crate::gc_map::compact_stack_map_asm(&asm, elf) { - fs::write(asm_path, rewritten).with_context(|| { - format!("Failed to write compacted assembly at {}", asm_path.display()) - })?; - GC_MAP_ORIGINAL_BYTES.fetch_add(stats.original_bytes as u64, Ordering::Relaxed); - GC_MAP_COMPACT_BYTES.fetch_add(stats.compact_bytes as u64, Ordering::Relaxed); - log::debug!( - "perry-codegen: gc map {} -> {} bytes ({} functions, {} records, {} roots)", - stats.original_bytes, - stats.compact_bytes, - stats.functions, - stats.records, - stats.roots, - ); - } - - let output = Command::new(&plan.clang) - .arg("-c") - .arg(asm_path) - .arg("-o") - .arg(obj_path) - .arg("-target") - .arg(&plan.effective_target) - .output() - .with_context(|| format!("Failed to invoke {}", plan.clang.display()))?; - if !output.status.success() { - return Err(anyhow!( - "assembling the compacted stack map failed (status={}).\n\ - assembly left at: {}\n\ - \n\ - stderr:\n{}", - output.status, - asm_path.display(), - String::from_utf8_lossy(&output.stderr) - )); - } - let _ = fs::remove_file(asm_path); - Ok(()) -} - -/// Totals for the whole process, so a build can report what compaction did. -/// A run where these stay zero did not compact anything — the distinction a -/// gate needs in order to be able to fail. -static GC_MAP_ORIGINAL_BYTES: AtomicU64 = AtomicU64::new(0); -static GC_MAP_COMPACT_BYTES: AtomicU64 = AtomicU64::new(0); - -/// `(llvm_bytes, compact_bytes)` summed across every module compiled so far. -pub fn gc_map_compaction_totals() -> (u64, u64) { - ( - GC_MAP_ORIGINAL_BYTES.load(Ordering::Relaxed), - GC_MAP_COMPACT_BYTES.load(Ordering::Relaxed), - ) -} - fn which_in_path(name: &str) -> Option { std::env::var_os("PATH").and_then(|paths| { std::env::split_paths(&paths) @@ -768,7 +690,12 @@ fn compile_ll_to_object_in( } if let Some(asm_path) = &plan.asm_path { - compact_gc_map_and_assemble(&plan, asm_path, &obj_path)?; + crate::gc_map::compact_and_assemble( + &plan.clang, + &plan.effective_target, + asm_path, + &obj_path, + )?; } let bytes = fs::read(&obj_path) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index ecc2309348..b4c8941415 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -24,7 +24,7 @@ use std::sync::OnceLock; /// statepoint constant preamble and base/derived duplicates that this parser /// discarded anyway, and shipping it cost 3.9 MB on a real application. const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; -const GC_MAP_VERSION: u8 = 1; +const GC_MAP_VERSION: u8 = 2; const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct StackMapLocation { @@ -361,19 +361,30 @@ fn parse_gc_map(bytes: &[u8]) -> Option<(Vec, Vec blob_end { + return None; + } + let mut record_index = 0usize; + for index in 0..function_count { let entry = table + index * 16; let function_address = read_u64(bytes, entry)? as usize; let stack_size = u64::from(read_u32(bytes, entry + 8)?); let record_count = read_u32(bytes, entry + 12)? as usize; - let mut instruction_offset = 0u32; let mut previous: Option<(u32, u32)> = None; for _ in 0..record_count { - let (delta, next) = read_varint(bytes, cursor, blob_end)?; - cursor = next; - instruction_offset = instruction_offset.wrapping_add(delta as u32); + let instruction_offset = read_u32(bytes, offsets + record_index * 4)?; + record_index += 1; let (header, next) = read_varint(bytes, cursor, blob_end)?; cursor = next; @@ -932,11 +943,10 @@ mod tests { /// `records` is `(instruction_offset, roots)`, roots as `(dwarf_reg, offset)`; /// an empty root slice with `repeat` set encodes the repeat flag. fn one_map(function: u64, records: &[(u32, Vec<(u16, i32)>, bool)]) -> Vec { + let mut offsets = Vec::new(); let mut stream = Vec::new(); - let mut previous_offset = 0u32; for (instruction_offset, roots, repeat) in records { - push_varint(&mut stream, u64::from(instruction_offset.wrapping_sub(previous_offset))); - previous_offset = *instruction_offset; + offsets.extend_from_slice(&instruction_offset.to_le_bytes()); if *repeat { push_varint(&mut stream, 1); continue; @@ -954,7 +964,7 @@ mod tests { } } - let total_len = 16 + 16 + stream.len(); + let total_len = 16 + 16 + offsets.len() + stream.len(); let mut bytes = Vec::new(); bytes.extend_from_slice(GC_MAP_MAGIC); bytes.push(GC_MAP_VERSION); @@ -964,6 +974,7 @@ mod tests { bytes.extend_from_slice(&function.to_le_bytes()); bytes.extend_from_slice(&32u32.to_le_bytes()); bytes.extend_from_slice(&(records.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&offsets); bytes.extend_from_slice(&stream); while bytes.len() % 8 != 0 { bytes.push(0); @@ -1031,9 +1042,18 @@ mod tests { assert_eq!( roots, vec![ - StackMapLocation { dwarf_reg: 29, offset: -64 }, - StackMapLocation { dwarf_reg: 29, offset: -8 }, - StackMapLocation { dwarf_reg: 31, offset: 24 }, + StackMapLocation { + dwarf_reg: 29, + offset: -64 + }, + StackMapLocation { + dwarf_reg: 29, + offset: -8 + }, + StackMapLocation { + dwarf_reg: 31, + offset: 24 + }, ] ); } @@ -1043,7 +1063,10 @@ mod tests { assert!(parse_gc_map(&[]).is_none() || parse_gc_map(&[]).unwrap().0.is_empty()); let mut bytes = simple(0x1000, 0x10, -8); bytes[4] = GC_MAP_VERSION + 1; - assert!(parse_gc_map(&bytes).is_none(), "an unknown version must not be guessed at"); + assert!( + parse_gc_map(&bytes).is_none(), + "an unknown version must not be guessed at" + ); // A total_len that runs past the section must fail rather than read on. let mut bytes = simple(0x1000, 0x10, -8); let len = bytes.len(); @@ -1065,8 +1088,14 @@ mod tests { let walkable = index_records( vec![rec(0x1000), rec(0x2000)], vec![ - StackMapLocation { dwarf_reg: DWARF_REG_FP_AARCH64, offset: -8 }, - StackMapLocation { dwarf_reg: DWARF_REG_SP_AARCH64, offset: -8 }, + StackMapLocation { + dwarf_reg: DWARF_REG_FP_AARCH64, + offset: -8, + }, + StackMapLocation { + dwarf_reg: DWARF_REG_SP_AARCH64, + offset: -8, + }, ], ); assert!(walkable.chain_walkable); @@ -1076,7 +1105,10 @@ mod tests { assert!( !index_records( vec![rec(0x1000)], - vec![StackMapLocation { dwarf_reg: 1, offset: -8 }], + vec![StackMapLocation { + dwarf_reg: 1, + offset: -8 + }], ) .chain_walkable, "a non-FP/SP register must disable the fast walk" diff --git a/crates/perry-runtime/src/gc/scan_fallback.rs b/crates/perry-runtime/src/gc/scan_fallback.rs index e7e7d17607..345a400c9f 100644 --- a/crates/perry-runtime/src/gc/scan_fallback.rs +++ b/crates/perry-runtime/src/gc/scan_fallback.rs @@ -133,6 +133,7 @@ impl ConservativeScanSite { Self::EmergencyReclaim, Self::ManualCollect, Self::ManualMinor, + Self::SafepointContractHeal, ]; } diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 533bba6f16..0c5dc08c78 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -419,19 +419,59 @@ stack map is a JIT-patching wire format; an AOT collector needs |---|---:|---:| | flat varint (drop constants + duplicate pairs) | 387,199 | 10.9× | | + roots sorted and delta-encoded | 286,258 | 14.7× | -| + "same live set as previous record" flag | **132,418** | **31.8×** | +| + "same live set as previous record" flag | 132,418 | 31.8× | +| **shipped**: as above, but offsets fixed-width | **224,832** | **18.7×** | -The last row is the big one and it is a fact about real programs, not a +The third row is the big one and it is a fact about real programs, not a coding trick: **77% of records have exactly the live set of the record before them**, because consecutive safepoints in a function share their roots. That same fact shrinks the in-memory index — the decoder points repeats at one copy instead of materialising 154k entries — so it is an RSS win as well as a file-size one. -**Projected onto the measured RS4GC arm:** 3,875,416 B of metadata becomes -~121 KB, taking the binary from 31,957,792 to ~28.20 MB against shadow's -28,474,576 — a **~271 KB win**, versus a 3.5 MB loss before. Statepoints -then lead on **all three axes** (wall-clock −0.93%, RSS flat, size −271 KB). +The fourth row is what actually ships, and the difference is a constraint +rather than a choice: **at `-O3` LLVM emits each record's instruction offset +as a label difference** (`.long Ltmp9-_main`) that only the assembler can +evaluate, so those offsets cannot be delta-varint-encoded at rewrite time. +They go in a fixed-width `u32` array instead, costing ~4 bytes per record. +Recovering the last 92 KB would mean assembling twice — once to learn the +numbers the assembler just computed, once to emit them — which is more +machinery than the bytes are worth. + +### Measured, not projected (2026-08-03) + +Built with one compiler, identical flags, and a **clean object cache per +arm** (a clean-cache rebuild reproduced the cached shadow figure to within +8 bytes, so nothing here is a stale-artifact reading): + +| arm | total | `__text` | `__perry_gcmap` | vs shadow | +|---|---:|---:|---:|---:| +| shadow (default) | 28,737,536 | 20,646,900 | 0 | — | +| statepoint + compact | 28,688,464 | 20,497,296 | 227,275 | **−49,072** | +| RS4GC + compact | 28,605,912 | 20,409,232 | 224,126 | **−131,624** | + +The emitted map came in at 227,275 B against the 224,832 B the encoder model +predicted — within 1%. Metadata fell from 4,214,384 B to 227,275 B (18.5×), +and `__text` is 149,604 B smaller than shadow's on the same build. + +**The file-size axis is flipped.** The statepoint backend now leads on +**all three axes** — wall-clock −0.93%, RSS flat, and size −131,624 B on the +RS4GC arm — where it previously lost size by 3.5 MB. + +Both arms pass the full gate: 8/8 probes byte-match the pinned Node oracle +normally *and* under `PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 +PERRY_STACKMAP_WALKER=verify`, which is the check that can actually fail if +the new format decoded to a smaller root set — a map that lost roots would +corrupt the heap under forced evacuation rather than merely print something +different. The gate also asserts its subject was live: `__llvm_stackmaps` +absent **and** `__perry_gcmap` non-empty, before any output is compared. +Its first run correctly reported 0/8 because the rewrite had silently not +run at all. + +**Compile-time cost, stated honestly:** 11.95s vs 10.43s for the whole +application (+14.6%), covering statepoint lowering plus the assembly round +trip. That is not one of the three axes being optimised, and it buys the +axis that was losing. ### Why the rewrite happens on assembly From d86d9080469043f60f7ecda26204a11818db16f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 14:07:29 +0200 Subject: [PATCH 43/53] gc: fail loudly on an undecodable GC map, and skip compaction off Mach-O/ELF Two holes left by the compact-map change, both silent by construction. 1. A GC map section that exists but does not decode returned an EMPTY index, which is indistinguishable downstream from "this is a shadow-stack build with no native frame roots". The consequences are not the same: with statepoints as the only root mechanism an empty index means the collector frees live objects and corrupts the heap with no diagnostic at all. That is CLAUDE.md's fourth gate-failure mode -- the gate runs, its subject never did. Now: no section at all still yields an empty index (correct for a shadow build), but a section that is present and undecodable panics at startup, naming the expected magic and version. In practice it can only mean a binary whose compiler and runtime disagree about the layout. 2. Compaction emitted the Mach-O `.section` directive for every target, so a COFF statepoint build would have failed to assemble. Rewriting is now gated to the two object formats whose syntax this module emits and whose section the runtime can find; anything else keeps LLVM's section, turning an unsupported-platform case back into a merely larger binary. Gates re-run after the change: 8/8 probes on both the explicit-bridge and RS4GC arms, byte-matching the pinned Node oracle normally and under PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 PERRY_STACKMAP_WALKER=verify. Note that the ELF path itself is still unverified on a Linux host (#7173): ELF has no `.no_dead_strip`, so whether the linker keeps a section nothing references is an open question, and the answer decides whether the map survives at all there. --- crates/perry-codegen/src/gc_map.rs | 17 +++++++++++++-- .../perry-runtime/src/gc/roots/stack_maps.rs | 21 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index 60a65712e9..4284911156 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -497,8 +497,17 @@ pub fn compact_and_assemble( let asm = fs::read_to_string(asm_path) .with_context(|| format!("Failed to read assembly at {}", asm_path.display()))?; - let elf = - !target.contains("apple") && !target.contains("darwin") && !target.contains("windows"); + // Only the two object formats whose section syntax this module emits, and + // whose section the runtime knows how to find, may be rewritten. Anything + // else (COFF today) keeps LLVM's section: emitting a Mach-O `.section` + // directive into COFF assembly would fail to assemble, turning an + // unsupported-platform case into a broken build. + let macho = target.contains("apple") || target.contains("darwin"); + let elf = !macho && !target.contains("windows") && !target.contains("msvc"); + if !macho && !elf { + return assemble(clang, target, asm_path, obj_path); + } + if let Some((rewritten, stats)) = compact_stack_map_asm(&asm, elf) { fs::write(asm_path, rewritten).with_context(|| { format!( @@ -518,6 +527,10 @@ pub fn compact_and_assemble( ); } + assemble(clang, target, asm_path, obj_path) +} + +fn assemble(clang: &Path, target: &str, asm_path: &Path, obj_path: &Path) -> Result<()> { let output = Command::new(clang) .arg("-c") .arg(asm_path) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index b4c8941415..31a34bab87 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -144,11 +144,30 @@ pub(in crate::gc) fn native_maps_active() -> bool { fn stack_maps() -> &'static StackMapIndex { STACK_MAPS.get_or_init(|| { + // No section at all is the ordinary shadow-stack build: there are no + // native frame roots to find, and an empty index is the right answer. let Some(section) = loaded_stack_map_section() else { return StackMapIndex::default(); }; + // A section that exists but does not decode is a different thing + // entirely, and it must never degrade to "no roots". The two failure + // shapes are indistinguishable downstream — both yield an empty index + // — but their consequences are not: with statepoints as the only root + // mechanism, an empty index means the collector frees live objects and + // corrupts the heap with no diagnostic at all. That is CLAUDE.md's + // fourth gate-failure mode (the gate runs, its subject never did), so + // fail loudly instead. In practice this can only mean a binary whose + // compiler and runtime disagree about the map format. let Some((mut records, roots)) = parse_gc_map(section) else { - return StackMapIndex::default(); + panic!( + "perry: the GC map section (__perry_gcmap / .perry_gcmap, {} bytes) is \ + present but could not be decoded — expected format {:?} v{}. This binary's \ + compiler and runtime disagree about the map layout; continuing would run \ + the collector with no roots and corrupt the heap silently.", + section.len(), + std::str::from_utf8(GC_MAP_MAGIC).unwrap_or("PGCM"), + GC_MAP_VERSION, + ); }; records.sort_unstable_by_key(|record| record.pc); index_records(records, roots) From 50408a955e63f4b2e7f7c4ae3e3ed1c17e4ec02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 14:15:31 +0200 Subject: [PATCH 44/53] gc: refuse to re-encode a stack map whose roots use a foreign register base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compact format stores a root's base as a single bit, FP-or-SP, using aarch64's DWARF numbers (29/31). Nothing checked that the incoming stack map actually used those. On x86-64 LLVM emits RBP=6 / RSP=7. Both would test false against SP, encode as bit 0, and decode back as aarch64's FP=29 — a wrong base, which is a wrong root address, which is a collector reading and rewriting the wrong words. No diagnostic anywhere in that chain. The native-frame-root backend is aarch64-only today (the runtime's prologue decoder and fast walker are both cfg(target_arch = "aarch64")), so this was dormant rather than live. It stops being dormant the moment anyone points PERRY_STATEPOINTS at another architecture, and it would not announce itself. Now any location whose base is neither FP nor SP aborts the rewrite and keeps LLVM's section. Falling back costs bytes; guessing costs correctness. Found by cross-compiling a probe with `--target linux` and reading the ELF: the section, its 8-byte alignment and its `.rela.perry_gcmap` relocations all came out right, but the object was x86-64 — which is what surfaced the register assumption. That ELF check also confirms the assembly-syntax path works for both object formats; what remains unverified there is whether the linker retains a section nothing references (ELF has no `.no_dead_strip`) and whether the runtime finds it, both of which need a real Linux host (#7173). Gates: 8/8 on both arms, normally and under forced evacuation with the verifying walker. --- crates/perry-codegen/src/gc_map.rs | 41 ++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index 4284911156..11e4a91c9c 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -273,11 +273,24 @@ fn decode_v3(block: &RawBlock) -> Option> { let offset = read_u32(bytes, pos + 8)? as i32; // Keep exactly what the collector keeps: 8-byte frame // slots, with the base/derived pair collapsed to one. - if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) - && size == 8 - && !roots.contains(&(dwarf_reg, offset)) - { - roots.push((dwarf_reg, offset)); + if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { + // The encoding stores the base as a single bit, + // FP-or-SP, using this architecture's DWARF numbers. + // Refuse anything else rather than silently rewriting + // it to FP: on x86-64 LLVM emits RBP=6/RSP=7, which + // would encode as "not SP" and decode back as + // aarch64's FP=29 — a wrong base, and a wrong base is + // a wrong root address. Bailing keeps LLVM's section, + // which costs bytes rather than correctness. (The + // native-frame-root backend is aarch64-only today; the + // runtime's prologue decoder and fast walker are both + // `cfg(target_arch = "aarch64")`.) + if dwarf_reg != DWARF_REG_FP_AARCH64 && dwarf_reg != DWARF_REG_SP_AARCH64 { + return None; + } + if !roots.contains(&(dwarf_reg, offset)) { + roots.push((dwarf_reg, offset)); + } } pos += 12; } @@ -328,6 +341,8 @@ fn zigzag(value: i32) -> u64 { /// DWARF register number for the stack pointer on aarch64; every other base /// this backend emits is the frame pointer. const DWARF_REG_SP_AARCH64: u16 = 31; +/// Frame pointer, the other base the single-bit encoding can express. +const DWARF_REG_FP_AARCH64: u16 = 29; fn encode_stream(functions: &[FunctionMap]) -> Vec { let mut stream = Vec::new(); @@ -665,6 +680,22 @@ mod tests { ); } + #[test] + fn foreign_register_bases_keep_llvm_section() { + // x86-64 records RBP=6 / RSP=7. The single-bit base encoding cannot + // express those, and guessing would decode them back as aarch64's + // FP=29 — a wrong base, therefore a wrong root address. Keeping + // LLVM's section costs bytes; guessing costs correctness. + let asm = sample_asm().replace( + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n", + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t6\n", + ); + assert!( + compact_stack_map_asm(&asm, true).is_none(), + "a non-FP/SP base must fall back rather than be re-encoded" + ); + } + #[test] fn no_stack_map_block_is_left_alone() { assert!(compact_stack_map_asm("\t.section\t__TEXT,__text\n\tret\n", false).is_none()); From ef3f36c9fa0b55b041140824c974943eef876f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 14:37:08 +0200 Subject: [PATCH 45/53] gc: probe live roots across a throw, and record the RS4GC/landingpad gap Nothing in the ratchet suite contained a `try` -- 0 of 8 probes -- so removing the `!has_try` statepoint exclusion was covered by no test whatsoever. A green run proved only that the eight try-free probes still worked. 09_try_catch_roots.ts exercises what the exclusion used to forbid: objects allocated inside a try surviving a collection inside the same try; locals live across a throw and read in the catch; a throw crossing several frames so the roots being rewritten sit in a caller's frame; finally on both the normal and unwinding edges; and a rethrow caught one frame up. Every survivor folds into the checksum, so a lost or stale root is a wrong number, not a crash. Its map is 1,116 bytes, the largest of any probe -- the liveness evidence that try-carrying functions now really do carry statepoint records. Explicit bridge: 9/9 against the oracle, normally and under forced evacuation with the verifying walker. RS4GC: 8/9. It cannot compile a try-carrying function -- the LLVM verifier rejects gc.relocate taking a landingpad's { ptr, i32 } result where a token is required, because statepoint-example expects a statepoint-invoke's unwind destination to carry `landingpad token` rather than the Itanium form try_stmt.rs emits. So the leanest arm on size (-131,624 B) is not the complete one; the explicit bridge (-49,072 B) is. Recorded rather than patched: it is an LLVM-convention problem, not something the compact map touches. --- .../gc_ratchet/probes/09_try_catch_roots.ts | 139 ++++++++++++++++++ docs/statepoint-gc-experiment.md | 53 +++++++ 2 files changed, 192 insertions(+) create mode 100644 benchmarks/gc_ratchet/probes/09_try_catch_roots.ts diff --git a/benchmarks/gc_ratchet/probes/09_try_catch_roots.ts b/benchmarks/gc_ratchet/probes/09_try_catch_roots.ts new file mode 100644 index 0000000000..343c19aa57 --- /dev/null +++ b/benchmarks/gc_ratchet/probes/09_try_catch_roots.ts @@ -0,0 +1,139 @@ +// GC ratchet probe: live roots held across a throw and a collection. +// +// This is the probe that had no equivalent while try/catch lowered to +// setjmp/longjmp. Under that lowering a longjmp could jump past a +// `gc.relocate`, so the relocated pointer was never written back and a local +// could be left pointing at a moved object. Functions containing `try` were +// therefore excluded from statepoints and routed to the plain-stack-map +// lowering, which is itself unsound — LLVM may record a root slot's address in +// a caller-saved register that cannot be recovered at collection time. +// +// With invoke/landingpad lowering (#7302) the unwind edge is explicit and +// relocations exist on BOTH edges, so statepoints cover try-carrying functions +// too. Nothing else in this suite has a `try` in it, so without this probe the +// newly covered case is exercised by nothing at all. +// +// What it checks, specifically: +// * objects allocated INSIDE a try survive a collection that happens inside +// the same try, and read back correctly afterwards; +// * locals live ACROSS the throw — allocated before it, read in the catch — +// still hold their contents once the collection has moved things; +// * the same holds when the throw crosses a frame boundary (thrown deep, +// caught shallow) so the roots being rewritten are in a caller's frame; +// * `finally` runs on both the normal and unwinding edges. +// +// A lost or stale root shows up as a wrong checksum rather than a crash, which +// is why every survivor is folded into the output. + +declare function gc(): void; + +const ROUNDS = 400; +const PER_ROUND = 96; + +let escape: object[] | null = null; + +class Payload { + tag: number; + body: string; + constructor(tag: number) { + this.tag = tag; + this.body = "p" + tag; + } + value(): number { + return (this.tag + this.body.length) | 0; + } +} + +// Thrown from the deepest frame so the unwind crosses several frames that hold +// live roots of their own. +function deep(level: number, seed: number): number { + if (level === 0) { + throw new Payload(seed); + } + const local = new Payload(seed + level); + const nested = deep(level - 1, seed); + // Unreachable, but keeps `local` live across the call in the eyes of any + // liveness analysis that is not lying to us. + return (local.value() + nested) | 0; +} + +function roundTrip(seed: number): number { + // Live across the whole try/catch, including the collection. + const survivors: Payload[] = []; + let acc = 0; + + try { + for (let i = 0; i < PER_ROUND; i++) { + survivors.push(new Payload(seed + i)); + } + // Collect with everything above live and reachable only from this frame. + if ((seed & 15) === 0) { + escape = survivors.slice(0, 8); + gc(); + escape = null; + } + acc = (acc + deep(6, seed)) | 0; + } catch (err) { + // The caught value must be the object that was thrown, after a collection + // that may have moved it. + const caught = err as Payload; + acc = (acc + caught.value()) | 0; + // Every survivor allocated before the throw must still be intact. + for (let i = 0; i < survivors.length; i++) { + acc = (acc + survivors[i].value()) | 0; + } + } finally { + // Runs on the unwinding edge; `survivors` must still be readable here. + acc = (acc + survivors.length) | 0; + } + + return acc; +} + +// Normal (non-throwing) exit through a try/finally, so the non-unwind edge of +// the same lowering is covered too. +function normalExit(seed: number): number { + const held = new Payload(seed); + try { + if ((seed & 31) === 0) { + gc(); + } + return held.value(); + } finally { + escape = null; + } +} + +let checksum = 0; +for (let r = 0; r < ROUNDS; r++) { + checksum = (checksum + roundTrip(r)) | 0; + checksum = (checksum + normalExit(r)) | 0; +} + +// A rethrow that is caught one frame up, with roots live in both frames. +function rethrower(seed: number): number { + const outer = new Payload(seed); + try { + try { + gc(); + throw new Payload(seed + 1); + } catch (inner) { + throw new Payload((inner as Payload).tag + outer.tag); + } + } catch (final) { + return ((final as Payload).value() + outer.value()) | 0; + } +} + +for (let r = 0; r < 32; r++) { + checksum = (checksum + rethrower(r)) | 0; +} + +gc(); +const mu = process.memoryUsage(); + +console.log("probe:09_try_catch_roots"); +console.log("checksum:" + checksum); +console.error("#gcmetric heap_used_bytes=" + mu.heapUsed); +console.error("#gcmetric heap_total_bytes=" + mu.heapTotal); +console.error("#gcmetric rss_bytes=" + mu.rss); diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 0c5dc08c78..9a302ab754 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -959,3 +959,56 @@ After that work lands: 5. Re-run the 11-way interleaved suite on an idle pinned host, with separate profiles for mutator root maintenance, relocation reloads, unwinding, and collector root scanning. + +## invoke-EH lands on main; try functions covered (2026-08-03) + +main replaced setjmp/longjmp exception lowering with `invoke`/`landingpad` +(#7302, PR #7305) and deleted `volatile_setjmp.rs` and `setjmp_abi.rs`. That +retires this branch's correctness blocker: a `longjmp` could jump past a +`gc.relocate`, which is why try-carrying functions were excluded from +statepoints and routed to the plain-stack-map lowering — itself unsound, since +LLVM may record a root slot's address in a caller-saved register that cannot +be recovered at collection time. + +The exclusion was not merely obsolete but **unrepresentable**: main deleted the +`has_try` field, so the compiler forced its removal. + +### The probe that had no equivalent + +Nothing in the suite contained a `try` — 0 of 8 probes — so the newly covered +case was exercised by nothing at all, and a green run said nothing about it. +`09_try_catch_roots.ts` closes that: objects allocated inside a `try` surviving +a collection inside the same `try`; locals live across a throw and read in the +`catch`; a throw crossing several frames so the rewritten roots sit in a +caller's frame; `finally` on both edges; and a rethrow caught one frame up. +Every survivor folds into the checksum, so a lost or stale root is a wrong +number rather than a crash. + +Its map is 1,116 bytes — the largest of any probe — which is the liveness +evidence that try-carrying functions now genuinely carry statepoint records. + +### Result, and a real limitation it exposed + +**Explicit statepoint bridge: 9/9**, byte-matching the pinned Node oracle +normally and under `PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 +PERRY_STACKMAP_WALKER=verify`. + +**RS4GC: 8/9 — it cannot compile a try-carrying function.** The LLVM verifier +rejects the module: + +``` +%lpad = landingpad { ptr, i32 } +token %r2.0.relocated = call coldcc ptr addrspace(1) + @llvm.experimental.gc.relocate.p1({ ptr, i32 } %lpad, i32 1, i32 0) +``` + +`gc.relocate`'s first operand must be a `token`. RS4GC emitted the relocates on +the unwind edge against the landing pad's `{ ptr, i32 }` result, because +`statepoint-example` expects a statepoint-invoke's unwind destination to carry +`landingpad token` rather than the Itanium form `try_stmt.rs` emits. + +This matters for the size ranking: **RS4GC is the leanest arm measured +(−131,624 B versus shadow) but cannot handle `try` yet, so the arm that is +actually complete today is the explicit bridge at −49,072 B.** Both still beat +the shadow stack on size; the RS4GC/EH interaction is the remaining work, and +it is an LLVM-convention problem rather than anything the compact map touches. From f89fa1d862dc34d306ab3ff262f2420f1821363d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 14:56:22 +0200 Subject: [PATCH 46/53] gc: RS4GC accepts try functions (landingpad token), and fix a merge regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things, both found by running arms I had not been running. 1. RS4GC could not compile any try-carrying function. It uses the unwind destination's landing pad AS the token for the relocates it inserts on the exceptional edge, so `statepoint-example` requires `landingpad token`. Perry emits the Itanium `landingpad { ptr, i32 }`, so RS4GC produced `gc.relocate({ ptr, i32 } %lpad, ...)` and the verifier rejected the module. Retyping is sound only because the pad's value is dead: try_stmt emits it to anchor the edge and branches straight on, taking the exception from the runtime rather than the pad payload. `retype_landing_pads_for_statepoints` therefore leaves a pad alone if its register is referenced anywhere — retyping a value someone reads would trade this loud failure for a silent miscompile. Whole-token register matching, so %r2 is not "used" by %r21. RS4GC goes 8/9 -> 9/9; the try probe's map is 1,931 B, the largest emitted. 2. The merge duplicated the return-site rewrite. main moved the shadow-stack pop into `for_each_final_item`, and the merge kept this branch's copy in `to_ir`, so both ran and every function with a shadow frame emitted `%shadow_pop_l_0` twice — clang rejected the module outright. This broke the DEFAULT path while all nine probes passed on both statepoint arms, because those arms route roots to statepoints and have no shadow frame. Verified now against the default arm too (9/9, both GC sections absent, which is what correct looks like there). --- crates/perry-codegen/src/function.rs | 157 +++++++++++++++++++++------ 1 file changed, 121 insertions(+), 36 deletions(-) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 2375918739..d63d4d5852 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -657,42 +657,11 @@ impl LlFunction { .unwrap_or_else(|e| match e {}); ir.push_str("}\n"); - // Return-site rewrite hooks. - // - // Shadow-stack pop (gen-GC Phase A sub-phase 2) and entry - // diagnostics both need to run before every normal return, - // regardless of which lowering path emitted it. Textual rewrite - // on the full IR catches implicit returns, Stmt::Return, and any - // hand-emitted `ret`. - let ir = if self.shadow_frame_slot.is_some() || !self.pre_return_void_calls.is_empty() { - let mut out = String::with_capacity(ir.len() + 512); - let mut seq: u32 = 0; - for line in ir.lines() { - let trimmed = line.trim_start(); - if (trimmed.starts_with("ret ") || trimmed == "ret void") - && !trimmed.starts_with("ret ptr ") - // skip rare ptr rets - { - for func_name in &self.pre_return_void_calls { - out.push_str(&format!(" call void @{}()\n", func_name)); - } - if let Some(handle_slot) = &self.shadow_frame_slot { - let load_reg = format!("%shadow_pop_l_{}", seq); - seq += 1; - out.push_str(&format!(" {} = load i64, ptr {}\n", load_reg, handle_slot)); - out.push_str(&format!( - " call void @js_shadow_frame_pop(i64 {})\n", - load_reg - )); - } - } - out.push_str(line); - out.push('\n'); - } - out - } else { - ir - }; + // The return-site rewrite hooks (shadow-stack pop, entry diagnostics) + // live in `for_each_final_item`, which the loop above already streamed + // through. This branch used to re-apply them here; after main moved + // them, doing both emitted `%shadow_pop_l_0` twice in the same + // function and clang rejected every module with a shadow frame. // Research backend: turn the existing shadow-slot binding IR into // native-frame stack maps only after lowering is complete, when every @@ -711,6 +680,27 @@ impl LlFunction { ir }; + // RS4GC uses the unwind destination's landing pad **as the token** for + // the relocates it inserts on the exceptional edge, so + // `statepoint-example` requires that pad to be `landingpad token`. + // Perry emits the Itanium `{ ptr, i32 }` form, which makes RS4GC + // produce `gc.relocate({ ptr, i32 } %lpad, ...)` and the verifier + // reject the module — a try-carrying function simply fails to compile. + // + // Retyping is safe because the pad's value is dead: `try_stmt` emits it + // purely to anchor the edge and branches straight on, taking the + // exception from the runtime rather than the pad payload. Only the type + // is load-bearing, and only to RS4GC. + // + // Conditioned on the same fact as `gc_strategy` above — a function that + // does not carry the strategy must keep the Itanium form, or its pad + // becomes untypeable for ordinary EH lowering. + let ir = if !gc_strategy.is_empty() && crate::codegen::helpers::rs4gc_enabled() { + retype_landing_pads_for_statepoints(&ir) + } else { + ir + }; + // Invoke-EH (#7302): inline invoke splits move a block's true CFG // tail behind `eh.contN:` labels; phi incoming-edge labels captured // at emit time must follow. Runs last so it sees the streamed text. @@ -1656,8 +1646,103 @@ fn lower_precise_roots_to_native_stack( out } +/// Retype Itanium landing pads to `token` for `statepoint-example`. +/// +/// RS4GC uses the unwind destination's landing pad **as the token** for the +/// relocates it inserts on the exceptional edge, so the pad must already be +/// `landingpad token`. Given `{ ptr, i32 }` it emits +/// `gc.relocate({ ptr, i32 } %lpad, ...)` and the verifier rejects the module, +/// which is why a try-carrying function failed to compile under RS4GC at all. +/// +/// This is only sound because the pad's value is **dead**: `try_stmt` emits it +/// to anchor the edge and branches straight on, taking the exception from the +/// runtime rather than the pad payload. So a pad whose register IS referenced +/// is left alone — retyping a value someone reads would swap a silent +/// miscompile for the loud one this fixes. +fn retype_landing_pads_for_statepoints(ir: &str) -> String { + const ITANIUM: &str = "landingpad { ptr, i32 } catch ptr null"; + if !ir.contains(ITANIUM) { + return ir.to_string(); + } + let mut out = String::with_capacity(ir.len()); + for line in ir.lines() { + let rewritten = match line.split_once(" = ") { + Some((reg, rest)) if rest.trim() == ITANIUM => { + let reg = reg.trim(); + // Referenced anywhere else? Then its payload is live. + let used = ir.lines().any(|other| { + !std::ptr::eq(other.as_ptr(), line.as_ptr()) && mentions_register(other, reg) + }); + if used { + None + } else { + Some(format!("{} = landingpad token cleanup", reg)) + } + } + _ => None, + }; + match rewritten { + Some(r) => out.push_str(&r), + None => out.push_str(line), + } + out.push('\n'); + } + out +} + +/// Whether `line` mentions SSA register `reg` as a whole token rather than as +/// a prefix of a longer name (`%r2` must not match `%r21`). +fn mentions_register(line: &str, reg: &str) -> bool { + let mut from = 0; + while let Some(idx) = line[from..].find(reg) { + let at = from + idx; + let after = line[at + reg.len()..].chars().next(); + if !matches!(after, Some(c) if c.is_ascii_alphanumeric() || c == '_' || c == '.') { + return true; + } + from = at + reg.len(); + } + false +} + #[cfg(test)] mod stack_map_tests { + + #[test] + fn retypes_dead_landing_pads_for_rs4gc() { + let ir = "define void @probe() {\n\ + entry:\n\ + %lp = landingpad { ptr, i32 } catch ptr null\n\ + br label %next\n\ + }\n"; + let out = super::retype_landing_pads_for_statepoints(ir); + assert!(out.contains("%lp = landingpad token cleanup"), "{out}"); + } + + #[test] + fn leaves_a_used_landing_pad_alone() { + // If the pad's payload is read, retyping it to `token` would break the + // consumer silently. Fail closed: RS4GC's loud verifier error is the + // better outcome. + let ir = "define void @probe() {\n\ + entry:\n\ + %lp = landingpad { ptr, i32 } catch ptr null\n\ + %exn = extractvalue { ptr, i32 } %lp, 0\n\ + br label %next\n\ + }\n"; + let out = super::retype_landing_pads_for_statepoints(ir); + assert!( + out.contains("%lp = landingpad { ptr, i32 } catch ptr null"), + "{out}" + ); + } + + #[test] + fn register_match_is_whole_token() { + // `%r2` must not be considered used by a mention of `%r21`. + assert!(super::mentions_register(" br label %r2", "%r2")); + assert!(!super::mentions_register(" %x = add i64 %r21, 1", "%r2")); + } use super::{ direct_callee_name, lower_precise_roots_to_native_stack, parse_direct_statepoint_call, PreciseRootBackend, From 6f134d20db195f85d87c35db37b3ba53d8566e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 14:56:41 +0200 Subject: [PATCH 47/53] =?UTF-8?q?docs:=20correct=20the=20size=20claim=20?= =?UTF-8?q?=E2=80=94=20statepoints=20tie,=20not=20win,=20after=20the=20mai?= =?UTF-8?q?n=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-merge the compact map measured -49,072 B (bridge) and -131,624 B (RS4GC) against the shadow stack. Re-measured after merging main: +496 B and +50,064 B. Main shrank every arm by ~1.7-1.8 MB but shrank SHADOW about 50 KB more than the statepoint arms, which is the whole swing. The generated-code advantage is intact (__text -151 KB bridge, -240 KB RS4GC, plus ~105 KB less __eh_frame); it is now exactly cancelled by the 189-221 KB of remaining metadata. The compaction is still load-bearing -- uncompacted that metadata is 4.2 MB and the arm loses by ~4 MB. It converted a 3.5 MB loss into a tie, not a win. Closing the axis needs fewer roots, not a tighter encoding: 221 KB for 154k roots is near this format's floor. --- docs/statepoint-gc-experiment.md | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index 9a302ab754..10d8fe3fc3 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -1012,3 +1012,38 @@ This matters for the size ranking: **RS4GC is the leanest arm measured actually complete today is the explicit bridge at −49,072 B.** Both still beat the shadow stack on size; the RS4GC/EH interaction is the remaining work, and it is an LLVM-convention problem rather than anything the compact map touches. + +### Correction: the size win does not survive the merge with main (2026-08-03) + +Re-measured on `test-drizzle-pg` after merging main, one compiler, clean +object cache per arm: + +| arm | total | `__text` | gcmap | unwind | eh_frame | vs shadow | +|---|---:|---:|---:|---:|---:|---:| +| shadow | 26,950,504 | 19,801,644 | 0 | 195,104 | 1,479,652 | — | +| bridge | 26,951,000 | 19,650,408 | 189,454 | 187,056 | 1,375,028 | **+496** | +| RS4GC | 27,000,568 | 19,562,088 | 220,936 | 187,064 | 1,374,892 | **+50,064** | + +Pre-merge the same measurement gave −49,072 (bridge) and −131,624 (RS4GC). +Main's own changes shrank every arm by ~1.7–1.8 MB, but shrank **shadow about +50 KB more than the statepoint arms**, which is the entire swing. + +What did not change is the reason to keep the compact map: statepoints still +generate less code (`__text` −151 KB for the bridge, −240 KB for RS4GC, plus +~105 KB less `__eh_frame`). Those savings are simply now cancelled by the +189–221 KB of remaining metadata. Without compaction that metadata is 4.2 MB +and the arm loses by ~4 MB, so the 18–19× is doing real work — it converted a +3.5 MB loss into a tie, not into a win. + +**Honest standing on the three axes**, post-merge: + +* **performance** — statepoints ahead (−0.93% RS4GC, measured earlier); +* **RSS** — tied; +* **file size** — the explicit bridge is *tied* with shadow (+496 B, 0.002%); + RS4GC is 50 KB behind despite the smallest `__text`, because its live-set is + larger and so is its map. + +Closing the last axis therefore needs the root set to shrink, not the encoding: +221 KB of map for 154k roots is already near this format's floor. That is the +repsel-promotion lever the earlier projection named, and it is still the +outstanding work. From e7a363ca532f1bffc9c659d426acd45a4c7a0b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 15:04:06 +0200 Subject: [PATCH 48/53] gc: unbreak the Linux build, and point the Linux gate at the compact map The gc-native-roots gate has been red on every push to this branch since the compact map landed, for two reasons I introduced. perry-runtime did not COMPILE on Linux. Removing the LLVM v3 parser orphaned read_u16 on macOS, so I deleted it -- but elf_section_vaddr is cfg(target_os = "linux") and therefore invisible to a macOS `cargo check`. Three E0425s plus one E0689 inference cascade. Restored, gated to Linux so it does not warn as dead code on the host. The gate's own liveness assert was stale: it required a non-empty .llvm_stackmaps section, which the compact rewrite deliberately removes. It now asserts BOTH directions -- .perry_gcmap present AND .llvm_stackmaps absent -- because checking only the former would still pass if compaction silently stopped running, and this project has been bitten by exactly that shape. Nothing here changes what runs on macOS; both arms remain 9/9 locally. What it buys is the first real ELF evidence: whether the linker retains a section nothing references (ELF has no .no_dead_strip) and whether the runtime finds it. That was the open question in #7173 and the gate answers it directly. --- .github/workflows/gc-native-roots.yml | 12 +++++++++--- crates/perry-runtime/src/gc/roots/stack_maps.rs | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gc-native-roots.yml b/.github/workflows/gc-native-roots.yml index 8628314860..d3addbda50 100644 --- a/.github/workflows/gc-native-roots.yml +++ b/.github/workflows/gc-native-roots.yml @@ -5,7 +5,8 @@ # oracle, natively on the Linux runner — the same matrix the branch runs on # macOS, webserver-class x86-64, and the Pi 5. Two liveness asserts keep # this from being a gate that cannot fail (CLAUDE.md's four ways): -# the binary must carry a non-empty .llvm_stackmaps section, and at least +# the binary must carry a .perry_gcmap section (and no .llvm_stackmaps, +# proving the compact rewrite ran), and at least # one probe must report a copying collection. name: gc-native-roots on: @@ -40,9 +41,14 @@ jobs: name=$(basename "$probe" .ts) node --expose-gc --experimental-strip-types "$probe" > "/tmp/$name.oracle" PERRY_STATEPOINTS=1 ./target/perry-dev/perry "$probe" -o "/tmp/$name" - # Liveness assert 1: the subject (stackmap section) must exist. + # Liveness assert 1: the subject must exist. The compact map + # replaced LLVM's section, so assert BOTH facts — the new section + # is present AND the old one is gone. Checking only the former + # would still pass if compaction silently stopped running. + readelf -S "/tmp/$name" | grep -q "\.perry_gcmap" \ + || { echo "::error::$name has no .perry_gcmap section — statepoint mode was not live"; exit 1; } readelf -S "/tmp/$name" | grep -q "\.llvm_stackmaps" \ - || { echo "::error::$name has no .llvm_stackmaps section — statepoint mode was not live"; exit 1; } + && { echo "::error::$name still carries .llvm_stackmaps — the compact rewrite did not run"; exit 1; } PERRY_STATEPOINTS=1 PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 \ "/tmp/$name" > "/tmp/$name.out" 2> "/tmp/$name.err" diff "/tmp/$name.oracle" "/tmp/$name.out" \ diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 31a34bab87..eca08764fb 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -483,6 +483,17 @@ fn read_u8(bytes: &[u8], offset: usize) -> Option { bytes.get(offset).copied() } +/// ELF section headers store their counts and offsets as 16-bit fields, so +/// this is used only by `elf_section_vaddr`. Gated to Linux because the +/// compact GC map itself needs no 16-bit reads — deleting it as "orphaned" +/// after a macOS-only `cargo check` is what broke the Linux build. +#[cfg(target_os = "linux")] +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_le_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + fn read_u32(bytes: &[u8], offset: usize) -> Option { Some(u32::from_le_bytes( bytes.get(offset..offset + 4)?.try_into().ok()?, From a10e6c2f32da1f07aa6b7ea231b5855bdd2d5870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 15:37:24 +0200 Subject: [PATCH 49/53] =?UTF-8?q?gc:=20delete=20the=20unsound=20plain=20st?= =?UTF-8?q?ack=20map=20=E2=80=94=20every=20root=20path=20now=20fails=20clo?= =?UTF-8?q?sed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plain `llvm.experimental.stackmap` lowering was the last way this backend could lose a root: LLVM may record a root slot's address as `Register R#N`, caller-saved and unrecoverable at collection time, so the collector silently misses it. Measured 3 of 60 locations on one probe. It survived as a fallback in three places, all of which failed OPEN. 1. `PreciseRootBackend::StackMap` was dead by construction. Both sites that set `stack_map_requested` are guarded by `native_stack_roots_enabled()`, which IS `statepoints_enabled() || rs4gc_enabled()`, so the `else` branch could never be reached. Variant and emitter deleted. 2. The Statepoint backend fell back to a plain map whenever a call with live roots would not parse as a statepoint — chiefly INDIRECT calls. That was a limitation of this textual parser, not of statepoints: `gc.statepoint` takes its callee as a `ptr` operand and `emit_statepoint` interpolates it verbatim, so `ptr elementtype(T) %fnptr` is as valid as `... @callee`. Indirect targets are now statepoint-able; an unknown callee simply cannot be audited as non-collecting, which is the conservative answer anyway. Anything still unparseable is a hard compile failure naming the call shape, because a loud stop beats silent heap corruption. 3. The compact-map rewriter fell back to keeping LLVM's section, and the comment claimed that "costs bytes rather than roots". That was exactly backwards. The runtime reads ONLY `__perry_gcmap`, so such a module's records sit in the binary unread and its roots are invisible — and because other modules still emit a valid section, the runtime's "present but undecodable" guard stays quiet too. Now a hard error. Evidence the removal is safe rather than merely bold, on test-drizzle-pg (133 modules, real dependency code): 23301 safepoints emitted: 23301 statepoints, 0 plain stack maps 35951 non-collecting calls skipped; 0 statepoint parser fallback(s) 129914 relocations, 0 plain-map operands Both statepoint arms build that application, and all three arms (explicit bridge, RS4GC, default shadow stack) pass 9/9 against the pinned Node oracle, under forced evacuation with the verifying walker where applicable. The report's fallback counters can now only ever read zero. Left in place because that zero is the evidence, not noise — but they are a candidate for deletion once this has soaked. --- crates/perry-codegen/src/function.rs | 147 ++++++++++-------- crates/perry-codegen/src/gc_map.rs | 31 +++- crates/perry-codegen/src/statepoint_report.rs | 26 +--- 3 files changed, 118 insertions(+), 86 deletions(-) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index d63d4d5852..933059370b 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -670,10 +670,13 @@ impl LlFunction { let ir = if self.stack_map_requested { let backend = if crate::codegen::helpers::rs4gc_enabled() { PreciseRootBackend::Rs4gc - } else if crate::codegen::helpers::statepoints_enabled() { - PreciseRootBackend::Statepoint } else { - PreciseRootBackend::StackMap + // Not `StackMap`: that variant is gone. Both sites that set + // `stack_map_requested` are guarded by + // `native_stack_roots_enabled()`, which is exactly + // `statepoints_enabled() || rs4gc_enabled()`, so this branch + // is only reachable with statepoints on. + PreciseRootBackend::Statepoint }; lower_precise_roots_to_native_stack(&ir, &self.name, self.stack_map_slot_count, backend) } else { @@ -1051,7 +1054,6 @@ fn stack_map_active_slots( #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum PreciseRootBackend { - StackMap, Statepoint, /// `PERRY_RS4GC=1` (#7174): retype every root alloca to /// `ptr addrspace(1)` with cast surgery at its load/store sites, tag the @@ -1070,7 +1072,6 @@ enum PreciseRootBackend { impl PreciseRootBackend { fn as_str(self) -> &'static str { match self { - Self::StackMap => "stack-map", Self::Statepoint => "statepoint", Self::Rs4gc => "rs4gc", } @@ -1296,7 +1297,16 @@ fn parse_direct_statepoint_call(line: &str) -> Option> { return None; } let callee = target_and_args[..open].trim(); - if !callee.starts_with('@') + // Indirect targets are statepoint-able: `gc.statepoint` takes the callee as + // a `ptr` operand, and `emit_statepoint` interpolates it verbatim, so + // `ptr elementtype(T) %fnptr` is as valid as `... @callee`. Rejecting them + // was a limitation of this textual parser, not of statepoints — and the + // fallback it forced is the unsound plain stack map. An unknown callee + // simply cannot be audited as non-collecting, which is the conservative + // (correct) answer anyway. + let direct = callee.starts_with('@'); + let indirect = callee.starts_with('%'); + if !(direct || indirect) || callee.starts_with("@llvm.") || matches!(callee, "@setjmp" | "@_setjmp" | "@longjmp" | "@_longjmp") { @@ -1348,20 +1358,6 @@ fn gc_result_suffix(ty: &str) -> Option<&'static str> { } } -fn emit_plain_stack_map(out: &mut String, line: &str, live: &[&String], map_id: u64) { - let operands = live - .iter() - .map(|ptr| format!(", ptr {ptr}")) - .collect::(); - out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); - out.push_str(&format!( - " call void (i64, i32, ...) @llvm.experimental.stackmap(i64 {map_id}, i32 0{operands})\n" - )); - out.push_str(line); - out.push('\n'); - out.push_str(" call void asm sideeffect \"\", \"~{memory}\"()\n"); -} - /// Emit one explicit statepoint relocation sequence. /// /// Perry roots remain ordinary NaN-boxed `i64` values everywhere else. At @@ -1630,15 +1626,27 @@ fn lower_precise_roots_to_native_stack( continue; } } - emit_plain_stack_map(&mut out, line, &live, map_id); - if let Some(report) = report.as_mut() { - report.note_plain_stack_map( - direct_callee.unwrap_or(""), - live.len(), - backend == PreciseRootBackend::Statepoint, - ); - } - map_id += 1; + // No statepoint could be formed for a call that has live roots. The + // old behaviour was to fall back to a plain `llvm.experimental.stackmap`, + // which is UNSOUND: LLVM may record a root slot's address as + // `Register R#N`, caller-saved and unrecoverable at collection time, + // so the collector silently misses that root. + // + // Measured on test-drizzle-pg (133 modules): 23,301 safepoints, ALL + // statepoints, 0 plain stack maps, 0 parser fallbacks. The path is not + // taken by real code, so failing closed costs nothing and removes the + // last way this backend can lose a root. A loud compile failure beats + // silent heap corruption. + panic!( + "perry: native-root lowering could not express a safepoint for \ + `{}` in @{} ({} live roots). Falling back to a plain stack map \ + here would record roots in caller-saved registers that the \ + collector cannot recover, so the compile stops instead. Report \ + this call shape on #7174.", + direct_callee.unwrap_or(""), + function_name, + live.len(), + ); } if let Some(report) = report { crate::statepoint_report::record(report); @@ -1748,10 +1756,6 @@ mod stack_map_tests { PreciseRootBackend, }; - fn lower_stack_maps(input: &str, slots: u32) -> String { - lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::StackMap) - } - fn lower_statepoints(input: &str, slots: u32) -> String { lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::Statepoint) } @@ -1769,16 +1773,25 @@ entry.0: ret i64 %r1 } "#; - let output = lower_stack_maps(input, 1); + let output = lower_statepoints(input, 1); assert!(!output.contains("@js_shadow_slot_bind")); assert!(!output.contains("@js_shadow_slot_set")); assert!(output.contains("%r0 = alloca i64\n store i64 0, ptr %r0")); - assert!(output.contains( - "@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)\n %r1 = call i64 \ - @may_collect()\n call void asm sideeffect \"\", \"~{memory}\"()" - )); + assert!( + output.contains("@llvm.experimental.gc.statepoint.p0"), + "the collecting call must become a statepoint:\n{output}" + ); + assert!( + output.contains("%r0"), + "the root slot must appear in the statepoint's live list:\n{output}" + ); assert_eq!(output.matches("store i64 0, ptr %r0").count(), 1); - assert_eq!(output.matches("@llvm.experimental.stackmap").count(), 1); + assert_eq!( + output + .matches("@llvm.experimental.gc.statepoint.p0") + .count(), + 1 + ); assert!(output.contains("call void @may_collect_again()")); } @@ -1793,13 +1806,17 @@ entry.0: ret void } "#; - let output = lower_stack_maps(input, 1); + let output = lower_statepoints(input, 1); let early = output.find("call void @early_call()").unwrap(); - let first_map = output.find("@llvm.experimental.stackmap").unwrap(); - assert!(early < first_map); - assert!(output.contains( - "@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)\n call void @late_call()" - )); + let first_map = output.find("@llvm.experimental.gc.statepoint.p0").unwrap(); + assert!( + early < first_map, + "no safepoint may reference a root before its alloca dominates:\n{output}" + ); + assert!( + output.contains("@late_call"), + "the dominated call must still be mapped:\n{output}" + ); } #[test] @@ -1821,16 +1838,10 @@ merge.3: ret void } "#; - let output = lower_stack_maps(input, 1); - assert!(output.contains( - "@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)\n call void @live_call()" - )); - assert!(!output.contains( - "@llvm.experimental.stackmap(i64 1, i32 0, ptr %r0)\n call void @dead_call()" - )); - assert!(output.contains( - "@llvm.experimental.stackmap(i64 1, i32 0, ptr %r0)\n call void @merge_call()" - )); + let output = lower_statepoints(input, 1); + assert!(output.contains("@llvm.experimental.gc.statepoint.p0")); + assert!(!output.contains("@dead_call, ptr %r0")); + assert!(output.contains("@merge_call")); } #[test] @@ -1889,20 +1900,34 @@ entry.0: } #[test] - fn statepoint_mode_falls_back_for_indirect_calls() { + fn statepoint_mode_maps_indirect_calls() { + // An indirect call used to fall back to a plain stack map, which is the + // unsound lowering: LLVM may record the root's address in a + // caller-saved register. `gc.statepoint` takes its callee as a `ptr` + // operand, so an indirect target is expressible — the restriction was + // in this textual parser, not in statepoints. let input = r#"define i64 @probe(i64 %arg, ptr %fn) { entry.0: %r0 = alloca i64 store i64 %arg, ptr %r0 call void @js_shadow_slot_bind(i32 0, ptr %r0) - %r1 = call i64 ()* %fn() + %r1 = call i64 %fn() ret i64 %r1 } "#; let output = lower_statepoints(input, 1); - assert!(output.contains("@llvm.experimental.stackmap(i64 0, i32 0, ptr %r0)")); - assert!(output.contains("%r1 = call i64 ()* %fn()")); - assert!(!output.contains("@llvm.experimental.gc.statepoint")); + assert!( + output.contains("@llvm.experimental.gc.statepoint.p0"), + "an indirect call with live roots must become a statepoint:\n{output}" + ); + assert!( + output.contains("%fn"), + "the indirect target must survive as the statepoint callee:\n{output}" + ); + assert!( + !output.contains("@llvm.experimental.stackmap"), + "no plain (unsound) stack map may remain:\n{output}" + ); } #[test] @@ -1941,7 +1966,7 @@ entry.0: ret void } "#; - for output in [lower_stack_maps(input, 1), lower_statepoints(input, 1)] { + for output in [lower_statepoints(input, 1)] { assert!(output.contains("call void @js_gc_temp_root_push(i64 %arg)")); assert!(output.contains("call void @js_write_barrier_root_nanbox(i64 %arg)")); assert_eq!( diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index 11e4a91c9c..0d182b7321 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -500,9 +500,16 @@ pub fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats /// Rewrite the stack map in `asm_path` into Perry's compact form, then /// assemble it to `obj_path`. /// -/// A module with no stack-map block, or one whose block does not parse, is -/// assembled unchanged — LLVM's section is correct, merely large, so falling -/// back costs bytes rather than roots. +/// A module with no stack-map block is assembled unchanged — there is nothing +/// to compact. +/// +/// A module that HAS a block which does not parse is a hard error, not a +/// fallback. Keeping LLVM's section there looks conservative and is not: the +/// runtime reads only `__perry_gcmap`, so that module's records would be +/// present in the binary, unread, and its roots invisible to the collector — +/// while other modules still emit a valid section, so even the "section +/// present but undecodable" guard in the runtime stays quiet. Silent lost +/// roots are precisely what this backend exists to make impossible. pub fn compact_and_assemble( clang: &Path, target: &str, @@ -523,7 +530,23 @@ pub fn compact_and_assemble( return assemble(clang, target, asm_path, obj_path); } - if let Some((rewritten, stats)) = compact_stack_map_asm(&asm, elf) { + let has_block = asm.lines().any(|l| { + let t = l.trim_start(); + t.starts_with(".section") + && (t.contains("__LLVM_STACKMAPS") || t.contains(".llvm_stackmaps")) + }); + let compacted = compact_stack_map_asm(&asm, elf); + if has_block && compacted.is_none() { + return Err(anyhow!( + "perry: this module emits an LLVM stack map that the compact-map \ + rewriter could not parse, so its GC roots would be invisible to \ + the collector (the runtime reads only the compact section). \ + Refusing to emit a binary that would lose roots silently. \ + Assembly left at: {}", + asm_path.display() + )); + } + if let Some((rewritten, stats)) = compacted { fs::write(asm_path, rewritten).with_context(|| { format!( "Failed to write compacted assembly at {}", diff --git a/crates/perry-codegen/src/statepoint_report.rs b/crates/perry-codegen/src/statepoint_report.rs index 733fc08923..4e469e860a 100644 --- a/crates/perry-codegen/src/statepoint_report.rs +++ b/crates/perry-codegen/src/statepoint_report.rs @@ -78,24 +78,6 @@ impl FunctionRecord { .entry(callee.to_string()) .or_default() += 1; } - - pub(crate) fn note_plain_stack_map( - &mut self, - callee: &str, - live_roots: usize, - is_statepoint_fallback: bool, - ) { - self.plain_stack_maps += 1; - self.stack_map_operands += live_roots as u64; - self.note_emitted_roots(live_roots); - if is_statepoint_fallback { - self.statepoint_fallbacks += 1; - *self - .fallbacks_by_callee - .entry(callee.to_string()) - .or_default() += 1; - } - } } pub fn enabled() -> bool { @@ -302,18 +284,20 @@ mod tests { record.note_call(1); record.note_skipped("@js_gc_temp_root_get"); record.note_call(1); - record.note_plain_stack_map("", 1, true); let text = render_text(std::slice::from_ref(&record)); assert!(text.contains("2 bound native root slots")); assert!(text.contains("1 non-collecting calls skipped")); - assert!(text.contains("1 statepoint parser fallback(s)")); + // The plain-map fallback is gone, so this can only ever report zero — + // which is the point: it is the report's evidence that no root was + // recorded in an unrecoverable location. + assert!(text.contains("0 statepoint parser fallback(s)")); assert!(text.contains("@js_gc_temp_root_get")); let json = render_json(&[record]); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["schema_version"], 1); assert_eq!(parsed["totals"]["relocations"], 2); - assert_eq!(parsed["totals"]["statepoint_fallbacks"], 1); + assert_eq!(parsed["totals"]["statepoint_fallbacks"], 0); } } From 76fda7f79baf316e5056fa112e8ba467c389be2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 15:49:18 +0200 Subject: [PATCH 50/53] gc: retain the compact map on ELF, and make the gate runnable on main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Linux gate answered the open ELF question from #7173, and the answer was that the map does not survive linking: `01_nursery_churn has no .perry_gcmap section`. Compaction was working — the object carries .perry_gcmap as PROGBITS/SHF_ALLOC with its relocations intact. The linker was discarding it. Perry links with -Wl,--gc-sections (link/build_and_run.rs), and nothing in the program references this section: the collector finds it by name at runtime. On Mach-O `.no_dead_strip` covers exactly this; ELF's analogue is SHF_GNU_RETAIN, so the section is now emitted "aR" rather than "a". Verified the assembler accepts it and emits flags AR. This is the failure mode the whole map format is meant to make impossible, and it was invisible on macOS: a binary that links fine, runs fine on every macOS arm, and on Linux would have had no GC map at all. Also makes the gate able to gate. It triggered only on `push: [exp/stackmap-viability]`, so on main it would never run — CLAUDE.md's second way a gate cannot fail. Now push:[main] + pull_request, with no cancel-in-progress so a main run cannot be cancelled by the next merge. Adds the changelog.d fragment the changeset-gate requires, and drops gc_map_compaction_totals plus its counters — nothing read them, and the gate asserting on the emitted binary's sections is stronger evidence than a process-local counter. --- .github/workflows/gc-native-roots.yml | 8 ++- changelog.d/7312-statepoint-native-roots.md | 72 +++++++++++++++++++++ crates/perry-codegen/src/gc_map.rs | 48 ++++++-------- 3 files changed, 99 insertions(+), 29 deletions(-) create mode 100644 changelog.d/7312-statepoint-native-roots.md diff --git a/.github/workflows/gc-native-roots.yml b/.github/workflows/gc-native-roots.yml index d3addbda50..3dc62f0f3a 100644 --- a/.github/workflows/gc-native-roots.yml +++ b/.github/workflows/gc-native-roots.yml @@ -10,8 +10,14 @@ # one probe must report a copying collection. name: gc-native-roots on: + # Must run where it can actually gate something. Branch-scoped triggers were + # right while this lived only on exp/stackmap-viability; on main that same + # filter would mean the job never runs at all — CLAUDE.md's second way a gate + # cannot fail. Cancellation is deliberately NOT set here: a `main` run that + # gets cancelled by the next merge is the third way. push: - branches: [exp/stackmap-viability] + branches: [main] + pull_request: workflow_dispatch: jobs: diff --git a/changelog.d/7312-statepoint-native-roots.md b/changelog.d/7312-statepoint-native-roots.md new file mode 100644 index 0000000000..2a3b9133fb --- /dev/null +++ b/changelog.d/7312-statepoint-native-roots.md @@ -0,0 +1,72 @@ +### Native-frame GC roots via LLVM statepoints, opt-in (#7173, #7174) + +Adds a second precise-root mechanism alongside the shadow stack, selected with +`PERRY_STATEPOINTS=1` (explicit bridge) or `PERRY_STATEPOINTS=1 PERRY_RS4GC=1` +(LLVM's `RewriteStatepointsForGC` owns statepoint and relocation insertion). +**The default path is unchanged**: without those flags nothing here runs, and +the shadow stack remains the shipping root mechanism. + +The point of the mechanism is that the forgot-to-root bug class becomes +structurally impossible — LLVM, not Perry, is responsible for knowing which +values are live across a call and for rewriting them after a collection moves +them. + +**Every root path fails closed.** The plain `llvm.experimental.stackmap` +lowering is deleted outright rather than kept as a fallback: LLVM may record a +root slot's address as `Register R#N`, caller-saved and unrecoverable at +collection time, so a fallback to it silently loses roots. It survived in three +places, all of which failed open: + +* `PreciseRootBackend::StackMap` was dead by construction (both sites setting + `stack_map_requested` are guarded by `native_stack_roots_enabled()`, which is + exactly `statepoints || rs4gc`); +* the statepoint backend fell back for calls it could not parse — chiefly + **indirect** calls. That was a limitation of Perry's textual parser, not of + statepoints: `gc.statepoint` takes its callee as a `ptr` operand, so + `ptr elementtype(T) %fnptr` is as valid as `... @callee`. Indirect targets are + now statepoint-able and anything still unparseable is a hard compile error; +* the compact-map rewriter fell back to keeping LLVM's section, which reads as + conservative and is not — the runtime reads only `__perry_gcmap`, so those + records sit unread and that module's roots go missing. Now a hard error. + +**The metadata is re-encoded rather than shipped as LLVM emits it.** Measured on +`test-drizzle-pg`, `__llvm_stackmaps` was 4.21 MB, of which >50% was data the +runtime already discarded at startup: three `Constant` slots per record +(`gc.statepoint`'s calling-convention preamble) and a duplicate of every root +(LLVM records base and derived; Perry has no interior pointers). Perry now +rewrites that block at assembly time — where LLVM prints the function addresses +as symbol names, so one text parser replaces Mach-O *and* ELF relocation +parsing plus a second link pass — into a compact map: 4,214,384 B → 224,832 B +(18.7×). The largest single lever is that **77% of records have the identical +live set as the record before them**, so a repeat flag replaces the payload; +that also lets the runtime share one copy per distinct set instead of +materialising 154k entries. + +**Try/catch is covered.** Now that exception lowering uses `invoke`/`landingpad` +(#7302), no jump can skip a `gc.relocate`, so try-carrying functions take +statepoints like any other. Under RS4GC they additionally need +`landingpad token` — RS4GC uses the landing pad *as* the relocate token — which +is sound here only because the pad's value is dead; the retype refuses if the +pad register is referenced anywhere. + +`benchmarks/gc_ratchet/probes/09_try_catch_roots.ts` is new and exists because +nothing in the suite contained a `try` at all: objects allocated inside a `try` +surviving a collection there, locals live across a throw and read in the +`catch`, a throw crossing several frames so the rewritten roots sit in a +caller's frame, `finally` on both edges, and a rethrow caught one frame up. + +**Measured on `test-drizzle-pg` (133 modules):** 23,301 safepoints, all +statepoints, 0 plain stack maps, 0 parser fallbacks, 129,914 relocations. +Binary size is a wash against the shadow stack (+496 B for the bridge, ++50,064 B for RS4GC): statepoints generate less code (`__text` −151 KB / −240 KB, +plus ~105 KB less `__eh_frame`) and that is cancelled by the remaining +189–221 KB of map. Runtime is −0.93% (RS4GC) and RSS is flat. + +Also lands three mode-independent codegen fixes that the work depended on: +codegen-unit globals are emitted only into units that reference them and +declarations are scoped to the unit that needs them (per-unit IR previously grew +with unit *count*, which is why the 13 MB `@anthropic-ai/claude-code` bundle hit +`clang: translation unit is too large` no matter how finely it was split — +885 KB → 299 KB per unit on a 4-unit module), and codegen units now compile with +bounded parallelism (`PERRY_CODEGEN_UNIT_JOBS`, default `parallelism/4` clamped +to `[1,4]`) instead of one at a time. diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index 0d182b7321..a1fb63bf8d 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -52,18 +52,23 @@ use std::collections::HashMap; use std::fs; use std::path::Path; use std::process::Command; -use std::sync::atomic::{AtomicU64, Ordering}; use anyhow::{anyhow, Context, Result}; /// Magic at the start of every emitted blob. -pub const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; +const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; /// Format version. Bump on any layout change — the runtime rejects others. -pub const GC_MAP_VERSION: u8 = 2; +const GC_MAP_VERSION: u8 = 2; /// Section the compact map is emitted into, and the label it is given. const GC_MAP_LABEL: &str = "_perry_gc_map"; const MACHO_SECTION: &str = "__PERRY_GCMAP,__perry_gcmap"; -const ELF_SECTION: &str = ".perry_gcmap,\"a\",@progbits"; +/// `R` is SHF_GNU_RETAIN, the ELF analogue of Mach-O's `.no_dead_strip`. +/// Perry links with `-Wl,--gc-sections`, and nothing in the program +/// references this section — the collector finds it by name at runtime — so +/// without RETAIN the linker discards it and the binary ships with no GC map +/// at all. Measured: the section is present in the object (PROGBITS, SHF_ALLOC, +/// with relocations) and absent from the linked binary. +const ELF_SECTION: &str = ".perry_gcmap,\"aR\",@progbits"; /// LLVM stack-map v3 location kinds. Only these two describe a frame slot; /// `Constant`/`ConstIndex` carry the statepoint preamble and `Register` cannot @@ -411,7 +416,10 @@ fn emit_asm(functions: &[FunctionMap], stream: &[u8], elf: bool) -> String { } out.push_str("\t.p2align\t3\n"); out.push_str(&format!("{GC_MAP_LABEL}:\n")); - out.push_str("\t.ascii\t\"PGCM\"\n"); + out.push_str(&format!( + "\t.ascii\t\"{}\"\n", + std::str::from_utf8(GC_MAP_MAGIC).expect("magic is ASCII") + )); out.push_str(&format!("\t.byte\t{GC_MAP_VERSION}\n")); out.push_str("\t.byte\t0\n"); out.push_str("\t.short\t0\n"); @@ -437,12 +445,12 @@ fn emit_asm(functions: &[FunctionMap], stream: &[u8], elf: bool) -> String { /// Statistics for the caller to log — a compaction that silently did nothing /// must be distinguishable from one that ran. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct GcMapStats { - pub original_bytes: usize, - pub compact_bytes: usize, - pub functions: usize, - pub records: usize, - pub roots: usize, +struct GcMapStats { + original_bytes: usize, + compact_bytes: usize, + functions: usize, + records: usize, + roots: usize, } /// Rewrite the LLVM stack-map block in `asm` into the compact map. @@ -451,7 +459,7 @@ pub struct GcMapStats { /// for a module without safepoints) or when the block does not parse — a /// module whose metadata we do not fully understand keeps LLVM's section /// rather than shipping a map that might be missing roots. -pub fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats)> { +fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats)> { let lines: Vec<&str> = asm.lines().collect(); let block = parse_block(&lines)?; let functions = decode_v3(&block)?; @@ -553,8 +561,6 @@ pub fn compact_and_assemble( asm_path.display() ) })?; - GC_MAP_ORIGINAL_BYTES.fetch_add(stats.original_bytes as u64, Ordering::Relaxed); - GC_MAP_COMPACT_BYTES.fetch_add(stats.compact_bytes as u64, Ordering::Relaxed); log::debug!( "perry-codegen: gc map {} -> {} bytes ({} functions, {} records, {} roots)", stats.original_bytes, @@ -593,20 +599,6 @@ fn assemble(clang: &Path, target: &str, asm_path: &Path, obj_path: &Path) -> Res Ok(()) } -/// Totals for the whole process, so a build can report what compaction did. -/// A run where these stay zero did not compact anything — the distinction a -/// gate needs in order to be able to fail. -static GC_MAP_ORIGINAL_BYTES: AtomicU64 = AtomicU64::new(0); -static GC_MAP_COMPACT_BYTES: AtomicU64 = AtomicU64::new(0); - -/// `(llvm_bytes, compact_bytes)` summed across every module compiled so far. -pub fn gc_map_compaction_totals() -> (u64, u64) { - ( - GC_MAP_ORIGINAL_BYTES.load(Ordering::Relaxed), - GC_MAP_COMPACT_BYTES.load(Ordering::Relaxed), - ) -} - #[cfg(test)] mod tests { use super::*; From ba879e088efacc3ddd8b96510f11fde637d3ba1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 16:30:10 +0200 Subject: [PATCH 51/53] docs: key the changelog fragment to the actual PR number (#7314) --- ...statepoint-native-roots.md => 7314-statepoint-native-roots.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{7312-statepoint-native-roots.md => 7314-statepoint-native-roots.md} (100%) diff --git a/changelog.d/7312-statepoint-native-roots.md b/changelog.d/7314-statepoint-native-roots.md similarity index 100% rename from changelog.d/7312-statepoint-native-roots.md rename to changelog.d/7314-statepoint-native-roots.md From afd3ad39669b13712ff298f8025aabd4714976cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 17:35:17 +0200 Subject: [PATCH 52/53] =?UTF-8?q?gc:=20address=20CodeRabbit=20review=20?= =?UTF-8?q?=E2=80=94=20two=20hangs/holes,=20one=20real=20format=20gap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit found nine issues worth acting on. Three were mine and material. **The gate could never pass.** `[ "$pass" -eq 8 ]` was hardcoded, and this PR adds a ninth probe, so a fully green matrix would still fail the step. Both the expected count and the stderr list are now derived from the glob, so adding a probe cannot silently break the gate or, if the literal were lowered to match, silently stop asserting full coverage. **A malformed blob hung the process.** `total_len` comes straight from the header; a zero (or too-small) value left `base` unchanged, and because the magic still matched at that offset the resynchronisation path never ran. This executes inside `OnceLock::get_or_init`, so it was a hang at the first collection rather than the fail-closed panic. Now rejects a `total_len` that cannot cover header + function table, and asserts forward progress regardless. **`unwrap_or(0)` masked a truncated function table**, mis-sizing the offset array so every later varint decoded from misaligned bytes — a wrong live set, which the fail-closed policy exists to prevent. Propagates the failure now. **COFF shipped roots the collector cannot read.** Assembling unchanged when the target is neither Mach-O nor ELF leaves LLVM's section and no `__perry_gcmap`, which is precisely the outcome the hard error two lines below exists to prevent — reached with no diagnostic. This is the same silent-roots class as the previous two commits, third instance. It refuses loudly now. **The `js_throw*` prefix rule was already unsound, not merely fragile.** CodeRabbit flagged that a future returning helper would match the prefix and lose its statepoint. The audit it rested on is ALREADY false — `js_throw_reference_error_tdz`, `js_throw_not_a_constructor` and others are declared `-> f64`, not `-> !`. Worse, since #7302 a throw unwinds rather than longjmps, so the call site is an `invoke` whose unwind edge needs relocations, and these helpers allocate the Error they raise and can therefore collect. Suppressing the safepoint left the catch handler's roots stale after a move. The arm is deleted; the family falls through to `Unknown` and is conservatively safepointed. Cost on test-drizzle-pg: 23,301 -> 24,809 statepoints. **That change then exposed a real gap in the format**, via the fail-closed error rather than via silent corruption. `@perryts/postgres/src/pool.ts` refused to compile: LLVM uses **x19** as a frame base pointer in functions with dynamic stack allocation — 66 root slots in that one module — and a single FP-or-SP bit cannot express it. The base is now a 2-bit tag (0 = FP, 1 = SP, 2 = explicit DWARF register as a following varint), format version 3. The runtime already handled arbitrary bases on the unwinder path and `chain_walkable` already disables the fast x29 walk for them, so only the encoding was the limit. The refusal added in 50408a955 is gone with the restriction that motivated it. **`caller_fp` was used before it was validated.** Every FP-relative root is based on that word and `fp_to_sp_offset` subtracts from it, while the only downstream filters were non-zero and 8-byte alignment — a corrupt frame could yield out-of-stack addresses that the collector reads and rewrites. It now gets the same bounds/alignment checks `fp` gets, before the root loop. **The analysis script understated its own numbers.** `offv` is unpacked signed and FP-relative offsets are negative; Python ints are unbounded, so `>> 31` gave -1 and `varint_len` returned 1 for every negative input. Masked to 32 bits, and `varint_len` now rejects negatives instead of silently returning 1. The reported ratios came from `otool` on real binaries rather than this model, so they stand — and the same-build figure is now measured directly from the per-module compaction log: 3,764,000 -> 203,296 B = 18.5x. Plus: the empty-report message named PERRY_STATEPOINTS twice instead of PERRY_RS4GC; `--statepoint-report`'s doc still pointed at the deleted PERRY_STACK_MAPS mode; and the changelog claimed RS4GC needs PERRY_STATEPOINTS when `native_stack_roots_enabled()` is `statepoints || rs4gc` and either activates on its own. Tests: perry-codegen 586, perry-runtime 1,673 (RUST_TEST_THREADS=1), and all three arms 9/9 including the app that exposed the x19 gap. --- .github/workflows/gc-native-roots.yml | 21 ++- changelog.d/7314-statepoint-native-roots.md | 15 +- crates/perry-codegen/src/gc_call_effects.rs | 16 +- crates/perry-codegen/src/gc_map.rs | 82 +++++----- crates/perry-codegen/src/statepoint_report.rs | 2 +- .../perry-runtime/src/gc/roots/stack_maps.rs | 146 ++++++++++++++++-- docs/src/cli/flags.md | 2 +- scripts/stackmap_anatomy.py | 13 +- 8 files changed, 233 insertions(+), 64 deletions(-) diff --git a/.github/workflows/gc-native-roots.yml b/.github/workflows/gc-native-roots.yml index 3dc62f0f3a..e53d14fe9c 100644 --- a/.github/workflows/gc-native-roots.yml +++ b/.github/workflows/gc-native-roots.yml @@ -43,7 +43,10 @@ jobs: export PERRY_RUNTIME_DIR="$PWD/target/perry-dev" export PERRY_NO_AUTO_OPTIMIZE=1 pass=0 + total=0 + errs="" for probe in benchmarks/gc_ratchet/probes/*.ts; do + total=$((total+1)) name=$(basename "$probe" .ts) node --expose-gc --experimental-strip-types "$probe" > "/tmp/$name.oracle" PERRY_STATEPOINTS=1 ./target/perry-dev/perry "$probe" -o "/tmp/$name" @@ -59,11 +62,19 @@ jobs: "/tmp/$name" > "/tmp/$name.out" 2> "/tmp/$name.err" diff "/tmp/$name.oracle" "/tmp/$name.out" \ || { echo "::error::$name output diverged from the pinned oracle"; exit 1; } + errs="$errs /tmp/$name.err" pass=$((pass+1)) done - echo "statepoint forced-evacuation matrix: $pass/8" - [ "$pass" -eq 8 ] - # Liveness assert 2: at least one probe actually collected - # (gcmetric lines are emitted on stderr by every probe). - grep -l "#gcmetric" /tmp/0*.err >/dev/null \ + # Derived from the glob, not hardcoded: a literal goes stale the + # moment a probe is added (it did — 09_try_catch_roots), and if it is + # ever lowered to match it silently stops asserting full coverage. + echo "statepoint forced-evacuation matrix: $pass/$total" + [ "$total" -gt 0 ] \ + || { echo "::error::no probes matched — the matrix ran on nothing"; exit 1; } + [ "$pass" -eq "$total" ] + # Liveness assert 2: at least one probe actually collected (gcmetric + # lines go to stderr). Collected during the loop rather than globbed + # as /tmp/0*.err, which silently depends on every probe name starting + # with a zero. + grep -l "#gcmetric" $errs >/dev/null \ || { echo "::error::no probe emitted gc metrics — the collector never ran"; exit 1; } diff --git a/changelog.d/7314-statepoint-native-roots.md b/changelog.d/7314-statepoint-native-roots.md index 2a3b9133fb..7bdb015b2d 100644 --- a/changelog.d/7314-statepoint-native-roots.md +++ b/changelog.d/7314-statepoint-native-roots.md @@ -1,10 +1,12 @@ ### Native-frame GC roots via LLVM statepoints, opt-in (#7173, #7174) -Adds a second precise-root mechanism alongside the shadow stack, selected with -`PERRY_STATEPOINTS=1` (explicit bridge) or `PERRY_STATEPOINTS=1 PERRY_RS4GC=1` -(LLVM's `RewriteStatepointsForGC` owns statepoint and relocation insertion). -**The default path is unchanged**: without those flags nothing here runs, and -the shadow stack remains the shipping root mechanism. +Adds a second precise-root mechanism alongside the shadow stack, selected by +`PERRY_STATEPOINTS=1` (explicit bridge) or `PERRY_RS4GC=1` (LLVM's +`RewriteStatepointsForGC` owns statepoint and relocation insertion). Either +one activates native roots on its own — `native_stack_roots_enabled()` is +`statepoints || rs4gc` — so `PERRY_RS4GC=1` does not require +`PERRY_STATEPOINTS=1`. **The default path is unchanged**: with neither set +nothing here runs, and the shadow stack remains the shipping root mechanism. The point of the mechanism is that the forgot-to-root bug class becomes structurally impossible — LLVM, not Perry, is responsible for knowing which @@ -37,7 +39,8 @@ runtime already discarded at startup: three `Constant` slots per record rewrites that block at assembly time — where LLVM prints the function addresses as symbol names, so one text parser replaces Mach-O *and* ELF relocation parsing plus a second link pass — into a compact map: 4,214,384 B → 224,832 B -(18.7×). The largest single lever is that **77% of records have the identical +(19.0× measured same-build; 18.5× once the conservative `js_throw` +classification below is accounted for). The largest single lever is that **77% of records have the identical live set as the record before them**, so a repeat flag replaces the payload; that also lets the runtime share one copy per distinct set instead of materialising 154k entries. diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 7f63d6eac8..2c379dd99e 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -112,7 +112,21 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_array_indexOf_jsvalue" | "js_validate_array_comparator" | "js_validate_array_map_callback" => GcCallEffect::AllocNoReentry, - name if name.starts_with("js_throw") => GcCallEffect::NeverReturns, + // NO `js_throw*` prefix arm. It used to classify the whole family + // `NeverReturns`, which suppresses the safepoint in every mode — the + // strongest classification in this table, and the only one applied by + // prefix rather than exact name. + // + // Two things make that unsafe. The audit it rested on is already + // false: `js_throw_reference_error_tdz`, `js_throw_not_a_constructor` + // and others are declared `-> f64`, not `-> !`. And since #7302 a + // throw UNWINDS rather than longjmps, so the call site is an `invoke` + // whose unwind edge needs relocations — while these helpers allocate + // the Error they raise and can therefore collect. Suppressing the + // safepoint would leave the catch handler's roots stale after a move. + // + // Falling through to `Unknown` costs a few statepoints and is + // conservative in the only direction that is safe. _ => GcCallEffect::Unknown, } } diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index a1fb63bf8d..af9bd85891 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -58,7 +58,7 @@ use anyhow::{anyhow, Context, Result}; /// Magic at the start of every emitted blob. const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; /// Format version. Bump on any layout change — the runtime rejects others. -const GC_MAP_VERSION: u8 = 2; +const GC_MAP_VERSION: u8 = 3; /// Section the compact map is emitted into, and the label it is given. const GC_MAP_LABEL: &str = "_perry_gc_map"; const MACHO_SECTION: &str = "__PERRY_GCMAP,__perry_gcmap"; @@ -279,20 +279,6 @@ fn decode_v3(block: &RawBlock) -> Option> { // Keep exactly what the collector keeps: 8-byte frame // slots, with the base/derived pair collapsed to one. if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { - // The encoding stores the base as a single bit, - // FP-or-SP, using this architecture's DWARF numbers. - // Refuse anything else rather than silently rewriting - // it to FP: on x86-64 LLVM emits RBP=6/RSP=7, which - // would encode as "not SP" and decode back as - // aarch64's FP=29 — a wrong base, and a wrong base is - // a wrong root address. Bailing keeps LLVM's section, - // which costs bytes rather than correctness. (The - // native-frame-root backend is aarch64-only today; the - // runtime's prologue decoder and fast walker are both - // `cfg(target_arch = "aarch64")`.) - if dwarf_reg != DWARF_REG_FP_AARCH64 && dwarf_reg != DWARF_REG_SP_AARCH64 { - return None; - } if !roots.contains(&(dwarf_reg, offset)) { roots.push((dwarf_reg, offset)); } @@ -366,14 +352,28 @@ fn encode_stream(functions: &[FunctionMap]) -> Vec { // delta sign-extends into a 10-byte varint and silently bloats the // map — the format must not depend on an ordering invariant held // somewhere else. + // Base is a 2-bit tag, not a single FP/SP bit: LLVM also uses a + // callee-saved register (x19 on aarch64) as a frame base pointer + // in functions with dynamic stack allocation — measured 66 root + // slots in one real module. A bit cannot express that, and the + // format must not be the reason a root is unrepresentable. + // 0 = frame pointer, 1 = stack pointer, 2 = explicit DWARF + // register number as a following varint. let mut previous: Option = None; for (reg, offset) in &record.roots { - let base_bit = u64::from(*reg == DWARF_REG_SP_AARCH64); + let tag = match *reg { + DWARF_REG_FP_AARCH64 => 0u64, + DWARF_REG_SP_AARCH64 => 1, + _ => 2, + }; let delta = match previous { None => *offset, Some(prev) => offset.wrapping_sub(prev), }; - push_varint(&mut stream, (zigzag(delta) << 1) | base_bit); + push_varint(&mut stream, (zigzag(delta) << 2) | tag); + if tag == 2 { + push_varint(&mut stream, u64::from(*reg)); + } previous = Some(*offset); } previous_roots = Some(&record.roots); @@ -456,9 +456,12 @@ struct GcMapStats { /// Rewrite the LLVM stack-map block in `asm` into the compact map. /// /// Returns `None` when there is no stack-map block to rewrite (the common case -/// for a module without safepoints) or when the block does not parse — a -/// module whose metadata we do not fully understand keeps LLVM's section -/// rather than shipping a map that might be missing roots. +/// for a module without safepoints) or when the block does not parse. +/// +/// Those two are NOT the same to the caller: no block is fine, while a block +/// that fails to parse is a hard error in `compact_and_assemble`. Keeping +/// LLVM's section in that case would look conservative and would in fact lose +/// the module's roots, because the runtime reads only the compact section. fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats)> { let lines: Vec<&str> = asm.lines().collect(); let block = parse_block(&lines)?; @@ -528,14 +531,23 @@ pub fn compact_and_assemble( .with_context(|| format!("Failed to read assembly at {}", asm_path.display()))?; // Only the two object formats whose section syntax this module emits, and - // whose section the runtime knows how to find, may be rewritten. Anything - // else (COFF today) keeps LLVM's section: emitting a Mach-O `.section` - // directive into COFF assembly would fail to assemble, turning an - // unsupported-platform case into a broken build. + // whose section the runtime knows how to find, can be rewritten. + // + // Assembling unchanged on anything else looks like a graceful degradation + // and is the opposite: the object would carry LLVM's `__llvm_stackmaps` + // and no `__perry_gcmap`, the runtime reads only the compact section, and + // the collector finds no native roots at all — the exact outcome the hard + // error below exists to prevent, reached with no diagnostic. The mode is + // opt-in, so refusing loudly costs nothing. let macho = target.contains("apple") || target.contains("darwin"); let elf = !macho && !target.contains("windows") && !target.contains("msvc"); if !macho && !elf { - return assemble(clang, target, asm_path, obj_path); + return Err(anyhow!( + "perry: native GC roots (PERRY_STATEPOINTS / PERRY_RS4GC) are not \ + supported for target `{target}` — only Mach-O and ELF have a \ + compact-map section this runtime can find. Continuing would emit \ + a binary whose GC roots are invisible to the collector." + )); } let has_block = asm.lines().any(|l| { @@ -696,19 +708,19 @@ mod tests { } #[test] - fn foreign_register_bases_keep_llvm_section() { - // x86-64 records RBP=6 / RSP=7. The single-bit base encoding cannot - // express those, and guessing would decode them back as aarch64's - // FP=29 — a wrong base, therefore a wrong root address. Keeping - // LLVM's section costs bytes; guessing costs correctness. + fn encodes_a_foreign_register_base() { + // A base that is neither FP nor SP is real: LLVM uses x19 as a frame + // base pointer in functions with dynamic stack allocation. The 2-bit + // tag carries the DWARF number explicitly rather than refusing — the + // format must never be the reason a root is unrepresentable. let asm = sample_asm().replace( "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n", - "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t6\n", - ); - assert!( - compact_stack_map_asm(&asm, true).is_none(), - "a non-FP/SP base must fall back rather than be re-encoded" + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t19\n", ); + let (out, stats) = + compact_stack_map_asm(&asm, true).expect("a foreign base must still encode"); + assert_eq!(stats.roots, 1); + assert!(out.contains("_perry_gc_map:")); } #[test] diff --git a/crates/perry-codegen/src/statepoint_report.rs b/crates/perry-codegen/src/statepoint_report.rs index 4e469e860a..b1b59a4cfa 100644 --- a/crates/perry-codegen/src/statepoint_report.rs +++ b/crates/perry-codegen/src/statepoint_report.rs @@ -199,7 +199,7 @@ pub fn render_text(records: &[FunctionRecord]) -> String { if records.is_empty() { out.push_str( "No native-stack lowering records were emitted. Enable PERRY_STATEPOINTS=1\n\ - or PERRY_STATEPOINTS=1 and ensure codegen is not served from cache.\n", + or PERRY_RS4GC=1 and ensure codegen is not served from cache.\n", ); return out; } diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index eca08764fb..77aca0824f 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -24,7 +24,7 @@ use std::sync::OnceLock; /// statepoint constant preamble and base/derived duplicates that this parser /// discarded anyway, and shipping it cost 3.9 MB on a real application. const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; -const GC_MAP_VERSION: u8 = 2; +const GC_MAP_VERSION: u8 = 3; const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct StackMapLocation { @@ -373,6 +373,15 @@ fn parse_gc_map(bytes: &[u8]) -> Option<(Vec, Vec Option<(Vec, Vec blob_end { @@ -417,12 +432,20 @@ fn parse_gc_map(bytes: &[u8]) -> Option<(Vec, Vec DWARF_REG_FP_AARCH64, + 1 => DWARF_REG_SP_AARCH64, + 2 => { + let (reg, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + u16::try_from(reg).ok()? + } + _ => return None, }; - let delta = unzigzag((value >> 1) as u32); + let delta = unzigzag((value >> 2) as u32); let offset = match last { None => delta, Some(previous_offset) => previous_offset.wrapping_add(delta), @@ -444,7 +467,11 @@ fn parse_gc_map(bytes: &[u8]) -> Option<(Vec, Vec top + { return None; } stats.records_matched = stats.records_matched.saturating_add(matched.len()); @@ -984,12 +1025,19 @@ mod tests { push_varint(&mut stream, (roots.len() as u64) << 1); let mut last: Option = None; for (reg, offset) in roots { - let bit = u64::from(*reg == DWARF_REG_SP_AARCH64); + let tag = match *reg { + DWARF_REG_FP_AARCH64 => 0u64, + DWARF_REG_SP_AARCH64 => 1, + _ => 2, + }; let delta = match last { None => *offset, Some(previous) => offset.wrapping_sub(previous), }; - push_varint(&mut stream, (zigzag(delta) << 1) | bit); + push_varint(&mut stream, (zigzag(delta) << 2) | tag); + if tag == 2 { + push_varint(&mut stream, u64::from(*reg)); + } last = Some(*offset); } } @@ -1088,6 +1136,78 @@ mod tests { ); } + #[test] + fn decodes_an_explicit_base_register() { + // LLVM uses x19 as a frame base pointer in functions with dynamic + // stack allocation — 66 root slots in one real module. A single FP/SP + // bit cannot express that, which is what forced the 2-bit base tag. + let bytes = one_map(0x1000, &[(0x10, vec![(19, -40), (29, -8)], false)]); + let (_, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!( + roots, + vec![ + StackMapLocation { + dwarf_reg: 19, + offset: -40 + }, + StackMapLocation { + dwarf_reg: 29, + offset: -8 + }, + ] + ); + } + + #[test] + fn an_explicit_base_register_disables_the_fast_walk() { + // The x29-chain walker can only recover FP and SP; anything else must + // fall back to the platform unwinder, which can. + let index = index_records( + vec![StackMapRecord { + pc: 0x1000, + function_address: 0x1000, + stack_size: 64, + roots_start: 0, + roots_len: 1, + }], + vec![StackMapLocation { + dwarf_reg: 19, + offset: -40, + }], + ); + assert!(!index.chain_walkable); + } + + #[test] + fn rejects_a_blob_whose_length_cannot_advance_the_cursor() { + // `total_len` comes straight from the header. A zero (or too-small) + // value leaves `base` where it was, and because the magic still + // matches there the resync path never runs — the loop spins forever + // inside `OnceLock::get_or_init`, hanging the process at the first + // collection instead of failing closed. + let mut bytes = simple(0x1000, 0x10, -8); + bytes[12..16].copy_from_slice(&0u32.to_le_bytes()); + assert!( + parse_gc_map(&bytes).is_none(), + "a blob that cannot advance the cursor must be rejected, not looped on" + ); + + // Long enough to look plausible, still short of header + function table. + let mut bytes = simple(0x1000, 0x10, -8); + bytes[12..16].copy_from_slice(&20u32.to_le_bytes()); + assert!(parse_gc_map(&bytes).is_none()); + } + + #[test] + fn rejects_a_truncated_function_table() { + // The record counts size the fixed-width offset array; a short read + // there must not be rounded down to zero, or every later varint is + // decoded from the wrong offset. + let bytes = simple(0x1000, 0x10, -8); + let truncated = &bytes[..20]; + assert!(parse_gc_map(truncated).is_none()); + } + #[test] fn rejects_truncated_or_wrong_version_sections() { assert!(parse_gc_map(&[]).is_none() || parse_gc_map(&[]).unwrap().0.is_empty()); diff --git a/docs/src/cli/flags.md b/docs/src/cli/flags.md index 9761f064ed..46a9f3615b 100644 --- a/docs/src/cli/flags.md +++ b/docs/src/cli/flags.md @@ -103,7 +103,7 @@ accept either the `$perryfs/` virtual path or the embed-relative key. | `--no-codegen` | Skip the `package.json` `perry.codegen` build-time steps (also `PERRY_SKIP_CODEGEN=1`). See [Project Configuration](../getting-started/project-config.md) | | `--keep-intermediates` | Keep `.o` and `.asm` intermediate files | | `--opt-report[=json]` | Report which values Perry could **not** statically type, why, and whether you can fix it. Text by default; `--opt-report=json` emits a stable schema for tooling. Also settable via `PERRY_OPT_REPORT=1` | -| `--statepoint-report[=json]` | Report native-stack GC root pressure: calls with live roots, audited non-collecting calls omitted, relocations, plain-map fallbacks, and live-root widths. Research-only; use with `PERRY_STACK_MAPS=1` or `PERRY_STATEPOINTS=1` | +| `--statepoint-report[=json]` | Report native-stack GC root pressure: calls with live roots, audited non-collecting calls omitted, relocations, plain-map fallbacks, and live-root widths. Research-only; requires `PERRY_STATEPOINTS=1` or `PERRY_RS4GC=1` (the plain stack-map mode it also named is gone) | The `--trace`/`--focus` pair localizes "compiled to the wrong thing" bugs: `perry compile foo.ts --trace hir,llvm --focus parseRow` dumps just the diff --git a/scripts/stackmap_anatomy.py b/scripts/stackmap_anatomy.py index d882cd1588..371241b9a3 100644 --- a/scripts/stackmap_anatomy.py +++ b/scripts/stackmap_anatomy.py @@ -132,6 +132,15 @@ def analyze(buf): def varint_len(value): + """Bytes a LEB128 encoding of `value` occupies. + + Rejects negatives rather than returning 1 for them: Python ints are + unbounded, so a negative slips straight past `>= 0x80` and every negative + frame offset would be counted as a single byte, understating the very + encoding this script claims to measure exactly. + """ + if value < 0: + raise ValueError(f"varint_len expects a non-negative value, got {value}") n = 1 while value >= 0x80: value >>= 7 @@ -169,7 +178,7 @@ def compact_size(buf): rec_start = pos _pid, instr_off, _res, nloc = struct.unpack_from("> 31) + zig = ((offv << 1) ^ (offv >> 31)) & 0xFFFFFFFF size += varint_len((zig << 1) | (1 if reg == 31 else 0)) roots_kept += len(seen) if (pos - rec_start) % 8: From 48c553abc846cf2214454d451060d8ac044cb918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 3 Aug 2026 18:04:10 +0200 Subject: [PATCH 53/53] gc: a stack-map record must belong to the function the ip is in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit's remaining major finding on #7314, now measured rather than assumed. `match_records` accepted the nearest safepoint within +-16 bytes, but that window is a distance, not a containment check. Functions are adjacent in .text, so an ip early in B can fall inside the window of a safepoint at the end of A — and the walkers would then use A's frame offsets against B's frame and rewrite unrelated stack words. Instrumented the whole probe suite before changing anything, because the obvious fix (require an exact pc) would have been wrong. Seven inexact matches occur; six are already rejected as out-of-window (deltas 32..64) and one is accepted at delta=8. All seven are same-function. So requiring an exact match would have DISCARDED a legitimate root, and no cross-function match happens today — the hazard is real but latent. The fix is containment, not tightening: the matched record's function must be the greatest mapped function start <= ip, which the index now precomputes. That rejects the cross-function case and keeps the legitimate near-match. Residual gap stated in the comment rather than papered over: a function with no safepoints is absent from the function list, so an ip inside one resolves to the previous mapped function. Closing that needs a per-function code extent, and Mach-O does not expose one cheaply — `Lfunc_end` covers only EH-carrying functions (5 of 43 in a sampled module) and there is no `.size` directive. All three arms remain 9/9. --- .../perry-runtime/src/gc/roots/stack_maps.rs | 76 ++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 77aca0824f..ad62fdeb0d 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -64,6 +64,9 @@ struct StackMapIndex { /// Every root slot, referenced by `StackMapRecord`'s range. Shared between /// records whose live sets are identical. roots: Vec, + /// Sorted, deduplicated start address of every function that has records. + /// Used to confirm a matched record belongs to the function `ip` is in. + function_starts: Vec, chain_walkable: bool, min_pc: usize, max_pc: usize, @@ -193,9 +196,16 @@ fn index_records(records: Vec, roots: Vec) -> }); let min_pc = records.first().map_or(usize::MAX, |record| record.pc); let max_pc = records.last().map_or(0, |record| record.pc); + let mut function_starts: Vec = records + .iter() + .map(|record| record.function_address) + .collect(); + function_starts.sort_unstable(); + function_starts.dedup(); StackMapIndex { records, roots, + function_starts, chain_walkable, min_pc, max_pc, @@ -275,13 +285,40 @@ impl StackMapIndex { if ip.abs_diff(candidate_pc) > MAX_SAFEPOINT_RETURN_DELTA { return &[]; } + // The ±16 window is a distance, not a containment check: nothing in it + // says the matched record belongs to the function `ip` is executing. + // Functions are adjacent in .text, so an `ip` early in B can sit within + // the window of a safepoint at the end of A — and the walker would then + // use A's frame offsets against B's frame and rewrite unrelated words. + // + // Require the record's function to be the one containing `ip`: the + // greatest mapped function start <= ip. Measured across the probe + // suite, every near-match is already same-function (deltas 8..64, all + // `same=true`), so this rejects only the cross-function case — and + // notably NOT the legitimate delta=8 match, which requiring an exact + // pc would have discarded along with its roots. + // + // Residual gap, stated rather than papered over: a function with no + // safepoints is absent from `function_starts`, so an `ip` inside one + // resolves to the previous mapped function. Closing that needs a + // per-function code extent, which Mach-O does not expose cheaply + // (`Lfunc_end` covers only EH-carrying functions; there is no `.size`). + let owning = self + .function_starts + .partition_point(|start| *start <= ip) + .checked_sub(1) + .map(|index| self.function_starts[index]); let first = self .records .partition_point(|record| record.pc < candidate_pc); let last = self .records .partition_point(|record| record.pc <= candidate_pc); - &self.records[first..last] + let matched = &self.records[first..last]; + match (matched.first(), owning) { + (Some(record), Some(owning)) if record.function_address == owning => matched, + _ => &[], + } } } @@ -1265,6 +1302,43 @@ mod tests { ); } + #[test] + fn rejects_a_record_from_an_adjacent_function() { + // A safepoint at the end of A must not be matched for an `ip` early in + // B just because it falls inside the +-16 window: the walker would use + // A's frame offsets against B's frame. + let index = index_records( + vec![ + StackMapRecord { + pc: 0x1ffc, + function_address: 0x1000, + stack_size: 32, + roots_start: 0, + roots_len: 1, + }, + StackMapRecord { + pc: 0x2040, + function_address: 0x2000, + stack_size: 32, + roots_start: 0, + roots_len: 1, + }, + ], + vec![StackMapLocation { + dwarf_reg: 29, + offset: -8, + }], + ); + // 0x2004 is 8 bytes past A's last safepoint but lives in B. + assert!( + index.match_records(0x2004).is_empty(), + "a record from the previous function must not match" + ); + // A same-function near-match is still accepted — requiring an exact pc + // would drop it, and the measured suite has one. + assert_eq!(index.match_records(0x2038).len(), 1); + } + #[test] fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { let rec = |pc: usize| StackMapRecord {