diff --git a/.github/workflows/gc-native-roots.yml b/.github/workflows/gc-native-roots.yml new file mode 100644 index 0000000000..e53d14fe9c --- /dev/null +++ b/.github/workflows/gc-native-roots.yml @@ -0,0 +1,80 @@ +# #7173: native-root (statepoint) GC verification on Linux. +# +# Runs the gc-ratchet probe matrix in statepoint mode under forced +# evacuation + evacuation verification, byte-diffed against the pinned Node +# oracle, natively on the Linux runner — the same matrix the branch runs on +# macOS, webserver-class x86-64, and the Pi 5. Two liveness asserts keep +# this from being a gate that cannot fail (CLAUDE.md's four ways): +# the binary must carry a .perry_gcmap section (and no .llvm_stackmaps, +# proving the compact rewrite ran), and at least +# one probe must report a copying collection. +name: gc-native-roots +on: + # Must run where it can actually gate something. Branch-scoped triggers were + # right while this lived only on exp/stackmap-viability; on main that same + # filter would mean the job never runs at all — CLAUDE.md's second way a gate + # cannot fail. Cancellation is deliberately NOT set here: a `main` run that + # gets cancelled by the next merge is the third way. + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + statepoint-linux: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: gc-native-roots + - name: Build compiler and static runtime (perry-dev profile) + run: | + export RUSTFLAGS="-Cforce-frame-pointers=yes" + cargo build --profile perry-dev -p perry -p perry-runtime-static -p perry-stdlib-static + - name: Probe matrix, statepoint mode, forced evacuation + run: | + set -euo pipefail + export PERRY_RUNTIME_DIR="$PWD/target/perry-dev" + export PERRY_NO_AUTO_OPTIMIZE=1 + pass=0 + total=0 + errs="" + for probe in benchmarks/gc_ratchet/probes/*.ts; do + total=$((total+1)) + name=$(basename "$probe" .ts) + node --expose-gc --experimental-strip-types "$probe" > "/tmp/$name.oracle" + PERRY_STATEPOINTS=1 ./target/perry-dev/perry "$probe" -o "/tmp/$name" + # Liveness assert 1: the subject must exist. The compact map + # replaced LLVM's section, so assert BOTH facts — the new section + # is present AND the old one is gone. Checking only the former + # would still pass if compaction silently stopped running. + readelf -S "/tmp/$name" | grep -q "\.perry_gcmap" \ + || { echo "::error::$name has no .perry_gcmap section — statepoint mode was not live"; exit 1; } + readelf -S "/tmp/$name" | grep -q "\.llvm_stackmaps" \ + && { echo "::error::$name still carries .llvm_stackmaps — the compact rewrite did not run"; exit 1; } + PERRY_STATEPOINTS=1 PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 \ + "/tmp/$name" > "/tmp/$name.out" 2> "/tmp/$name.err" + diff "/tmp/$name.oracle" "/tmp/$name.out" \ + || { echo "::error::$name output diverged from the pinned oracle"; exit 1; } + errs="$errs /tmp/$name.err" + pass=$((pass+1)) + done + # Derived from the glob, not hardcoded: a literal goes stale the + # moment a probe is added (it did — 09_try_catch_roots), and if it is + # ever lowered to match it silently stops asserting full coverage. + echo "statepoint forced-evacuation matrix: $pass/$total" + [ "$total" -gt 0 ] \ + || { echo "::error::no probes matched — the matrix ran on nothing"; exit 1; } + [ "$pass" -eq "$total" ] + # Liveness assert 2: at least one probe actually collected (gcmetric + # lines go to stderr). Collected during the loop rather than globbed + # as /tmp/0*.err, which silently depends on every probe name starting + # with a zero. + grep -l "#gcmetric" $errs >/dev/null \ + || { echo "::error::no probe emitted gc metrics — the collector never ran"; exit 1; } diff --git a/benchmarks/gc_ratchet/probes/09_try_catch_roots.ts b/benchmarks/gc_ratchet/probes/09_try_catch_roots.ts new file mode 100644 index 0000000000..343c19aa57 --- /dev/null +++ b/benchmarks/gc_ratchet/probes/09_try_catch_roots.ts @@ -0,0 +1,139 @@ +// GC ratchet probe: live roots held across a throw and a collection. +// +// This is the probe that had no equivalent while try/catch lowered to +// setjmp/longjmp. Under that lowering a longjmp could jump past a +// `gc.relocate`, so the relocated pointer was never written back and a local +// could be left pointing at a moved object. Functions containing `try` were +// therefore excluded from statepoints and routed to the plain-stack-map +// lowering, which is itself unsound — LLVM may record a root slot's address in +// a caller-saved register that cannot be recovered at collection time. +// +// With invoke/landingpad lowering (#7302) the unwind edge is explicit and +// relocations exist on BOTH edges, so statepoints cover try-carrying functions +// too. Nothing else in this suite has a `try` in it, so without this probe the +// newly covered case is exercised by nothing at all. +// +// What it checks, specifically: +// * objects allocated INSIDE a try survive a collection that happens inside +// the same try, and read back correctly afterwards; +// * locals live ACROSS the throw — allocated before it, read in the catch — +// still hold their contents once the collection has moved things; +// * the same holds when the throw crosses a frame boundary (thrown deep, +// caught shallow) so the roots being rewritten are in a caller's frame; +// * `finally` runs on both the normal and unwinding edges. +// +// A lost or stale root shows up as a wrong checksum rather than a crash, which +// is why every survivor is folded into the output. + +declare function gc(): void; + +const ROUNDS = 400; +const PER_ROUND = 96; + +let escape: object[] | null = null; + +class Payload { + tag: number; + body: string; + constructor(tag: number) { + this.tag = tag; + this.body = "p" + tag; + } + value(): number { + return (this.tag + this.body.length) | 0; + } +} + +// Thrown from the deepest frame so the unwind crosses several frames that hold +// live roots of their own. +function deep(level: number, seed: number): number { + if (level === 0) { + throw new Payload(seed); + } + const local = new Payload(seed + level); + const nested = deep(level - 1, seed); + // Unreachable, but keeps `local` live across the call in the eyes of any + // liveness analysis that is not lying to us. + return (local.value() + nested) | 0; +} + +function roundTrip(seed: number): number { + // Live across the whole try/catch, including the collection. + const survivors: Payload[] = []; + let acc = 0; + + try { + for (let i = 0; i < PER_ROUND; i++) { + survivors.push(new Payload(seed + i)); + } + // Collect with everything above live and reachable only from this frame. + if ((seed & 15) === 0) { + escape = survivors.slice(0, 8); + gc(); + escape = null; + } + acc = (acc + deep(6, seed)) | 0; + } catch (err) { + // The caught value must be the object that was thrown, after a collection + // that may have moved it. + const caught = err as Payload; + acc = (acc + caught.value()) | 0; + // Every survivor allocated before the throw must still be intact. + for (let i = 0; i < survivors.length; i++) { + acc = (acc + survivors[i].value()) | 0; + } + } finally { + // Runs on the unwinding edge; `survivors` must still be readable here. + acc = (acc + survivors.length) | 0; + } + + return acc; +} + +// Normal (non-throwing) exit through a try/finally, so the non-unwind edge of +// the same lowering is covered too. +function normalExit(seed: number): number { + const held = new Payload(seed); + try { + if ((seed & 31) === 0) { + gc(); + } + return held.value(); + } finally { + escape = null; + } +} + +let checksum = 0; +for (let r = 0; r < ROUNDS; r++) { + checksum = (checksum + roundTrip(r)) | 0; + checksum = (checksum + normalExit(r)) | 0; +} + +// A rethrow that is caught one frame up, with roots live in both frames. +function rethrower(seed: number): number { + const outer = new Payload(seed); + try { + try { + gc(); + throw new Payload(seed + 1); + } catch (inner) { + throw new Payload((inner as Payload).tag + outer.tag); + } + } catch (final) { + return ((final as Payload).value() + outer.value()) | 0; + } +} + +for (let r = 0; r < 32; r++) { + checksum = (checksum + rethrower(r)) | 0; +} + +gc(); +const mu = process.memoryUsage(); + +console.log("probe:09_try_catch_roots"); +console.log("checksum:" + checksum); +console.error("#gcmetric heap_used_bytes=" + mu.heapUsed); +console.error("#gcmetric heap_total_bytes=" + mu.heapTotal); +console.error("#gcmetric rss_bytes=" + mu.rss); diff --git a/changelog.d/7314-statepoint-native-roots.md b/changelog.d/7314-statepoint-native-roots.md new file mode 100644 index 0000000000..7bdb015b2d --- /dev/null +++ b/changelog.d/7314-statepoint-native-roots.md @@ -0,0 +1,75 @@ +### Native-frame GC roots via LLVM statepoints, opt-in (#7173, #7174) + +Adds a second precise-root mechanism alongside the shadow stack, selected by +`PERRY_STATEPOINTS=1` (explicit bridge) or `PERRY_RS4GC=1` (LLVM's +`RewriteStatepointsForGC` owns statepoint and relocation insertion). Either +one activates native roots on its own — `native_stack_roots_enabled()` is +`statepoints || rs4gc` — so `PERRY_RS4GC=1` does not require +`PERRY_STATEPOINTS=1`. **The default path is unchanged**: with neither set +nothing here runs, and the shadow stack remains the shipping root mechanism. + +The point of the mechanism is that the forgot-to-root bug class becomes +structurally impossible — LLVM, not Perry, is responsible for knowing which +values are live across a call and for rewriting them after a collection moves +them. + +**Every root path fails closed.** The plain `llvm.experimental.stackmap` +lowering is deleted outright rather than kept as a fallback: LLVM may record a +root slot's address as `Register R#N`, caller-saved and unrecoverable at +collection time, so a fallback to it silently loses roots. It survived in three +places, all of which failed open: + +* `PreciseRootBackend::StackMap` was dead by construction (both sites setting + `stack_map_requested` are guarded by `native_stack_roots_enabled()`, which is + exactly `statepoints || rs4gc`); +* the statepoint backend fell back for calls it could not parse — chiefly + **indirect** calls. That was a limitation of Perry's textual parser, not of + statepoints: `gc.statepoint` takes its callee as a `ptr` operand, so + `ptr elementtype(T) %fnptr` is as valid as `... @callee`. Indirect targets are + now statepoint-able and anything still unparseable is a hard compile error; +* the compact-map rewriter fell back to keeping LLVM's section, which reads as + conservative and is not — the runtime reads only `__perry_gcmap`, so those + records sit unread and that module's roots go missing. Now a hard error. + +**The metadata is re-encoded rather than shipped as LLVM emits it.** Measured on +`test-drizzle-pg`, `__llvm_stackmaps` was 4.21 MB, of which >50% was data the +runtime already discarded at startup: three `Constant` slots per record +(`gc.statepoint`'s calling-convention preamble) and a duplicate of every root +(LLVM records base and derived; Perry has no interior pointers). Perry now +rewrites that block at assembly time — where LLVM prints the function addresses +as symbol names, so one text parser replaces Mach-O *and* ELF relocation +parsing plus a second link pass — into a compact map: 4,214,384 B → 224,832 B +(19.0× measured same-build; 18.5× once the conservative `js_throw` +classification below is accounted for). The largest single lever is that **77% of records have the identical +live set as the record before them**, so a repeat flag replaces the payload; +that also lets the runtime share one copy per distinct set instead of +materialising 154k entries. + +**Try/catch is covered.** Now that exception lowering uses `invoke`/`landingpad` +(#7302), no jump can skip a `gc.relocate`, so try-carrying functions take +statepoints like any other. Under RS4GC they additionally need +`landingpad token` — RS4GC uses the landing pad *as* the relocate token — which +is sound here only because the pad's value is dead; the retype refuses if the +pad register is referenced anywhere. + +`benchmarks/gc_ratchet/probes/09_try_catch_roots.ts` is new and exists because +nothing in the suite contained a `try` at all: objects allocated inside a `try` +surviving a collection there, locals live across a throw and read in the +`catch`, a throw crossing several frames so the rewritten roots sit in a +caller's frame, `finally` on both edges, and a rethrow caught one frame up. + +**Measured on `test-drizzle-pg` (133 modules):** 23,301 safepoints, all +statepoints, 0 plain stack maps, 0 parser fallbacks, 129,914 relocations. +Binary size is a wash against the shadow stack (+496 B for the bridge, ++50,064 B for RS4GC): statepoints generate less code (`__text` −151 KB / −240 KB, +plus ~105 KB less `__eh_frame`) and that is cancelled by the remaining +189–221 KB of map. Runtime is −0.93% (RS4GC) and RSS is flat. + +Also lands three mode-independent codegen fixes that the work depended on: +codegen-unit globals are emitted only into units that reference them and +declarations are scoped to the unit that needs them (per-unit IR previously grew +with unit *count*, which is why the 13 MB `@anthropic-ai/claude-code` bundle hit +`clang: translation unit is too large` no matter how finely it was split — +885 KB → 299 KB per unit on a 4-unit module), and codegen units now compile with +bounded parallelism (`PERRY_CODEGEN_UNIT_JOBS`, default `parallelism/4` clamped +to `[1,4]`) instead of one at a time. diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index f13844aee3..10123d4cb9 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -74,6 +74,67 @@ pub(super) fn shadow_stack_enabled() -> bool { }) } +/// Research-only moving-GC backend using LLVM's explicit statepoint +/// relocation sequence (`PERRY_STATEPOINTS=1`). +/// +/// The standalone plain-stack-map mode (`PERRY_STACK_MAPS`) was deleted per +/// the GC knob kill-policy after the quiet-host matrix: statepoints matched +/// it within timer quantization, and it is structurally unsound — LLVM's +/// stackmap intrinsic can record a root slot's address as `Register R#N` +/// (caller-saved, unrecoverable at collection time), making those roots +/// invisible to the collector by construction. The plain-map LOWERING +/// survives only as this mode's internal fallback for `try`/setjmp +/// functions and unsupported call forms. The Register hazard exists there +/// too, which is why shrinking the fallback set is the remaining +/// correctness work for this backend, tracked in the experiment doc. +pub(crate) fn statepoints_enabled() -> bool { + matches!( + std::env::var("PERRY_STATEPOINTS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) +} + +/// `PERRY_RS4GC=1` — research pipeline for #7174: root allocas become +/// `ptr addrspace(1)`, functions are tagged `gc "statepoint-example"`, and +/// each module is piped through `opt -passes='function(mem2reg), +/// rewrite-statepoints-for-gc'` before clang. LLVM then inserts every +/// statepoint, relocation, and downstream-use rewrite itself — replacing the +/// explicit bridge's hand emission and its conservative CFG-union liveness. +/// Requires an `opt` binary (`PERRY_LLVM_OPT`, Homebrew LLVM, or PATH). +pub(crate) fn rs4gc_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_RS4GC").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + +/// Whether precise roots should use a native-stack metadata backend rather +/// than Perry's heap-backed shadow frame. +pub(crate) fn native_stack_roots_enabled() -> bool { + statepoints_enabled() || rs4gc_enabled() +} + +/// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract +/// (research, `exp/stackmap-viability`). The runtime enforces that a +/// precise-root collection only begins at a declared safepoint; under that +/// guarantee, audited allocate-but-never-reenter helpers +/// (`GcCallEffect::AllocNoReentry`) need no statepoint. Participates in both +/// build and object cache keys. +pub(crate) fn gc_safepoint_only_contract_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref(), + Ok("1") | Ok("on") | Ok("true") | Ok("strict") + ) + }) +} + /// Inline shadow-slot store gate (#7088). Default ON. /// /// When enabled, a store to a GC-rooted local is emitted as an address diff --git a/crates/perry-codegen/src/expr/shadow_slot.rs b/crates/perry-codegen/src/expr/shadow_slot.rs index 00d03be235..b41fd6b57a 100644 --- a/crates/perry-codegen/src/expr/shadow_slot.rs +++ b/crates/perry-codegen/src/expr/shadow_slot.rs @@ -206,6 +206,19 @@ pub(crate) fn emit_shadow_slot_bind_for_local(ctx: &mut FnCtx<'_>, local_id: u32 return; }; ctx.shadow_slots_bound.insert(slot_idx); + if crate::codegen::helpers::native_stack_roots_enabled() { + // Kept temporarily as a textual marker: LlFunction's final stack-map + // lowering records `slot_idx -> local_slot` and removes this call. + // The incremental root barrier remains real because the native slot + // can be updated after an in-flight cycle scanned this frame. + ctx.block().call_void( + "js_shadow_slot_bind", + &[(I32, &slot_idx.to_string()), (PTR, &local_slot)], + ); + let value_bits = ctx.block().load(I64, &local_slot); + emit_persistent_shadow_root_barrier(ctx, &value_bits); + return; + } // #7088: the hot per-store root write. Emitted inline against this // activation's cached `ShadowStackState` pointer when it has one; falls // through to the call otherwise. diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 44f13e740f..933059370b 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -115,6 +115,14 @@ pub struct LlFunction { shadow_frame_push: Option, /// Slot count currently baked into that push line. shadow_frame_slot_count: u32, + /// Research backend: preserve the existing precise-root slot numbering, + /// but encode the slots in LLVM stack maps instead of allocating a + /// parallel runtime shadow frame. + stack_map_requested: bool, + /// Logical root slots reserved by the existing liveness analysis. The + /// final IR pass resolves these indices to the native allocas named by + /// `js_shadow_slot_bind` calls, removes the calls, and emits stack maps. + stack_map_slot_count: u32, /// Runtime hooks emitted immediately before each non-pointer `ret`. /// Entry/module-init functions use this for process-level diagnostics /// that must run regardless of which block reaches the normal epilogue. @@ -215,6 +223,8 @@ impl LlFunction { shadow_frame_post_init_region: false, shadow_frame_push: None, shadow_frame_slot_count: 0, + stack_map_requested: false, + stack_map_slot_count: 0, pre_return_void_calls: Vec::new(), } } @@ -255,6 +265,13 @@ impl LlFunction { } fn enable_shadow_frame_inner(&mut self, slot_count: u32, post_init: bool) { + if crate::codegen::helpers::native_stack_roots_enabled() { + self.shadow_frame_requested = true; + self.shadow_frame_post_init_region = post_init; + self.stack_map_requested = slot_count != 0; + self.stack_map_slot_count = slot_count; + return; + } if self.shadow_frame_slot.is_some() { return; } @@ -342,6 +359,12 @@ impl LlFunction { if !self.shadow_frame_requested { return None; } + if crate::codegen::helpers::native_stack_roots_enabled() { + let idx = self.stack_map_slot_count; + self.stack_map_slot_count += 1; + self.stack_map_requested = true; + return Some(idx); + } if self.shadow_frame_push.is_none() { let post_init = self.shadow_frame_post_init_region; self.emit_shadow_frame_push(0, post_init); @@ -588,16 +611,43 @@ impl LlFunction { } else { "" }; + // The native-stack walker recovers frames through the x29 chain, so + // every generated function must link one; without the attribute, + // textual-IR input gets no frame-pointer default from the clang + // driver and LLVM may omit the chain even while saving x29. + let frame_pointer = if crate::codegen::helpers::native_stack_roots_enabled() { + " \"frame-pointer\"=\"non-leaf\"" + } else { + "" + }; + // #7174: the `!has_try` exclusion is gone with the field. Try/catch no + // longer lowers to setjmp/longjmp (#7302), so nothing can jump past a + // `gc.relocate` any more and statepoints cover every function. + let gc_strategy = if self.stack_map_requested + && (crate::codegen::helpers::statepoints_enabled() + || crate::codegen::helpers::rs4gc_enabled()) + { + " gc \"statepoint-example\"" + } else { + "" + }; // Invoke-EH (#7302): functions containing landing/funclet pads name - // their personality on the define line (LLVM: `define ... [fn attrs] - // [personality] { ... }`). + // their personality on the define line. LLVM's grammar orders these + // `[fn attrs] [gc] [personality]`, so the strategy precedes it. let personality = match self.personality { Some(p) => format!(" personality ptr @{}", p), None => String::new(), }; let mut ir = format!( - "define {}{} @{}({}){}{} {{\n", - linkage, self.return_type, self.name, param_str, attrs, personality + "define {}{} @{}({}){}{}{}{} {{\n", + linkage, + self.return_type, + self.name, + param_str, + attrs, + frame_pointer, + gc_strategy, + personality ); self.for_each_final_line::(&mut |line| { ir.push_str(line); @@ -607,6 +657,53 @@ impl LlFunction { .unwrap_or_else(|e| match e {}); ir.push_str("}\n"); + // The return-site rewrite hooks (shadow-stack pop, entry diagnostics) + // live in `for_each_final_item`, which the loop above already streamed + // through. This branch used to re-apply them here; after main moved + // them, doing both emitted `%shadow_pop_l_0` twice in the same + // function and clang rejected every module with a shadow frame. + + // Research backend: turn the existing shadow-slot binding IR into + // native-frame stack maps only after lowering is complete, when every + // lazily-reserved scalar root and every call site is visible. + // + let ir = if self.stack_map_requested { + let backend = if crate::codegen::helpers::rs4gc_enabled() { + PreciseRootBackend::Rs4gc + } else { + // Not `StackMap`: that variant is gone. Both sites that set + // `stack_map_requested` are guarded by + // `native_stack_roots_enabled()`, which is exactly + // `statepoints_enabled() || rs4gc_enabled()`, so this branch + // is only reachable with statepoints on. + PreciseRootBackend::Statepoint + }; + lower_precise_roots_to_native_stack(&ir, &self.name, self.stack_map_slot_count, backend) + } else { + ir + }; + + // RS4GC uses the unwind destination's landing pad **as the token** for + // the relocates it inserts on the exceptional edge, so + // `statepoint-example` requires that pad to be `landingpad token`. + // Perry emits the Itanium `{ ptr, i32 }` form, which makes RS4GC + // produce `gc.relocate({ ptr, i32 } %lpad, ...)` and the verifier + // reject the module — a try-carrying function simply fails to compile. + // + // Retyping is safe because the pad's value is dead: `try_stmt` emits it + // purely to anchor the edge and branches straight on, taking the + // exception from the runtime rather than the pad payload. Only the type + // is load-bearing, and only to RS4GC. + // + // Conditioned on the same fact as `gc_strategy` above — a function that + // does not carry the strategy must keep the Itanium form, or its pad + // becomes untypeable for ordinary EH lowering. + let ir = if !gc_strategy.is_empty() && crate::codegen::helpers::rs4gc_enabled() { + retype_landing_pads_for_statepoints(&ir) + } else { + ir + }; + // Invoke-EH (#7302): inline invoke splits move a block's true CFG // tail behind `eh.contN:` labels; phi incoming-edge labels captured // at emit time must follow. Runs last so it sees the streamed text. @@ -791,3 +888,1095 @@ pub enum FinalItem<'a> { /// A typed instruction — the native backend constructs it directly. Inst(&'a crate::inst::LlInst), } + +fn parse_shadow_bind(line: &str) -> Option<(usize, String)> { + let rest = line + .trim() + .strip_prefix("call void @js_shadow_slot_bind(i32 ")?; + let (idx, ptr) = rest.split_once(", ptr ")?; + let ptr = ptr.strip_suffix(')')?.trim(); + Some((idx.parse().ok()?, ptr.to_string())) +} + +fn parse_shadow_set(line: &str) -> Option<(usize, String)> { + let rest = line + .trim() + .strip_prefix("call void @js_shadow_slot_set(i32 ")?; + let (idx, value) = rest.split_once(", i64 ")?; + let value = value.strip_suffix(')')?.trim(); + Some((idx.parse().ok()?, value.to_string())) +} + +/// Compute a conservative set of active logical shadow slots before each IR +/// line. Joins use union ("active on any incoming path"), so a stale local can +/// be retained but a live root cannot be omitted. +fn stack_map_active_slots( + lines: &[&str], + slot_count: u32, +) -> Vec>> { + use std::collections::{HashMap, HashSet, VecDeque}; + + #[derive(Debug)] + struct Block { + first_line: usize, + end_line: usize, + successors: Vec, + } + + fn label_name(line: &str) -> Option<&str> { + if line.starts_with(char::is_whitespace) { + return None; + } + line.strip_suffix(':') + .filter(|name| !name.is_empty() && !name.starts_with(';')) + } + + fn referenced_labels(line: &str) -> Vec<&str> { + let mut labels = Vec::new(); + let mut rest = line; + while let Some(pos) = rest.find("label %") { + let after = &rest[pos + "label %".len()..]; + let len = after + .bytes() + .take_while(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b'$') + }) + .count(); + if len == 0 { + break; + } + labels.push(&after[..len]); + rest = &after[len..]; + } + labels + } + + let labels: Vec<(usize, &str)> = lines + .iter() + .enumerate() + .filter_map(|(idx, line)| label_name(line).map(|name| (idx, name))) + .collect(); + let mut states = vec![None; lines.len()]; + if labels.is_empty() { + return states; + } + + let label_to_block: HashMap<&str, usize> = labels + .iter() + .enumerate() + .map(|(block, (_, name))| (*name, block)) + .collect(); + let mut blocks: Vec = labels + .iter() + .enumerate() + .map(|(block, (label_line, _))| Block { + first_line: label_line + 1, + end_line: labels + .get(block + 1) + .map_or(lines.len(), |(next_line, _)| *next_line), + successors: Vec::new(), + }) + .collect(); + for block in &mut blocks { + let mut seen = HashSet::new(); + for line in &lines[block.first_line..block.end_line] { + for label in referenced_labels(line) { + if let Some(&successor) = label_to_block.get(label) { + if seen.insert(successor) { + block.successors.push(successor); + } + } + } + } + } + + fn apply_root_op(state: &mut HashSet, line: &str, slot_count: u32) { + if let Some((idx, _)) = parse_shadow_bind(line) { + if idx < slot_count as usize { + state.insert(idx); + } + } else if let Some((idx, value)) = parse_shadow_set(line) { + if idx < slot_count as usize { + if value == "0" { + state.remove(&idx); + } else { + state.insert(idx); + } + } + } + } + + let mut entries: Vec>> = vec![None; blocks.len()]; + entries[0] = Some(HashSet::new()); + let mut work = VecDeque::from([0usize]); + while let Some(block_idx) = work.pop_front() { + let Some(mut state) = entries[block_idx].clone() else { + continue; + }; + let block = &blocks[block_idx]; + for line in &lines[block.first_line..block.end_line] { + apply_root_op(&mut state, line, slot_count); + } + for &successor in &block.successors { + let changed = match &mut entries[successor] { + Some(existing) => { + let old_len = existing.len(); + existing.extend(state.iter().copied()); + existing.len() != old_len + } + entry @ None => { + *entry = Some(state.clone()); + true + } + }; + if changed { + work.push_back(successor); + } + } + } + + for (block_idx, block) in blocks.iter().enumerate() { + let Some(mut state) = entries[block_idx].clone() else { + continue; + }; + for (line_idx, line) in lines + .iter() + .enumerate() + .take(block.end_line) + .skip(block.first_line) + { + states[line_idx] = Some(state.clone()); + apply_root_op(&mut state, line, slot_count); + } + } + states +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PreciseRootBackend { + Statepoint, + /// `PERRY_RS4GC=1` (#7174): retype every root alloca to + /// `ptr addrspace(1)` with cast surgery at its load/store sites, tag the + /// function `gc "statepoint-example"`, mark audited non-collecting + /// callees `"gc-leaf-function"` at the call site, and emit NO per-call + /// safepoint machinery — `opt -passes='function(mem2reg), + /// rewrite-statepoints-for-gc'` promotes the allocas to SSA and inserts + /// every statepoint, relocation, and downstream-use rewrite itself. + /// After mem2reg, each former load site is a cast site, which is exactly + /// the placement RS4GC needs to rewrite uses with relocated values. + /// Fail-closed: any use of a root alloca outside the recognized + /// load/store shapes bails the whole function to the Statepoint backend. + Rs4gc, +} + +impl PreciseRootBackend { + fn as_str(self) -> &'static str { + match self { + Self::Statepoint => "statepoint", + Self::Rs4gc => "rs4gc", + } + } +} + +/// RS4GC surgery (#7174): retype root allocas to `ptr addrspace(1)` and cast +/// at every recognized load/store site. Returns `None` when any root alloca +/// appears in an unrecognized shape (the caller falls back to the explicit +/// statepoint backend for the whole function). +fn lower_roots_for_rs4gc(lines: &[&str], root_ptrs: &[String]) -> Option { + let roots: std::collections::HashSet<&str> = root_ptrs.iter().map(String::as_str).collect(); + let mut out = String::with_capacity(lines.len() * 48 + root_ptrs.len() * 96); + let mut cast_counter = 0usize; + + for line in lines { + if parse_shadow_bind(line).is_some() || parse_shadow_set(line).is_some() { + continue; + } + let trimmed = line.trim_start(); + + // Root-alloca definition: retype + null-init (mem2reg needs a + // dominating definition for paths that read before the first bind, + // same reason the i64 zero-init existed). + // Root locals are emitted as `alloca double` (the NaN-box home) or + // occasionally `alloca i64`; both become an addrspace(1) slot. + if let Some(reg) = trimmed + .strip_suffix("= alloca i64") + .or_else(|| trimmed.strip_suffix("= alloca double")) + .map(str::trim_end) + .filter(|reg| roots.contains(reg)) + { + out.push_str(&format!(" {reg} = alloca ptr addrspace(1)\n")); + out.push_str(&format!(" store ptr addrspace(1) null, ptr {reg}\n")); + continue; + } + + let mut handled = false; + for ptr in root_ptrs { + if let Some(rest) = trimmed.strip_prefix("store i64 ") { + if let Some(value) = rest.strip_suffix(&format!(", ptr {ptr}")) { + let value = value.trim(); + if value == "0" { + out.push_str(&format!(" store ptr addrspace(1) null, ptr {ptr}\n")); + } else { + cast_counter += 1; + out.push_str(&format!( + " %rs4gc.s{cast_counter} = inttoptr i64 {value} to ptr addrspace(1)\n store ptr addrspace(1) %rs4gc.s{cast_counter}, ptr {ptr}\n" + )); + } + handled = true; + break; + } + } + if let Some(rest) = trimmed.strip_prefix("store double ") { + if let Some(value) = rest.strip_suffix(&format!(", ptr {ptr}")) { + let value = value.trim(); + cast_counter += 1; + out.push_str(&format!( + " %rs4gc.b{cast_counter} = bitcast double {value} to i64\n %rs4gc.s{cast_counter} = inttoptr i64 %rs4gc.b{cast_counter} to ptr addrspace(1)\n store ptr addrspace(1) %rs4gc.s{cast_counter}, ptr {ptr}\n" + )); + handled = true; + break; + } + } + if trimmed + == format!( + "{} = load i64, ptr {ptr}", + trimmed.split(' ').next().unwrap_or("") + ) + { + let result = trimmed.split(' ').next().unwrap_or(""); + out.push_str(&format!( + " {result}.rs4p = load ptr addrspace(1), ptr {ptr}\n {result} = ptrtoint ptr addrspace(1) {result}.rs4p to i64\n" + )); + handled = true; + break; + } + if trimmed + == format!( + "{} = load double, ptr {ptr}", + trimmed.split(' ').next().unwrap_or("") + ) + { + let result = trimmed.split(' ').next().unwrap_or(""); + out.push_str(&format!( + " {result}.rs4p = load ptr addrspace(1), ptr {ptr}\n {result}.rs4i = ptrtoint ptr addrspace(1) {result}.rs4p to i64\n {result} = bitcast i64 {result}.rs4i to double\n" + )); + handled = true; + break; + } + } + if handled { + continue; + } + + // Fail closed: any other appearance of a root alloca name. + if root_ptrs.iter().any(|ptr| { + line.contains(ptr.as_str()) + && line + .split(|c: char| !(c.is_alphanumeric() || c == '%' || c == '_' || c == '.')) + .any(|tok| tok == ptr) + }) { + return None; + } + + // Audited non-collecting callees become RS4GC leaf calls: the pass + // will not treat them as safepoints, transferring the call-effect + // table wholesale. AllocNoReentry keeps its contract gating. + let is_call = trimmed.starts_with("call ") + || trimmed.contains(" = call ") + || trimmed.starts_with("tail call ") + || trimmed.contains(" = tail call "); + // Inline asm must be marked leaf explicitly: RS4GC otherwise rewrites + // it into a statepoint whose callee is the asm value, which the + // verifier rejects outright ("Cannot take the address of an inline + // asm!"). Found on the Claude Code bundle, where other codegen paths + // emit zero-instruction asm barriers. + if is_call && trimmed.ends_with(')') && trimmed.contains(" asm ") { + out.push_str(line.trim_end()); + out.push_str(" \"gc-leaf-function\"\n"); + continue; + } + if is_call && trimmed.ends_with(')') && !trimmed.contains(" asm ") { + if let Some(callee) = direct_callee_name(line) { + let leaf = match crate::gc_call_effects::classify_direct_callee(callee) { + crate::gc_call_effects::GcCallEffect::CannotCollect + | crate::gc_call_effects::GcCallEffect::NeverReturns => true, + crate::gc_call_effects::GcCallEffect::AllocNoReentry => { + crate::codegen::helpers::gc_safepoint_only_contract_enabled() + } + crate::gc_call_effects::GcCallEffect::Unknown => false, + }; + if leaf && !callee.starts_with("llvm.") { + out.push_str(line.trim_end()); + out.push_str(" \"gc-leaf-function\"\n"); + continue; + } + } + } + + out.push_str(line); + out.push('\n'); + } + Some(out) +} + +#[derive(Debug, Eq, PartialEq)] +struct DirectCall<'a> { + result: Option<&'a str>, + return_type: &'a str, + callee: &'a str, + args: Vec<&'a str>, + arg_types: Vec<&'a str>, +} + +fn split_call_args(args: &str) -> Option> { + if args.trim().is_empty() { + return Some(Vec::new()); + } + let mut out = Vec::new(); + let mut depth = 0i32; + let mut start = 0usize; + for (idx, ch) in args.char_indices() { + match ch { + '(' | '[' | '{' | '<' => depth += 1, + ')' | ']' | '}' | '>' => { + depth -= 1; + if depth < 0 { + return None; + } + } + ',' if depth == 0 => { + out.push(args[start..idx].trim()); + start = idx + 1; + } + _ => {} + } + } + if depth != 0 { + return None; + } + out.push(args[start..].trim()); + Some(out) +} + +fn statepoint_scalar_type(arg: &str) -> Option<&str> { + let ty = arg.split_ascii_whitespace().next()?; + matches!( + ty, + "i1" | "i8" | "i16" | "i32" | "i64" | "i128" | "float" | "double" | "ptr" + ) + .then_some(ty) +} + +/// Parse the deliberately small direct-call subset emitted by `LlBlock`. +/// +/// Calls with tail markers, operand attributes, aggregate types, inline asm, +/// indirect targets, or call-site suffixes stay on the plain stack-map +/// fallback. That keeps the research mode correct while making its explicit +/// statepoint coverage measurable and easy to expand. +fn parse_direct_statepoint_call(line: &str) -> Option> { + let trimmed = line.trim(); + let (result, call) = if let Some(call) = trimmed.strip_prefix("call ") { + (None, call) + } else { + let (result, call) = trimmed.split_once(" = call ")?; + (Some(result.trim()), call) + }; + let (return_type, target_and_args) = call.split_once(' ')?; + if !matches!( + return_type, + "void" | "i1" | "i8" | "i16" | "i32" | "i64" | "i128" | "float" | "double" | "ptr" + ) { + return None; + } + if return_type != "void" && result.is_none() { + return None; + } + let open = target_and_args.find('(')?; + let close = target_and_args.rfind(')')?; + if close + 1 != target_and_args.len() { + return None; + } + let callee = target_and_args[..open].trim(); + // Indirect targets are statepoint-able: `gc.statepoint` takes the callee as + // a `ptr` operand, and `emit_statepoint` interpolates it verbatim, so + // `ptr elementtype(T) %fnptr` is as valid as `... @callee`. Rejecting them + // was a limitation of this textual parser, not of statepoints — and the + // fallback it forced is the unsound plain stack map. An unknown callee + // simply cannot be audited as non-collecting, which is the conservative + // (correct) answer anyway. + let direct = callee.starts_with('@'); + let indirect = callee.starts_with('%'); + if !(direct || indirect) + || callee.starts_with("@llvm.") + || matches!(callee, "@setjmp" | "@_setjmp" | "@longjmp" | "@_longjmp") + { + return None; + } + let args = split_call_args(&target_and_args[open + 1..close])?; + let arg_types = args + .iter() + .map(|arg| statepoint_scalar_type(arg)) + .collect::>>()?; + Some(DirectCall { + result, + return_type, + callee, + args, + arg_types, + }) +} + +/// Return a direct callee name without the leading `@`. +/// +/// This accepts more call syntax than the statepoint parser because the +/// GC-effect audit only needs to recognize a direct target. Unsupported and +/// indirect forms return `None` and therefore stay conservative. +fn direct_callee_name(line: &str) -> Option<&str> { + let call = line.trim().split_once("call ")?.1; + let args_open = call.find('(')?; + let target = call[..args_open].trim(); + let name = target.split_ascii_whitespace().last()?.strip_prefix('@')?; + (!name.is_empty() + && name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '$'))) + .then_some(name) +} + +fn gc_result_suffix(ty: &str) -> Option<&'static str> { + match ty { + "i1" => Some("i1"), + "i8" => Some("i8"), + "i16" => Some("i16"), + "i32" => Some("i32"), + "i64" => Some("i64"), + "i128" => Some("i128"), + "float" => Some("f32"), + "double" => Some("f64"), + "ptr" => Some("p0"), + _ => None, + } +} + +/// Emit one explicit statepoint relocation sequence. +/// +/// Perry roots remain ordinary NaN-boxed `i64` values everywhere else. At +/// this boundary we load each live word, carry its exact bits through a +/// temporary addrspace(1) pointer, and convert the `gc.relocate` result back +/// into the existing slot. LLVM therefore owns the spill/reload and the +/// post-safepoint SSA transition without requiring a whole-program +/// representation change for this prototype. +fn emit_statepoint(out: &mut String, call: &DirectCall<'_>, live: &[&String], statepoint_id: u64) { + for (root_idx, ptr) in live.iter().enumerate() { + out.push_str(&format!( + " %perry_sp_bits_{statepoint_id}_{root_idx} = load i64, ptr {ptr}\n" + )); + out.push_str(&format!( + " %perry_sp_root_{statepoint_id}_{root_idx} = inttoptr i64 \ + %perry_sp_bits_{statepoint_id}_{root_idx} to ptr addrspace(1)\n" + )); + } + + let function_type = format!("{} ({})", call.return_type, call.arg_types.join(", ")); + let call_args = call + .args + .iter() + .map(|arg| format!(", {arg}")) + .collect::(); + let gc_live = live + .iter() + .enumerate() + .map(|(root_idx, _)| format!("ptr addrspace(1) %perry_sp_root_{statepoint_id}_{root_idx}")) + .collect::>() + .join(", "); + out.push_str(&format!( + " %perry_sp_token_{statepoint_id} = call token (i64, i32, ptr, i32, i32, ...) \ + @llvm.experimental.gc.statepoint.p0(i64 {statepoint_id}, i32 0, \ + ptr elementtype({function_type}) {}, i32 {}, i32 0{call_args}, i32 0, i32 0) \ + [\"gc-live\"({gc_live})]\n", + call.callee, + call.args.len() + )); + + if let Some(result) = call.result { + let suffix = gc_result_suffix(call.return_type) + .expect("non-void statepoint return type was validated by the parser"); + out.push_str(&format!( + " {result} = call {} @llvm.experimental.gc.result.{suffix}(token \ + %perry_sp_token_{statepoint_id})\n", + call.return_type + )); + } + + for (root_idx, ptr) in live.iter().enumerate() { + out.push_str(&format!( + " %perry_sp_relocated_{statepoint_id}_{root_idx} = call ptr addrspace(1) \ + @llvm.experimental.gc.relocate.p1(token %perry_sp_token_{statepoint_id}, \ + i32 {root_idx}, i32 {root_idx})\n" + )); + out.push_str(&format!( + " %perry_sp_relocated_bits_{statepoint_id}_{root_idx} = ptrtoint \ + ptr addrspace(1) %perry_sp_relocated_{statepoint_id}_{root_idx} to i64\n" + )); + out.push_str(&format!( + " store i64 %perry_sp_relocated_bits_{statepoint_id}_{root_idx}, ptr {ptr}\n" + )); + } +} + +/// Lower Perry's existing precise-root operations to native-stack metadata. +/// +/// The old binding calls already name exactly the mutable native alloca that a +/// moving collection must rewrite. We use them as compile-time markers: +/// +/// * collect `logical slot -> native alloca`; +/// * remove the runtime bind calls and shadow-frame traffic; +/// * compute conservative per-call liveness from bind/clear markers without +/// mutating the native slot; +/// * either place a plain stack map before a call, or replace a supported call +/// with a statepoint/result/relocate sequence. +/// +/// Statepoint mode deliberately retains a plain-stack-map fallback for call +/// forms outside the narrow parser above. The fallback preserves correctness +/// while the report records how much of real Perry code reaches the explicit +/// relocation path. +fn lower_precise_roots_to_native_stack( + ir: &str, + function_name: &str, + slot_count: u32, + backend: PreciseRootBackend, +) -> String { + let lines: Vec<&str> = ir.lines().collect(); + let active_slots = stack_map_active_slots(&lines, slot_count); + let mut roots: Vec> = vec![None; slot_count as usize]; + for line in &lines { + if let Some((idx, ptr)) = parse_shadow_bind(line) { + if let Some(root) = roots.get_mut(idx) { + match root { + Some(existing) => { + debug_assert_eq!( + existing, &ptr, + "one precise-root slot must not bind two native allocas" + ); + } + None => *root = Some(ptr), + } + } + } + } + + let slot_roots = roots; + let root_ptrs: Vec = slot_roots.iter().flatten().cloned().collect(); + let mut report = crate::statepoint_report::enabled().then(|| { + crate::statepoint_report::FunctionRecord::new( + function_name, + backend.as_str(), + slot_count, + root_ptrs.len(), + ) + }); + // RS4GC runs BEFORE the empty-roots early return on purpose: a function + // can reserve slots (so it carries `gc "statepoint-example"`) yet bind + // none, and it still contains inline asm that RS4GC would rewrite into an + // invalid statepoint. Found on the Claude Code bundle, where the early + // return skipped leaf-marking and the verifier aborted with "Cannot take + // the address of an inline asm!". + if backend == PreciseRootBackend::Rs4gc { + if let Some(out) = lower_roots_for_rs4gc(&lines, &root_ptrs) { + if let Some(mut report) = report { + report.note_call(root_ptrs.len()); + crate::statepoint_report::record(report); + } + return out; + } + return lower_precise_roots_to_native_stack( + ir, + function_name, + slot_count, + PreciseRootBackend::Statepoint, + ); + } + + if root_ptrs.is_empty() { + let out = ir + .lines() + .filter(|line| parse_shadow_bind(line).is_none() && parse_shadow_set(line).is_none()) + .map(|line| format!("{line}\n")) + .collect(); + if let Some(report) = report { + crate::statepoint_report::record(report); + } + return out; + } + + let mut out = String::with_capacity(ir.len() + root_ptrs.len() * 128); + let mut available = std::collections::HashSet::::new(); + let mut initialized = std::collections::HashSet::::new(); + let mut map_id = 0u64; + + for (line_idx, line) in lines.iter().enumerate() { + if parse_shadow_bind(line).is_some() { + // Compile-time marker only. The real slot is already populated by + // the local store immediately preceding this old bind. + continue; + } + if parse_shadow_set(line).is_some() { + // This marker changes stack-map liveness, not the program local. + // Shadow-stack clears only flipped SLOT_ACTIVE for the same + // reason: a value can be semantically read after its final + // GC-capable call. + continue; + } + + // A stack-map operand must dominate the intrinsic. Root allocas are + // normally entry-hoisted, but tracking definitions here also handles + // the few block-local scalar-replacement slots without emitting + // invalid SSA. + for ptr in &root_ptrs { + if line.trim_start().starts_with(&format!("{ptr} = ")) { + available.insert(ptr.clone()); + } + } + + out.push_str(line); + out.push('\n'); + + // Slots can be named by a stack map before their source-level `let` + // executes. Zero them directly after the alloca so an earlier + // safepoint never exposes uninitialized stack bytes as roots. + for ptr in &root_ptrs { + if available.contains(ptr) + && !initialized.contains(ptr) + && line.trim_start().starts_with(&format!("{ptr} = alloca ")) + { + out.push_str(&format!(" store i64 0, ptr {ptr}\n")); + initialized.insert(ptr.clone()); + } + } + + // Insert before calls, not after. Rebuild the tail when the line just + // appended is a call so the intrinsic's instruction offset is the + // actual call-site offset in the final machine function. + let trimmed = line.trim_start(); + let is_call = trimmed.starts_with("call ") + || trimmed.contains(" = call ") + || trimmed.starts_with("tail call ") + || trimmed.contains(" = tail call "); + if !is_call || trimmed.contains("@llvm.experimental.stackmap") { + continue; + } + let active = active_slots.get(line_idx).and_then(Option::as_ref); + let live: Vec<&String> = slot_roots + .iter() + .enumerate() + .filter(|(idx, _)| active.is_some_and(|slots| slots.contains(idx))) + .filter_map(|(_, ptr)| ptr.as_ref()) + .filter(|ptr| available.contains(*ptr) && initialized.contains(*ptr)) + .collect(); + if let Some(report) = report.as_mut() { + report.note_call(live.len()); + } + if live.is_empty() { + continue; + } + + let direct_callee = direct_callee_name(line); + let is_compiler_only = direct_callee.is_some_and(|callee| callee.starts_with("llvm.")) + || trimmed.contains("call void asm "); + let cannot_collect = direct_callee.is_some_and(|callee| { + match crate::gc_call_effects::classify_direct_callee(callee) { + crate::gc_call_effects::GcCallEffect::CannotCollect => true, + // Control never returns here: no relocation is consumed and + // the frame's roots are dead past the call. Deeper frames + // carry their own records. + crate::gc_call_effects::GcCallEffect::NeverReturns => true, + // Under the explicit-safepoint contract the runtime + // guarantees these helpers' triggers never consume this + // frame's precise roots (they defer to a declared safepoint + // or collect behind a forced conservative scan), so the + // call site needs no metadata. Without the contract they + // stay safepoints. + crate::gc_call_effects::GcCallEffect::AllocNoReentry => { + crate::codegen::helpers::gc_safepoint_only_contract_enabled() + } + crate::gc_call_effects::GcCallEffect::Unknown => false, + } + }); + if is_compiler_only || cannot_collect { + // LLVM intrinsics, zero-instruction compiler barriers, and + // runtime helpers in the audited GC-effect table cannot enter + // Perry's allocator. Neither native-stack backend needs metadata + // around them. + if let Some(report) = report.as_mut() { + report.note_skipped(direct_callee.unwrap_or("")); + } + continue; + } + + // Move the call line behind the intrinsic. + let call_len = line.len() + 1; + out.truncate(out.len() - call_len); + if backend == PreciseRootBackend::Statepoint { + if let Some(call) = parse_direct_statepoint_call(line) { + emit_statepoint(&mut out, &call, &live, map_id); + if let Some(report) = report.as_mut() { + report.note_statepoint(call.callee.trim_start_matches('@'), live.len()); + } + map_id += 1; + continue; + } + } + // No statepoint could be formed for a call that has live roots. The + // old behaviour was to fall back to a plain `llvm.experimental.stackmap`, + // which is UNSOUND: LLVM may record a root slot's address as + // `Register R#N`, caller-saved and unrecoverable at collection time, + // so the collector silently misses that root. + // + // Measured on test-drizzle-pg (133 modules): 23,301 safepoints, ALL + // statepoints, 0 plain stack maps, 0 parser fallbacks. The path is not + // taken by real code, so failing closed costs nothing and removes the + // last way this backend can lose a root. A loud compile failure beats + // silent heap corruption. + panic!( + "perry: native-root lowering could not express a safepoint for \ + `{}` in @{} ({} live roots). Falling back to a plain stack map \ + here would record roots in caller-saved registers that the \ + collector cannot recover, so the compile stops instead. Report \ + this call shape on #7174.", + direct_callee.unwrap_or(""), + function_name, + live.len(), + ); + } + if let Some(report) = report { + crate::statepoint_report::record(report); + } + out +} + +/// Retype Itanium landing pads to `token` for `statepoint-example`. +/// +/// RS4GC uses the unwind destination's landing pad **as the token** for the +/// relocates it inserts on the exceptional edge, so the pad must already be +/// `landingpad token`. Given `{ ptr, i32 }` it emits +/// `gc.relocate({ ptr, i32 } %lpad, ...)` and the verifier rejects the module, +/// which is why a try-carrying function failed to compile under RS4GC at all. +/// +/// This is only sound because the pad's value is **dead**: `try_stmt` emits it +/// to anchor the edge and branches straight on, taking the exception from the +/// runtime rather than the pad payload. So a pad whose register IS referenced +/// is left alone — retyping a value someone reads would swap a silent +/// miscompile for the loud one this fixes. +fn retype_landing_pads_for_statepoints(ir: &str) -> String { + const ITANIUM: &str = "landingpad { ptr, i32 } catch ptr null"; + if !ir.contains(ITANIUM) { + return ir.to_string(); + } + let mut out = String::with_capacity(ir.len()); + for line in ir.lines() { + let rewritten = match line.split_once(" = ") { + Some((reg, rest)) if rest.trim() == ITANIUM => { + let reg = reg.trim(); + // Referenced anywhere else? Then its payload is live. + let used = ir.lines().any(|other| { + !std::ptr::eq(other.as_ptr(), line.as_ptr()) && mentions_register(other, reg) + }); + if used { + None + } else { + Some(format!("{} = landingpad token cleanup", reg)) + } + } + _ => None, + }; + match rewritten { + Some(r) => out.push_str(&r), + None => out.push_str(line), + } + out.push('\n'); + } + out +} + +/// Whether `line` mentions SSA register `reg` as a whole token rather than as +/// a prefix of a longer name (`%r2` must not match `%r21`). +fn mentions_register(line: &str, reg: &str) -> bool { + let mut from = 0; + while let Some(idx) = line[from..].find(reg) { + let at = from + idx; + let after = line[at + reg.len()..].chars().next(); + if !matches!(after, Some(c) if c.is_ascii_alphanumeric() || c == '_' || c == '.') { + return true; + } + from = at + reg.len(); + } + false +} + +#[cfg(test)] +mod stack_map_tests { + + #[test] + fn retypes_dead_landing_pads_for_rs4gc() { + let ir = "define void @probe() {\n\ + entry:\n\ + %lp = landingpad { ptr, i32 } catch ptr null\n\ + br label %next\n\ + }\n"; + let out = super::retype_landing_pads_for_statepoints(ir); + assert!(out.contains("%lp = landingpad token cleanup"), "{out}"); + } + + #[test] + fn leaves_a_used_landing_pad_alone() { + // If the pad's payload is read, retyping it to `token` would break the + // consumer silently. Fail closed: RS4GC's loud verifier error is the + // better outcome. + let ir = "define void @probe() {\n\ + entry:\n\ + %lp = landingpad { ptr, i32 } catch ptr null\n\ + %exn = extractvalue { ptr, i32 } %lp, 0\n\ + br label %next\n\ + }\n"; + let out = super::retype_landing_pads_for_statepoints(ir); + assert!( + out.contains("%lp = landingpad { ptr, i32 } catch ptr null"), + "{out}" + ); + } + + #[test] + fn register_match_is_whole_token() { + // `%r2` must not be considered used by a mention of `%r21`. + assert!(super::mentions_register(" br label %r2", "%r2")); + assert!(!super::mentions_register(" %x = add i64 %r21, 1", "%r2")); + } + use super::{ + direct_callee_name, lower_precise_roots_to_native_stack, parse_direct_statepoint_call, + PreciseRootBackend, + }; + + fn lower_statepoints(input: &str, slots: u32) -> String { + lower_precise_roots_to_native_stack(input, "probe", slots, PreciseRootBackend::Statepoint) + } + + #[test] + fn lowers_bind_and_liveness_clear_to_native_frame_maps() { + let input = r#"define i64 @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 @may_collect() + call void @js_shadow_slot_set(i32 0, i64 0) + call void @may_collect_again() + ret i64 %r1 +} +"#; + let output = lower_statepoints(input, 1); + assert!(!output.contains("@js_shadow_slot_bind")); + assert!(!output.contains("@js_shadow_slot_set")); + assert!(output.contains("%r0 = alloca i64\n store i64 0, ptr %r0")); + assert!( + output.contains("@llvm.experimental.gc.statepoint.p0"), + "the collecting call must become a statepoint:\n{output}" + ); + assert!( + output.contains("%r0"), + "the root slot must appear in the statepoint's live list:\n{output}" + ); + assert_eq!(output.matches("store i64 0, ptr %r0").count(), 1); + assert_eq!( + output + .matches("@llvm.experimental.gc.statepoint.p0") + .count(), + 1 + ); + assert!(output.contains("call void @may_collect_again()")); + } + + #[test] + fn does_not_reference_a_root_before_its_alloca_dominates() { + let input = r#"define void @probe() { +entry.0: + call void @early_call() + %r0 = alloca i64 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @late_call() + ret void +} +"#; + let output = lower_statepoints(input, 1); + let early = output.find("call void @early_call()").unwrap(); + let first_map = output.find("@llvm.experimental.gc.statepoint.p0").unwrap(); + assert!( + early < first_map, + "no safepoint may reference a root before its alloca dominates:\n{output}" + ); + assert!( + output.contains("@late_call"), + "the dominated call must still be mapped:\n{output}" + ); + } + + #[test] + fn unions_root_liveness_at_control_flow_joins() { + let input = r#"define void @probe(i1 %cond) { +entry.0: + %r0 = alloca i64 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + br i1 %cond, label %live.1, label %dead.2 +live.1: + call void @live_call() + br label %merge.3 +dead.2: + call void @js_shadow_slot_set(i32 0, i64 0) + call void @dead_call() + br label %merge.3 +merge.3: + call void @merge_call() + ret void +} +"#; + let output = lower_statepoints(input, 1); + assert!(output.contains("@llvm.experimental.gc.statepoint.p0")); + assert!(!output.contains("@dead_call, ptr %r0")); + assert!(output.contains("@merge_call")); + } + + #[test] + fn parses_the_scalar_direct_call_subset() { + assert_eq!( + direct_callee_name(" %r7 = call double @foo(i64 %r1, ptr %r2)"), + Some("foo") + ); + assert_eq!( + direct_callee_name(" %r7 = call i64 ()* %fn()"), + None, + "an indirect target must not be inferred from its arguments" + ); + assert_eq!( + parse_direct_statepoint_call(" %r7 = call double @foo(i64 %r1, ptr %r2)"), + Some(super::DirectCall { + result: Some("%r7"), + return_type: "double", + callee: "@foo", + args: vec!["i64 %r1", "ptr %r2"], + arg_types: vec!["i64", "ptr"], + }) + ); + assert!(parse_direct_statepoint_call( + " %r7 = call double (i64, ptr)* %fn(i64 %r1, ptr %r2)" + ) + .is_none()); + assert!(parse_direct_statepoint_call(" call void @llvm.assume(i1 %ok)").is_none()); + assert!(parse_direct_statepoint_call(" %r7 = tail call i64 @foo()").is_none()); + } + + #[test] + fn lowers_direct_calls_to_explicit_statepoint_relocations() { + let input = r#"define i64 @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 @may_collect(i64 %arg) + ret i64 %r1 +} +"#; + let output = lower_statepoints(input, 1); + assert!(!output.contains("call i64 @may_collect")); + assert!(!output.contains("asm sideeffect")); + assert!(output + .contains("%perry_sp_root_0_0 = inttoptr i64 %perry_sp_bits_0_0 to ptr addrspace(1)")); + assert!(output.contains( + "ptr elementtype(i64 (i64)) @may_collect, i32 1, i32 0, i64 %arg, i32 0, i32 0" + )); + assert!(output + .contains("%r1 = call i64 @llvm.experimental.gc.result.i64(token %perry_sp_token_0)")); + assert!(output + .contains("@llvm.experimental.gc.relocate.p1(token %perry_sp_token_0, i32 0, i32 0)")); + assert!(output.contains("store i64 %perry_sp_relocated_bits_0_0, ptr %r0")); + } + + #[test] + fn statepoint_mode_maps_indirect_calls() { + // An indirect call used to fall back to a plain stack map, which is the + // unsound lowering: LLVM may record the root's address in a + // caller-saved register. `gc.statepoint` takes its callee as a `ptr` + // operand, so an indirect target is expressible — the restriction was + // in this textual parser, not in statepoints. + let input = r#"define i64 @probe(i64 %arg, ptr %fn) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + %r1 = call i64 %fn() + ret i64 %r1 +} +"#; + let output = lower_statepoints(input, 1); + assert!( + output.contains("@llvm.experimental.gc.statepoint.p0"), + "an indirect call with live roots must become a statepoint:\n{output}" + ); + assert!( + output.contains("%fn"), + "the indirect target must survive as the statepoint callee:\n{output}" + ); + assert!( + !output.contains("@llvm.experimental.stackmap"), + "no plain (unsound) stack map may remain:\n{output}" + ); + } + + #[test] + fn statepoint_mode_does_not_map_non_allocating_llvm_intrinsics() { + let input = r#"define void @probe(i64 %arg, i1 %condition) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @llvm.assume(i1 %condition) + call void @may_collect() + ret void +} +"#; + let output = lower_statepoints(input, 1); + assert!(output.contains("call void @llvm.assume(i1 %condition)")); + assert_eq!( + output + .matches("@llvm.experimental.gc.statepoint.p0") + .count(), + 1 + ); + assert!(!output.contains("@llvm.experimental.stackmap")); + } + + #[test] + fn audited_non_collecting_helpers_are_not_safepoints_in_either_backend() { + let input = r#"define void @probe(i64 %arg) { +entry.0: + %r0 = alloca i64 + store i64 %arg, ptr %r0 + call void @js_shadow_slot_bind(i32 0, ptr %r0) + call void @js_gc_temp_root_push(i64 %arg) + call void @js_write_barrier_root_nanbox(i64 %arg) + call void @js_gc_loop_safepoint() + ret void +} +"#; + for output in [lower_statepoints(input, 1)] { + assert!(output.contains("call void @js_gc_temp_root_push(i64 %arg)")); + assert!(output.contains("call void @js_write_barrier_root_nanbox(i64 %arg)")); + assert_eq!( + output.matches("@llvm.experimental.stackmap").count() + + output + .matches("@llvm.experimental.gc.statepoint.p0") + .count(), + 1, + "only the explicit collection boundary should be a safepoint:\n{output}" + ); + } + } +} diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs new file mode 100644 index 0000000000..2c379dd99e --- /dev/null +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -0,0 +1,213 @@ +//! Perry-GC call effects for native-stack safepoint lowering. +//! +//! This is deliberately narrower than LLVM's memory-effect attributes. A +//! helper may mutate runtime metadata, take a lock, or allocate through the +//! system allocator and still be safe to omit as a Perry GC safepoint. The +//! only question answered here is: can this call enter Perry's collector? +//! +//! Unknown is the safe default. Adding a helper to the allowlist requires +//! auditing the complete runtime call graph for `gc_check_trigger`, +//! `js_gc_collect`, `js_gc_loop_safepoint`, or another route into collection. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum GcCallEffect { + CannotCollect, + /// May allocate — and therefore arm a GC trigger — but never runs a + /// collection synchronously inside the call and never re-enters generated + /// JS (no getters, setters, valueOf/toString coercion, or callbacks). + /// + /// Only meaningful under `PERRY_GC_SAFEPOINT_ONLY`: the runtime contract + /// guarantees any trigger these helpers arm either defers to a declared + /// safepoint (moving) or collects behind a forced conservative scan (the + /// alloc-point valve), so the caller's precise frame roots are never + /// consumed at this call site and it needs no statepoint. Without the + /// contract these remain safepoints. + AllocNoReentry, + /// The callee never returns to this call site (audited 2026-08-01: every + /// `js_throw*` helper funnels into `exception::js_throw`, which is + /// `-> !` — the `f64` results are unreachable ABI shape). No relocation + /// can ever be consumed downstream and the frame's roots are dead past + /// the call, so the site needs no metadata in ANY mode. Values the + /// helper itself holds are its own frame's responsibility + /// (`RuntimeHandleScope`/temp roots), exactly as for every helper call. + NeverReturns, + Unknown, +} + +/// Classify one direct LLVM callee name, without the leading `@`. +pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { + match name { + // Pure/read-only ABI helpers audited in `module::helper_decl_attrs`. + "js_nanbox_pointer" + | "js_nanbox_get_pointer" + | "js_typed_f64_arg_guard" + | "js_typed_i32_arg_guard" + | "js_typed_i1_arg_guard" + | "js_typed_i1_arg_to_raw" + | "js_typed_i32_arg_to_raw" + | "js_typed_string_arg_guard" + | "js_is_truthy" + | "js_typed_feedback_plain_array_index_get_guard" + | "js_typed_feedback_numeric_array_index_get_guard" + | "js_typed_feedback_plain_array_index_set_guard" + | "js_typed_feedback_numeric_array_index_set_guard" + | "js_typed_feedback_numeric_array_push_guard" + | "js_array_numeric_value_to_raw_f64" + // `gc/roots/temp_roots.rs`: TLS vector operations and an incremental + // marking barrier only. They never run a Perry collection. + | "js_gc_temp_root_push" + | "js_gc_temp_root_get" + | "js_gc_temp_root_set" + | "js_gc_temp_root_truncate" + // `gc/barrier.rs`: remembered-set / incremental-marking maintenance. + | "js_write_barrier" + | "js_write_barrier_slot" + | "js_write_barrier_root_heap_word" + | "js_write_barrier_root_nanbox" + // `gc/layout.rs`: side-table metadata updates only. + | "js_gc_note_slot_layout" + | "js_gc_note_slot_layout_aware" + | "js_gc_init_typed_shape_layout" + | "js_gc_init_unboxed_object_layout" + // `typed_feedback.rs`: counters/registries only. This intentionally + // does not include feedback wrappers that perform the actual object + // get/set operation. + | "js_typed_feedback_record_guard_pass" + | "js_typed_feedback_record_guard_fail" + | "js_typed_feedback_record_fallback_call" + | "js_typed_feedback_class_field_get_guard" + | "js_typed_feedback_class_field_set_guard" + | "js_typed_feedback_observe_property_get" + | "js_typed_feedback_observe_property_set" + // Refcount writes and array-layout observations; none enters GC. + | "js_string_addref" + | "js_string_addref_if_heap_string" + | "js_array_clear_numeric_layout" + | "js_array_note_numeric_write" + | "js_array_is_numeric_f64_layout" + // TLS dynamic-call context only. + | "js_implicit_this_set" + | "js_new_target_get" + | "js_new_target_set" => GcCallEffect::CannotCollect, + // Audited allocate-but-never-reenter helpers (2026-07-31): each body + // was checked for closure invocation, coercion (valueOf/toString), + // and accessor dispatch — none present, and none takes a receiver + // that could route through user code (`js_array_length` takes a + // typed `*const ArrayHeader`, not a JSValue). The forced-evacuation + // probe gates backstop the audit. + "js_closure_alloc_singleton" + | "js_object_alloc_class_inline_keys" + | "js_array_push_f64" + | "js_array_length" + | "js_array_slice_values" + // Second audit round (2026-08-01): ctor-return semantics check + // (inspects the returned value, calls nothing), strict-equality + // indexOf scan (strict equality never runs user code), and the two + // callback-type validators (type check + static-message throw; their + // throw path is the audited noreturn funnel). Deliberately NOT + // admitted: js_value_length_f64 — its plain-object arm calls + // js_object_get_field_by_name_f64, a transitive getter path; and + // js_array_get_f64 — hole/accessor paths. + | "js_ctor_return_override" + | "js_array_indexOf_jsvalue" + | "js_validate_array_comparator" + | "js_validate_array_map_callback" => GcCallEffect::AllocNoReentry, + // NO `js_throw*` prefix arm. It used to classify the whole family + // `NeverReturns`, which suppresses the safepoint in every mode — the + // strongest classification in this table, and the only one applied by + // prefix rather than exact name. + // + // Two things make that unsafe. The audit it rested on is already + // false: `js_throw_reference_error_tdz`, `js_throw_not_a_constructor` + // and others are declared `-> f64`, not `-> !`. And since #7302 a + // throw UNWINDS rather than longjmps, so the call site is an `invoke` + // whose unwind edge needs relocations — while these helpers allocate + // the Error they raise and can therefore collect. Suppressing the + // safepoint would leave the catch handler's roots stale after a move. + // + // Falling through to `Unknown` costs a few statepoints and is + // conservative in the only direction that is safe. + _ => GcCallEffect::Unknown, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audited_runtime_bookkeeping_cannot_collect() { + for name in [ + "js_gc_temp_root_push", + "js_write_barrier_root_nanbox", + "js_gc_note_slot_layout", + "js_typed_feedback_record_guard_pass", + "js_string_addref", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::CannotCollect, + "{name}" + ); + } + } + + #[test] + fn collection_and_unknown_calls_stay_conservative() { + for name in [ + "js_gc_collect", + "js_gc_loop_safepoint", + "js_alloc_object", + "user_function", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::Unknown, + "{name}" + ); + } + } + + #[test] + fn audited_alloc_helpers_are_contract_only_non_safepoints() { + for name in [ + "js_closure_alloc_singleton", + "js_array_push_f64", + "js_ctor_return_override", + "js_array_indexOf_jsvalue", + "js_validate_array_comparator", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::AllocNoReentry, + "{name}" + ); + } + // Transitive re-entry paths found by the body audit must stay out: + // js_value_length_f64 reaches js_object_get_field_by_name_f64 for + // plain objects; js_array_get_f64 has hole/accessor paths. + for name in ["js_value_length_f64", "js_array_get_f64"] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::Unknown, + "{name}" + ); + } + // Re-entering helpers must never be in the AllocNoReentry class: + // a poll can fire inside the callback/getter with this frame + // mid-stack, and the caller's roots must be findable. + for name in [ + "js_array_map", + "js_array_sort_with_comparator", + "js_number_coerce", + "js_dynamic_string_or_number_add", + "js_object_get_field_by_name_f64", + ] { + assert_eq!( + classify_direct_callee(name), + GcCallEffect::Unknown, + "{name}" + ); + } + } +} diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs new file mode 100644 index 0000000000..af9bd85891 --- /dev/null +++ b/crates/perry-codegen/src/gc_map.rs @@ -0,0 +1,738 @@ +//! Re-encode LLVM's stack-map section into Perry's compact GC map. +//! +//! # Why this exists +//! +//! `gc.statepoint` metadata is the statepoint backend's *only* losing axis +//! against the shadow stack. Measured on `test-drizzle-pg`: generated `__text` +//! is 248 KB **smaller** under statepoints, but `__llvm_stackmaps` adds 3.9 MB, +//! so the binary loses by 3.5 MB overall. +//! +//! Almost none of those bytes carry information an AOT collector can use. +//! Measured composition of that section: +//! +//! * **60% of all location slots are `Constant`** — exactly three per record, +//! `gc.statepoint`'s calling-convention / flags / num-deopt preamble. +//! * every root is recorded as a **(base, derived) pair**, and Perry has no +//! interior pointers, so half of the remainder is the same slot twice; +//! * each record carries a 16-byte header whose 8-byte **patchpoint ID** only +//! matters to a JIT that patches call sites, plus inter-record padding. +//! +//! The runtime already threw all of that away at startup (see +//! `perry-runtime/src/gc/roots/stack_maps.rs`): it kept `{dwarf_reg, offset}` +//! per distinct root and nothing else. This module simply stops shipping what +//! was always discarded. +//! +//! # Where the remaining win comes from +//! +//! Dropping the dead weight alone is ~11x. Two further facts about real +//! programs take it to ~32x: +//! +//! * roots within a record cluster in the frame, so **sorting by frame offset +//! and delta-encoding** them makes most roots a single byte; +//! * **77% of records have exactly the live set of the record before them** — +//! consecutive safepoints in a function usually share their roots — so a +//! repeat flag replaces the whole payload. +//! +//! On drizzle: 4,214,384 B -> 131,402 B, which turns the 3.5 MB file-size loss +//! into a ~271 KB win and lets the statepoint backend lead on size, speed and +//! RSS simultaneously. +//! +//! # Why the rewrite happens on assembly rather than the object +//! +//! `clang -S` prints the stack map as ordinary directives with the function +//! addresses as **symbol names in plain text** (`.quad _main`). Rewriting there +//! needs one text parser. Rewriting the object instead would need Mach-O *and* +//! ELF relocation parsing (the addresses are external relocations), plus +//! `llvm-objcopy` to drop the old section, plus a second link pass. +//! +//! It costs almost nothing: `-S` takes the same time as `-c` (the codegen is +//! the cost; printing text is free), and assembling the result is ~0.02s. + +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::process::Command; + +use anyhow::{anyhow, Context, Result}; + +/// Magic at the start of every emitted blob. +const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; +/// Format version. Bump on any layout change — the runtime rejects others. +const GC_MAP_VERSION: u8 = 3; +/// Section the compact map is emitted into, and the label it is given. +const GC_MAP_LABEL: &str = "_perry_gc_map"; +const MACHO_SECTION: &str = "__PERRY_GCMAP,__perry_gcmap"; +/// `R` is SHF_GNU_RETAIN, the ELF analogue of Mach-O's `.no_dead_strip`. +/// Perry links with `-Wl,--gc-sections`, and nothing in the program +/// references this section — the collector finds it by name at runtime — so +/// without RETAIN the linker discards it and the binary ships with no GC map +/// at all. Measured: the section is present in the object (PROGBITS, SHF_ALLOC, +/// with relocations) and absent from the linked binary. +const ELF_SECTION: &str = ".perry_gcmap,\"aR\",@progbits"; + +/// LLVM stack-map v3 location kinds. Only these two describe a frame slot; +/// `Constant`/`ConstIndex` carry the statepoint preamble and `Register` cannot +/// be recovered at collection time (which is what made plain stack maps +/// unsound — see the experiment write-up). +const LOCATION_DIRECT: u8 = 2; +const LOCATION_INDIRECT: u8 = 3; + +/// One safepoint: where it is in its function, and which frame slots are live. +/// +/// `instruction_offset` is the **assembly expression**, not a number: at `-O3` +/// LLVM emits it as a label difference (`Ltmp9-_main`) that only the assembler +/// can evaluate. That is why the emitted map stores offsets in a fixed-width +/// `u32` array rather than folding them into the varint stream. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Record { + instruction_offset: String, + /// `(dwarf_reg, frame_offset)`, deduplicated and sorted by frame offset. + roots: Vec<(u16, i32)>, +} + +/// One function's safepoints, keyed by the symbol the linker will relocate. +#[derive(Debug, Clone)] +struct FunctionMap { + symbol: String, + stack_size: u64, + records: Vec, +} + +/// Byte width contributed by each data directive LLVM emits in the block. +fn directive_width(directive: &str) -> Option { + match directive { + ".byte" => Some(1), + ".short" | ".value" | ".hword" => Some(2), + ".long" | ".word" => Some(4), + ".quad" | ".xword" => Some(8), + _ => None, + } +} + +/// The assembled bytes of the stack-map block, plus the byte offsets at which +/// a `.quad` referenced a symbol instead of a literal. +struct RawBlock { + start_line: usize, + end_line: usize, + bytes: Vec, + symbols: HashMap, +} + +fn parse_block(lines: &[&str]) -> Option { + let start_line = lines.iter().position(|line| { + let t = line.trim_start(); + t.starts_with(".section") + && (t.contains("__LLVM_STACKMAPS") || t.contains(".llvm_stackmaps")) + })?; + + let mut bytes: Vec = Vec::new(); + let mut symbols: HashMap = HashMap::new(); + let mut end_line = lines.len(); + + for (index, raw) in lines.iter().enumerate().skip(start_line + 1) { + let line = raw.trim(); + // The block runs to the next section or to the Mach-O epilogue. + if line.starts_with(".section") || line.starts_with(".subsections_via_symbols") { + end_line = index; + break; + } + if line.is_empty() || line.starts_with('#') || line.starts_with("//") || line.ends_with(':') + { + continue; + } + let mut parts = line.splitn(2, char::is_whitespace); + let directive = parts.next().unwrap_or_default(); + let operand = parts.next().unwrap_or_default(); + let operand = operand + .split('#') + .next() + .unwrap_or_default() + .split("//") + .next() + .unwrap_or_default() + .trim(); + + // Alignment is real content: LLVM aligns every record, and skipping + // the padding desynchronises every offset that follows it. + if directive == ".p2align" || directive == ".align" { + let first = operand.split(',').next().unwrap_or_default().trim(); + let value: u32 = first.parse().ok()?; + let align = if directive == ".p2align" { + 1usize << value + } else { + value as usize + }; + while align > 1 && bytes.len() % align != 0 { + bytes.push(0); + } + continue; + } + + let Some(width) = directive_width(directive) else { + continue; + }; + match parse_int(operand) { + Some(value) => bytes.extend_from_slice(&value.to_le_bytes()[..width]), + None => { + // A symbolic operand. Two kinds appear: the `.quad` function + // address, and — at `-O3` — the `.long` instruction offset as + // a label difference. Remember the expression and reserve the + // slot so every later structural offset stays correct. + symbols.insert(bytes.len(), operand.to_string()); + bytes.extend_from_slice(&0u64.to_le_bytes()[..width]); + } + } + } + + Some(RawBlock { + start_line, + end_line, + bytes, + symbols, + }) +} + +fn parse_int(text: &str) -> Option { + let text = text.trim(); + if let Some(hex) = text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + return u64::from_str_radix(hex, 16).ok(); + } + if let Some(negative) = text.strip_prefix('-') { + return negative + .parse::() + .ok() + .map(|v| (v as i64).wrapping_neg() as u64); + } + text.parse::().ok() +} + +fn read_u16(bytes: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(bytes.get(at..at + 2)?.try_into().ok()?)) +} + +fn read_u32(bytes: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) +} + +fn read_u64(bytes: &[u8], at: usize) -> Option { + Some(u64::from_le_bytes(bytes.get(at..at + 8)?.try_into().ok()?)) +} + +fn align_up(value: usize, alignment: usize) -> usize { + value.div_ceil(alignment) * alignment +} + +/// Decode every concatenated v3 map in the block. +/// +/// The section is a *sequence* of maps, one per object the linker saw — a +/// decoder that reads only the first header silently drops the rest, so this +/// walks until the bytes are consumed. +fn decode_v3(block: &RawBlock) -> Option> { + let bytes = &block.bytes; + let mut out: Vec = Vec::new(); + let mut pos = 0usize; + + while pos + 16 <= bytes.len() { + if bytes[pos] != 3 { + // Inter-map alignment padding. + pos += 1; + continue; + } + let function_count = read_u32(bytes, pos + 4)? as usize; + let constant_count = read_u32(bytes, pos + 8)? as usize; + let record_count = read_u32(bytes, pos + 12)? as usize; + pos += 16; + + let mut heads = Vec::with_capacity(function_count); + let mut expected = 0usize; + for _ in 0..function_count { + let symbol = block.symbols.get(&pos)?.clone(); + let stack_size = read_u64(bytes, pos + 8)?; + let records = read_u64(bytes, pos + 16)? as usize; + expected = expected.checked_add(records)?; + heads.push((symbol, stack_size, records)); + pos += 24; + } + if expected != record_count { + return None; + } + pos = pos.checked_add(constant_count.checked_mul(8)?)?; + + for (symbol, stack_size, count) in heads { + let mut records = Vec::with_capacity(count); + for _ in 0..count { + let record_start = pos; + let instruction_offset = block + .symbols + .get(&(pos + 8)) + .cloned() + .unwrap_or_else(|| read_u32(bytes, pos + 8).unwrap_or(0).to_string()); + let location_count = read_u16(bytes, pos + 14)? as usize; + pos += 16; + + let mut roots: Vec<(u16, i32)> = Vec::new(); + for _ in 0..location_count { + let kind = *bytes.get(pos)?; + let size = read_u16(bytes, pos + 2)?; + let dwarf_reg = read_u16(bytes, pos + 4)?; + let offset = read_u32(bytes, pos + 8)? as i32; + // Keep exactly what the collector keeps: 8-byte frame + // slots, with the base/derived pair collapsed to one. + if matches!(kind, LOCATION_DIRECT | LOCATION_INDIRECT) && size == 8 { + if !roots.contains(&(dwarf_reg, offset)) { + roots.push((dwarf_reg, offset)); + } + } + pos += 12; + } + + pos = align_up(pos - record_start, 8) + record_start; + let live_out_count = read_u16(bytes, pos + 2)? as usize; + pos = pos + .checked_add(4)? + .checked_add(live_out_count.checked_mul(4)?)?; + pos = align_up(pos - record_start, 8) + record_start; + if pos > bytes.len() { + return None; + } + + roots.sort_unstable_by_key(|(_, offset)| *offset); + records.push(Record { + instruction_offset, + roots, + }); + } + out.push(FunctionMap { + symbol, + stack_size, + records, + }); + } + } + + if out.is_empty() { + None + } else { + Some(out) + } +} + +fn push_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8 & 0x7F) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +fn zigzag(value: i32) -> u64 { + ((value << 1) ^ (value >> 31)) as u32 as u64 +} + +/// DWARF register number for the stack pointer on aarch64; every other base +/// this backend emits is the frame pointer. +const DWARF_REG_SP_AARCH64: u16 = 31; +/// Frame pointer, the other base the single-bit encoding can express. +const DWARF_REG_FP_AARCH64: u16 = 29; + +fn encode_stream(functions: &[FunctionMap]) -> Vec { + let mut stream = Vec::new(); + for function in functions { + let mut previous_roots: Option<&Vec<(u16, i32)>> = None; + for record in &function.records { + if previous_roots == Some(&record.roots) { + // Repeat flag: the live set is the previous record's. + push_varint(&mut stream, 1); + continue; + } + push_varint(&mut stream, (record.roots.len() as u64) << 1); + + // Deltas are zigzagged rather than emitted raw. `decode_v3` sorts + // roots so they are non-negative in practice, but a raw negative + // delta sign-extends into a 10-byte varint and silently bloats the + // map — the format must not depend on an ordering invariant held + // somewhere else. + // Base is a 2-bit tag, not a single FP/SP bit: LLVM also uses a + // callee-saved register (x19 on aarch64) as a frame base pointer + // in functions with dynamic stack allocation — measured 66 root + // slots in one real module. A bit cannot express that, and the + // format must not be the reason a root is unrepresentable. + // 0 = frame pointer, 1 = stack pointer, 2 = explicit DWARF + // register number as a following varint. + let mut previous: Option = None; + for (reg, offset) in &record.roots { + let tag = match *reg { + DWARF_REG_FP_AARCH64 => 0u64, + DWARF_REG_SP_AARCH64 => 1, + _ => 2, + }; + let delta = match previous { + None => *offset, + Some(prev) => offset.wrapping_sub(prev), + }; + push_varint(&mut stream, (zigzag(delta) << 2) | tag); + if tag == 2 { + push_varint(&mut stream, u64::from(*reg)); + } + previous = Some(*offset); + } + previous_roots = Some(&record.roots); + } + } + stream +} + +/// Assemble the emitted directives for one compact blob. +/// +/// Layout (little-endian), mirrored by the runtime decoder: +/// +/// ```text +/// 0 "PGCM" +/// 4 u8 version, u8 reserved, u16 reserved +/// 8 u32 function_count +/// 12 u32 total_len -- lets the runtime walk concatenated blobs +/// 16 function_count x { u64 address, u32 stack_size, u32 record_count } +/// record_count_total x u32 instruction_offset +/// varint root stream (see `encode_stream`) +/// ``` +/// +/// The function table starts at 16 so every relocated address is 8-byte +/// aligned, and the offset array that follows it is 4-byte aligned. +/// +/// Instruction offsets are a fixed-width array rather than part of the varint +/// stream because at `-O3` they are **label differences the assembler +/// evaluates** (`Ltmp9-_main`), so their values do not exist at rewrite time. +/// That costs ~4 bytes per record — 18.7x compaction instead of 31.8x — and +/// buys not having to assemble twice just to learn numbers the assembler is +/// about to compute anyway. +fn emit_asm(functions: &[FunctionMap], stream: &[u8], elf: bool) -> String { + let record_total: usize = functions.iter().map(|f| f.records.len()).sum(); + let total_len = 16 + functions.len() * 16 + record_total * 4 + stream.len(); + let mut out = String::new(); + if elf { + out.push_str(&format!("\t.section\t{ELF_SECTION}\n")); + } else { + out.push_str(&format!("\t.section\t{MACHO_SECTION}\n")); + } + out.push_str("\t.p2align\t3\n"); + out.push_str(&format!("{GC_MAP_LABEL}:\n")); + out.push_str(&format!( + "\t.ascii\t\"{}\"\n", + std::str::from_utf8(GC_MAP_MAGIC).expect("magic is ASCII") + )); + out.push_str(&format!("\t.byte\t{GC_MAP_VERSION}\n")); + out.push_str("\t.byte\t0\n"); + out.push_str("\t.short\t0\n"); + out.push_str(&format!("\t.long\t{}\n", functions.len())); + out.push_str(&format!("\t.long\t{total_len}\n")); + for function in functions { + out.push_str(&format!("\t.quad\t{}\n", function.symbol)); + out.push_str(&format!("\t.long\t{}\n", function.stack_size as u32)); + out.push_str(&format!("\t.long\t{}\n", function.records.len())); + } + for function in functions { + for record in &function.records { + out.push_str(&format!("\t.long\t{}\n", record.instruction_offset)); + } + } + for chunk in stream.chunks(32) { + let bytes: Vec = chunk.iter().map(|b| b.to_string()).collect(); + out.push_str(&format!("\t.byte\t{}\n", bytes.join(","))); + } + out +} + +/// Statistics for the caller to log — a compaction that silently did nothing +/// must be distinguishable from one that ran. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct GcMapStats { + original_bytes: usize, + compact_bytes: usize, + functions: usize, + records: usize, + roots: usize, +} + +/// Rewrite the LLVM stack-map block in `asm` into the compact map. +/// +/// Returns `None` when there is no stack-map block to rewrite (the common case +/// for a module without safepoints) or when the block does not parse. +/// +/// Those two are NOT the same to the caller: no block is fine, while a block +/// that fails to parse is a hard error in `compact_and_assemble`. Keeping +/// LLVM's section in that case would look conservative and would in fact lose +/// the module's roots, because the runtime reads only the compact section. +fn compact_stack_map_asm(asm: &str, elf: bool) -> Option<(String, GcMapStats)> { + let lines: Vec<&str> = asm.lines().collect(); + let block = parse_block(&lines)?; + let functions = decode_v3(&block)?; + let stream = encode_stream(&functions); + + let stats = GcMapStats { + original_bytes: block.bytes.len(), + compact_bytes: 16 + + functions.len() * 16 + + functions.iter().map(|f| f.records.len()).sum::() * 4 + + stream.len(), + functions: functions.len(), + records: functions.iter().map(|f| f.records.len()).sum(), + roots: functions + .iter() + .flat_map(|f| f.records.iter()) + .map(|r| r.roots.len()) + .sum(), + }; + + let replacement = emit_asm(&functions, &stream, elf); + let mut out = String::with_capacity(asm.len()); + for line in &lines[..block.start_line] { + // `.no_dead_strip` names the block's label from outside it. It is also + // the only thing keeping a section nothing references from being + // discarded, so retarget it instead of dropping it — without it the + // map is stripped and the collector finds no roots at all. + if line.contains(".no_dead_strip") && line.contains("__LLVM_StackMaps") { + out.push_str(&format!("\t.no_dead_strip\t{GC_MAP_LABEL}\n")); + } else { + out.push_str(line); + out.push('\n'); + } + } + out.push_str(&replacement); + for line in &lines[block.end_line..] { + if line.contains("__LLVM_StackMaps") { + continue; + } + out.push_str(line); + out.push('\n'); + } + Some((out, stats)) +} + +/// Rewrite the stack map in `asm_path` into Perry's compact form, then +/// assemble it to `obj_path`. +/// +/// A module with no stack-map block is assembled unchanged — there is nothing +/// to compact. +/// +/// A module that HAS a block which does not parse is a hard error, not a +/// fallback. Keeping LLVM's section there looks conservative and is not: the +/// runtime reads only `__perry_gcmap`, so that module's records would be +/// present in the binary, unread, and its roots invisible to the collector — +/// while other modules still emit a valid section, so even the "section +/// present but undecodable" guard in the runtime stays quiet. Silent lost +/// roots are precisely what this backend exists to make impossible. +pub fn compact_and_assemble( + clang: &Path, + target: &str, + asm_path: &Path, + obj_path: &Path, +) -> Result<()> { + let asm = fs::read_to_string(asm_path) + .with_context(|| format!("Failed to read assembly at {}", asm_path.display()))?; + + // Only the two object formats whose section syntax this module emits, and + // whose section the runtime knows how to find, can be rewritten. + // + // Assembling unchanged on anything else looks like a graceful degradation + // and is the opposite: the object would carry LLVM's `__llvm_stackmaps` + // and no `__perry_gcmap`, the runtime reads only the compact section, and + // the collector finds no native roots at all — the exact outcome the hard + // error below exists to prevent, reached with no diagnostic. The mode is + // opt-in, so refusing loudly costs nothing. + let macho = target.contains("apple") || target.contains("darwin"); + let elf = !macho && !target.contains("windows") && !target.contains("msvc"); + if !macho && !elf { + return Err(anyhow!( + "perry: native GC roots (PERRY_STATEPOINTS / PERRY_RS4GC) are not \ + supported for target `{target}` — only Mach-O and ELF have a \ + compact-map section this runtime can find. Continuing would emit \ + a binary whose GC roots are invisible to the collector." + )); + } + + let has_block = asm.lines().any(|l| { + let t = l.trim_start(); + t.starts_with(".section") + && (t.contains("__LLVM_STACKMAPS") || t.contains(".llvm_stackmaps")) + }); + let compacted = compact_stack_map_asm(&asm, elf); + if has_block && compacted.is_none() { + return Err(anyhow!( + "perry: this module emits an LLVM stack map that the compact-map \ + rewriter could not parse, so its GC roots would be invisible to \ + the collector (the runtime reads only the compact section). \ + Refusing to emit a binary that would lose roots silently. \ + Assembly left at: {}", + asm_path.display() + )); + } + if let Some((rewritten, stats)) = compacted { + fs::write(asm_path, rewritten).with_context(|| { + format!( + "Failed to write compacted assembly at {}", + asm_path.display() + ) + })?; + log::debug!( + "perry-codegen: gc map {} -> {} bytes ({} functions, {} records, {} roots)", + stats.original_bytes, + stats.compact_bytes, + stats.functions, + stats.records, + stats.roots, + ); + } + + assemble(clang, target, asm_path, obj_path) +} + +fn assemble(clang: &Path, target: &str, asm_path: &Path, obj_path: &Path) -> Result<()> { + let output = Command::new(clang) + .arg("-c") + .arg(asm_path) + .arg("-o") + .arg(obj_path) + .arg("-target") + .arg(target) + .output() + .with_context(|| format!("Failed to invoke {}", clang.display()))?; + if !output.status.success() { + return Err(anyhow!( + "assembling the compacted stack map failed (status={}).\n\ + assembly left at: {}\n\ + \n\ + stderr:\n{}", + output.status, + asm_path.display(), + String::from_utf8_lossy(&output.stderr) + )); + } + let _ = fs::remove_file(asm_path); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a minimal but structurally real v3 block: one function, one + /// record, two roots of which the second is the base/derived duplicate. + fn sample_asm() -> String { + let mut asm = String::new(); + asm.push_str("\t.no_dead_strip\t__LLVM_StackMaps\n"); + asm.push_str("\t.section\t__LLVM_STACKMAPS,__llvm_stackmaps\n"); + asm.push_str("__LLVM_StackMaps:\n"); + asm.push_str("\t.byte\t3\n\t.byte\t0\n\t.short\t0\n"); + asm.push_str("\t.long\t1\n"); // functions + asm.push_str("\t.long\t0\n"); // constants + asm.push_str("\t.long\t1\n"); // records + asm.push_str("\t.quad\t_probe_fn\n"); + asm.push_str("\t.quad\t144\n"); // stack size + asm.push_str("\t.quad\t1\n"); // record count + // record: id, instruction offset, reserved, location count + asm.push_str("\t.quad\t0\n"); + asm.push_str("\t.long\t64\n"); + asm.push_str("\t.short\t0\n"); + asm.push_str("\t.short\t4\n"); + // three statepoint preamble constants, then base/derived pair + for _ in 0..3 { + asm.push_str( + "\t.byte\t4\n\t.byte\t0\n\t.short\t8\n\t.short\t0\n\t.short\t0\n\t.long\t0\n", + ); + } + asm.push_str( + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n\t.short\t0\n\t.long\t4294967272\n", + ); + asm.push_str("\t.p2align\t3\n"); + asm.push_str("\t.short\t0\n\t.short\t0\n"); // live-out header + asm.push_str("\t.p2align\t3\n"); + asm.push_str("\t.subsections_via_symbols\n"); + asm + } + + #[test] + fn compacts_and_keeps_only_real_roots() { + let (out, stats) = compact_stack_map_asm(&sample_asm(), false).expect("block rewritten"); + assert_eq!(stats.functions, 1); + assert_eq!(stats.records, 1); + // Four locations in, one root out: three constants dropped. + assert_eq!(stats.roots, 1); + assert!( + stats.compact_bytes < stats.original_bytes, + "compact {} should beat original {}", + stats.compact_bytes, + stats.original_bytes + ); + assert!(out.contains("_perry_gc_map:")); + assert!(out.contains(".quad\t_probe_fn")); + // The old section must be gone, and nothing may still name its label. + assert!(!out.contains("__llvm_stackmaps")); + assert!(!out.contains("__LLVM_StackMaps")); + // The dead-strip guard must survive, retargeted. + assert!(out.contains(".no_dead_strip\t_perry_gc_map")); + } + + #[test] + fn repeated_live_sets_cost_one_byte() { + let shared = vec![(29u16, -24i32), (29, -32)]; + let functions = vec![FunctionMap { + symbol: "_f".to_string(), + stack_size: 64, + records: vec![ + Record { + instruction_offset: "0".to_string(), + roots: shared.clone(), + }, + Record { + instruction_offset: "8".to_string(), + roots: shared.clone(), + }, + Record { + instruction_offset: "16".to_string(), + roots: shared, + }, + ], + }]; + let one_record = vec![FunctionMap { + symbol: functions[0].symbol.clone(), + stack_size: functions[0].stack_size, + records: functions[0].records[..1].to_vec(), + }]; + // Offsets live in their own fixed-width array now, so in the varint + // stream the two extra records cost exactly one repeat byte each, + // regardless of how many roots the shared live set holds. + assert_eq!( + encode_stream(&functions).len(), + encode_stream(&one_record).len() + 2 + ); + } + + #[test] + fn encodes_a_foreign_register_base() { + // A base that is neither FP nor SP is real: LLVM uses x19 as a frame + // base pointer in functions with dynamic stack allocation. The 2-bit + // tag carries the DWARF number explicitly rather than refusing — the + // format must never be the reason a root is unrepresentable. + let asm = sample_asm().replace( + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t29\n", + "\t.byte\t3\n\t.byte\t0\n\t.short\t8\n\t.short\t19\n", + ); + let (out, stats) = + compact_stack_map_asm(&asm, true).expect("a foreign base must still encode"); + assert_eq!(stats.roots, 1); + assert!(out.contains("_perry_gc_map:")); + } + + #[test] + fn no_stack_map_block_is_left_alone() { + assert!(compact_stack_map_asm("\t.section\t__TEXT,__text\n\tret\n", false).is_none()); + } + + #[test] + fn unparsable_block_keeps_llvm_section() { + // Truncated header: better to ship LLVM's section than a map that may + // be missing roots. + let asm = "\t.section\t__LLVM_STACKMAPS,__llvm_stackmaps\n\t.byte\t3\n"; + assert!(compact_stack_map_asm(asm, false).is_none()); + } +} diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index fbc57cd2fb..3282a1456d 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -14,6 +14,8 @@ pub(crate) mod eh_mode; pub mod expr; pub mod ext_registry; pub mod function; +pub(crate) mod gc_call_effects; +pub mod gc_map; #[cfg(feature = "llvm-inprocess")] pub mod inprocess; pub mod inst; @@ -31,6 +33,7 @@ pub(crate) mod native_value; pub(crate) mod nm_install; pub mod opt_report; pub mod runtime_decls; +pub mod statepoint_report; pub(crate) mod stmt; pub mod strings; pub mod stubs; diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index a0429d2821..ab42708e4f 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -223,6 +223,10 @@ struct ClangCompilePlan { ll_path: PathBuf, obj_path: PathBuf, stderr_remarks_path: PathBuf, + /// Set when the stack map is being compacted: clang emits assembly here + /// instead of an object, the stack-map block is rewritten (see + /// `crate::gc_map`), and the result is assembled to `obj_path`. + asm_path: Option, } fn native_tuning_arg_for_host() -> &'static str { @@ -395,7 +399,20 @@ fn build_clang_compile_plan( "-O3" }; - let mut clang_args = vec!["-c".to_string(), opt_flag.to_string()]; + // Compacting the stack map means going through assembly, because that is + // where LLVM prints the map's function addresses as symbol *names* — the + // one form that needs neither relocation parsing nor a second link. Only + // the statepoint backends emit a stack map, so only they pay for it, and + // the cost is small: `-S` takes the same time as `-c` (codegen is the + // cost, printing text is free) and assembling is ~0.02s per module. + let compact_gc_map = + crate::codegen::helpers::statepoints_enabled() || crate::codegen::helpers::rs4gc_enabled(); + let asm_path = compact_gc_map.then(|| PathBuf::from(format!("{}.s", obj_path.display()))); + + let mut clang_args = vec![ + if compact_gc_map { "-S" } else { "-c" }.to_string(), + opt_flag.to_string(), + ]; // A parameter rather than an env probe so a test can pin what `-g` does // and does not reach — measured in #7144: on a Perry `.ll` it produces a // byte-identical object with no `.debug_*` sections, because Perry's @@ -423,7 +440,7 @@ fn build_clang_compile_plan( } clang_args.push(ll_path.display().to_string()); clang_args.push("-o".to_string()); - clang_args.push(obj_path.display().to_string()); + clang_args.push(asm_path.as_ref().unwrap_or(&obj_path).display().to_string()); clang_args.push("-target".to_string()); clang_args.push(effective_target.clone()); @@ -443,6 +460,7 @@ fn build_clang_compile_plan( ll_path, obj_path, stderr_remarks_path, + asm_path, } } @@ -453,7 +471,74 @@ fn build_clang_compile_plan( /// more reliably from disk than from stdin), invoke `clang -c`, read the /// resulting `.o`, and clean up both on success. On failure the temp files /// are left behind for debugging — the caller can `grep /tmp/perry_llvm_*`. +/// #7174 research pipe: run `opt -passes='function(mem2reg), +/// rewrite-statepoints-for-gc'` over the module before clang when +/// `PERRY_RS4GC=1`. mem2reg promotes the retyped `ptr addrspace(1)` root +/// allocas into SSA (their only uses are the surgery's loads/stores, so +/// promotion always succeeds), and RS4GC then owns every statepoint, +/// relocation, and downstream-use rewrite. Fails the compile loudly when no +/// `opt` is available or the pass pipeline errors — a silent skip would be a +/// vacuous mode. +fn maybe_rs4gc_preprocess(ll_text: &str) -> Result> { + if !crate::codegen::helpers::rs4gc_enabled() { + return Ok(None); + } + let opt = std::env::var("PERRY_LLVM_OPT") + .map(PathBuf::from) + .ok() + .filter(|p| p.exists()) + .or_else(|| { + [ + "/opt/homebrew/opt/llvm/bin/opt", + "/usr/local/opt/llvm/bin/opt", + ] + .iter() + .map(PathBuf::from) + .find(|p| p.exists()) + }) + .or_else(|| which_in_path("opt")) + .context( + "PERRY_RS4GC=1 requires an LLVM `opt` binary: set PERRY_LLVM_OPT, \ + install Homebrew LLVM, or put `opt` on PATH", + )?; + let mut child = Command::new(&opt) + .args([ + "-passes=function(mem2reg),rewrite-statepoints-for-gc", + "-S", + "-", + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn {}", opt.display()))?; + use std::io::Write as _; + child + .stdin + .take() + .expect("piped stdin") + .write_all(ll_text.as_bytes())?; + let output = child.wait_with_output()?; + if !output.status.success() { + return Err(anyhow!( + "PERRY_RS4GC: opt pipeline failed:\n{}", + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(Some(String::from_utf8(output.stdout)?)) +} + +fn which_in_path(name: &str) -> Option { + std::env::var_os("PATH").and_then(|paths| { + std::env::split_paths(&paths) + .map(|dir| dir.join(name)) + .find(|p| p.exists()) + }) +} + pub fn compile_ll_to_object(ll_text: &str, target_triple: Option<&str>) -> Result> { + let rs4gc_ll = maybe_rs4gc_preprocess(ll_text)?; + let ll_text: &str = rs4gc_ll.as_deref().unwrap_or(ll_text); compile_ll_to_object_in( &env::temp_dir(), ll_text, @@ -726,6 +811,15 @@ fn compile_ll_to_object_in( )); } + if let Some(asm_path) = &plan.asm_path { + crate::gc_map::compact_and_assemble( + &plan.clang, + &plan.effective_target, + asm_path, + &obj_path, + )?; + } + let bytes = fs::read(&obj_path) .with_context(|| format!("Failed to read clang output at {}", obj_path.display()))?; @@ -780,11 +874,63 @@ pub fn compile_units_to_object(units: &[String], target_triple: Option<&str>) -> _ => {} } + // Units are independent clang invocations, so compile them concurrently. + // Measured before this: the 13 MB Claude Code bundle spent 4,939 s wall + // against 4,672 s user — the split existed for memory (#5391) but the + // clang phase, which dominates, ran one unit at a time. + // + // Concurrency is BOUNDED rather than one-thread-per-unit: each job parses + // a multi-hundred-megabyte translation unit, so unbounded fan-out trades + // wall time for an OOM (and would undo the peak-memory win the split was + // introduced for). Default is a quarter of the machine's parallelism, + // clamped to [1, 4]; `PERRY_CODEGEN_UNIT_JOBS` overrides. + let jobs = std::env::var("PERRY_CODEGEN_UNIT_JOBS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|p| (p.get() / 4).clamp(1, 4)) + .unwrap_or(1) + }) + .min(units.len()); + + let mut compiled: Vec>>> = (0..units.len()).map(|_| None).collect(); + if jobs <= 1 { + for (i, unit) in units.iter().enumerate() { + compiled[i] = Some(compile_ll_to_object(unit, target_triple)); + } + } else { + let slots: Vec>>>> = (0..units.len()) + .map(|_| std::sync::Mutex::new(None)) + .collect(); + let next = std::sync::atomic::AtomicUsize::new(0); + std::thread::scope(|scope| { + for _ in 0..jobs { + scope.spawn(|| loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= units.len() { + break; + } + let out = compile_ll_to_object(&units[i], target_triple); + *slots[i].lock().expect("codegen-unit slot poisoned") = Some(out); + }); + } + }); + for (i, slot) in slots.into_iter().enumerate() { + compiled[i] = slot.into_inner().expect("codegen-unit slot poisoned"); + } + } + let mut objs: Vec> = Vec::with_capacity(units.len()); - for (i, unit) in units.iter().enumerate() { - objs.push(compile_ll_to_object(unit, target_triple).with_context(|| { - format!("codegen unit {}/{} failed to compile", i + 1, units.len()) - })?); + for (i, result) in compiled.into_iter().enumerate() { + objs.push( + result + .expect("every codegen unit is compiled") + .with_context(|| { + format!("codegen unit {}/{} failed to compile", i + 1, units.len()) + })?, + ); } merge_unit_objects(&objs) } diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 9cdc603ef0..56f159340f 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -44,6 +44,39 @@ fn strip_leading_linkage(s: &str) -> &str { /// single copy when the same global is emitted into multiple units. `external` /// declarations (no initializer) are returned unchanged — duplicating a /// declaration is harmless. + +/// Symbol name of a global/string definition line (`@name = ...`). +fn global_symbol_name(line: &str) -> Option<&str> { + let line = line.trim_start(); + if !line.starts_with('@') { + return None; + } + let end = line.find(" = ")?; + Some(&line[..end]) +} + +/// Collect every `@symbol` referenced in a chunk of IR text. +fn collect_symbol_refs(text: &str, out: &mut HashSet) { + let b = text.as_bytes(); + let mut i = 0usize; + while i < b.len() { + if b[i] == b'@' { + let start = i; + i += 1; + while i < b.len() + && (b[i].is_ascii_alphanumeric() || matches!(b[i], b'_' | b'.' | b'$' | b'-')) + { + i += 1; + } + if i > start + 1 { + out.insert(text[start..i].to_string()); + } + } else { + i += 1; + } + } +} + fn promote_global_for_units(line: &str) -> String { if line.contains(" = external ") { return line.to_string(); @@ -180,6 +213,30 @@ fn render_fn_external(f: &LlFunction) -> String { ir } +fn push_statepoint_declarations(ir: &mut String) { + ir.push_str( + "declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, \ + i32 immarg, i32 immarg, ...)\n\ + declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token, i32 immarg, \ + i32 immarg)\n", + ); + for (suffix, ty) in [ + ("i1", "i1"), + ("i8", "i8"), + ("i16", "i16"), + ("i32", "i32"), + ("i64", "i64"), + ("i128", "i128"), + ("f32", "float"), + ("f64", "double"), + ("p0", "ptr"), + ] { + ir.push_str(&format!( + "declare {ty} @llvm.experimental.gc.result.{suffix}(token)\n" + )); + } +} + pub struct LlModule { pub target_triple: String, declarations: Vec<(String, String)>, // (name, full "declare …" line) @@ -542,6 +599,16 @@ impl LlModule { let mut ir = String::new(); ir.push_str("; Generated by perry-codegen\n"); ir.push_str(&format!("target triple = \"{}\"\n\n", self.target_triple)); + if crate::codegen::helpers::native_stack_roots_enabled() + && self.target_triple.contains("apple") + { + // LLVM emits one local `__LLVM_StackMaps` atom per object. Perry's + // normal `-dead_strip` link otherwise discards those unreferenced + // atoms. This Mach-O directive marks each local atom live without + // globalizing the repeated symbol (which would collide across + // codegen units). + ir.push_str("module asm \".no_dead_strip __LLVM_StackMaps\"\n\n"); + } for sc in &self.string_constants { ir.push_str(sc); @@ -567,6 +634,12 @@ impl LlModule { ir.push_str(decl); ir.push('\n'); } + if crate::codegen::helpers::native_stack_roots_enabled() { + ir.push_str("declare void @llvm.experimental.stackmap(i64, i32, ...)\n"); + } + if crate::codegen::helpers::statepoints_enabled() { + push_statepoint_declarations(&mut ir); + } ir.push('\n'); for func in &funcs { @@ -700,35 +773,145 @@ impl LlModule { .or_insert_with(|| declare_line_for(f)); } + // #7174 (real-app scaling): render each bucket's functions first, then + // give every global/string exactly ONE defining unit and hand the rest + // an `external` declaration. Replicating all definitions into every + // unit made per-unit IR grow with unit COUNT — on the 13 MB Claude Code + // bundle that meant ~400 MB units and `clang: translation unit is too + // large ... ran out of source locations`, no matter how finely it was + // split. Definitions are already `linkonce_odr` (visible), so an + // external declaration resolves to the same symbol at link time. + let bucket_texts: Vec = buckets + .iter() + .map(|bucket| { + let mut t = String::new(); + for func in bucket { + t.push_str(&render_fn_external(func)); + t.push('\n'); + } + t + }) + .collect(); + let bucket_refs: Vec> = bucket_texts + .iter() + .map(|t| { + let mut refs = HashSet::new(); + collect_symbol_refs(t, &mut refs); + refs + }) + .collect(); + + // A global is emitted into every unit that REFERENCES it — normally + // exactly one, and `linkonce_odr` lets the linker fold the rare + // multi-unit case. Definition-in-one-unit + `external` elsewhere was + // tried first and is subtly wrong under `-dead_strip`: the sole + // definition can be discarded with its unit's atoms while a live + // reference survives in another object. + let all_globals: Vec<&String> = + shared_strings.iter().chain(shared_globals.iter()).collect(); + // Globals reference OTHER globals in their initializers (a string + // header pointing at its `.bytes` payload, a closure record naming its + // thunk). Function-text references alone therefore under-approximate + // what a unit needs — the first cut emitted `@....str.N.bytes` nowhere + // and clang rejected the unit with "use of undefined value". Close the + // reference set transitively per unit before deciding what to emit. + let global_index: std::collections::HashMap<&str, usize> = all_globals + .iter() + .enumerate() + .filter_map(|(i, def)| global_symbol_name(def).map(|nm| (nm, i))) + .collect(); + let global_refs: Vec> = all_globals + .iter() + .map(|def| { + let mut refs = HashSet::new(); + collect_symbol_refs(def, &mut refs); + refs + }) + .collect(); + let bucket_needs: Vec> = bucket_refs + .iter() + .map(|refs| { + let mut need: HashSet = refs + .iter() + .filter_map(|nm| global_index.get(nm.as_str()).copied()) + .collect(); + let mut work: Vec = need.iter().copied().collect(); + while let Some(gi) = work.pop() { + for nm in &global_refs[gi] { + if let Some(&next) = global_index.get(nm.as_str()) { + if need.insert(next) { + work.push(next); + } + } + } + } + need + }) + .collect(); + let referenced_anywhere: Vec = (0..all_globals.len()) + .map(|gi| bucket_needs.iter().any(|need| need.contains(&gi))) + .collect(); + let mut post = String::new(); self.push_attrs_and_metadata(&mut post); let mut parts = Vec::with_capacity(n); - for bucket in buckets { + for (bi, bucket) in buckets.into_iter().enumerate() { let defined: HashSet<&str> = bucket.iter().map(|f| f.name.as_str()).collect(); let mut pre = String::new(); pre.push_str("; Generated by perry-codegen (codegen unit)\n"); pre.push_str(&format!("target triple = \"{}\"\n\n", self.target_triple)); - - for sc in &shared_strings { - pre.push_str(sc); - pre.push('\n'); + if crate::codegen::helpers::native_stack_roots_enabled() + && self.target_triple.contains("apple") + { + pre.push_str("module asm \".no_dead_strip __LLVM_StackMaps\"\n\n"); } - pre.push('\n'); - for g in &shared_globals { - pre.push_str(g); - pre.push('\n'); + + for (gi, def) in all_globals.iter().enumerate() { + let referenced = bucket_needs[bi].contains(&gi); + // Unreferenced globals (anchors, `llvm.*`, appending lists) + // keep a home in unit 0 so nothing is lost. + if referenced || (!referenced_anywhere[gi] && bi == 0) { + pre.push_str(def); + pre.push('\n'); + } } pre.push('\n'); - // Declares for everything this unit references but does not define. + // Declares for everything this unit REFERENCES but does not + // define. Emitting the whole module's declaration list into every + // unit left a per-unit floor that splitting cannot reduce: a + // 24-function benchmark carried 2,972 declares (149 KB) per unit, + // and the 13 MB Claude Code bundle carried ~16,700 — which is how + // units stayed above a gigabyte and hit clang's 2^31 source-location + // ceiling ("translation unit is too large ... ran out of source + // locations") regardless of unit count. Referenced names include + // those reached through the initializers of the globals this unit + // emits, so the closure computed above feeds this filter too. + // `collect_symbol_refs` yields `@name`; `decl_by_name` is keyed on + // the bare name, so strip the sigil or nothing ever matches. + let mut needed: HashSet<&str> = bucket_refs[bi] + .iter() + .map(|nm| nm.trim_start_matches('@')) + .collect(); + for gi in &bucket_needs[bi] { + for nm in &global_refs[*gi] { + needed.insert(nm.trim_start_matches('@')); + } + } for (name, decl) in &decl_by_name { - if defined.contains(name) { + if defined.contains(name) || !needed.contains(*name) { continue; } pre.push_str(decl); pre.push('\n'); } + if crate::codegen::helpers::native_stack_roots_enabled() { + pre.push_str("declare void @llvm.experimental.stackmap(i64, i32, ...)\n"); + } + if crate::codegen::helpers::statepoints_enabled() { + push_statepoint_declarations(&mut pre); + } pre.push('\n'); parts.push(CodegenUnitPart { @@ -820,11 +1003,41 @@ mod tests { .unwrap(); assert!(u_with_f.contains("declare double @perry_fn_m__g()")); - // Shared globals appear in BOTH units, promoted to linkonce_odr. + // #7174: each shared global is DEFINED exactly once across units; + // units that reference it get an `external` declaration instead of a + // copy. Replicating definitions made per-unit IR grow with the unit + // count and broke clang's translation-unit limit on real bundles. + let global_defs = units + .iter() + .filter(|u| u.contains("@perry_global_x = linkonce_odr global double 0.0")) + .count(); + assert_eq!(global_defs, 1, "global must be defined in exactly one unit"); + let str_defs = units + .iter() + .filter(|u| u.contains("@.str.0 = linkonce_odr unnamed_addr constant")) + .count(); + assert_eq!(str_defs, 1, "string must be defined in exactly one unit"); + + // Every unit that mentions the symbol either defines it or declares it + // external — never neither. for u in &units { - assert!(u.contains("@perry_global_x = linkonce_odr global double 0.0")); - assert!(u.contains("@.str.0 = linkonce_odr unnamed_addr constant")); - assert!(u.contains("declare void @js_console_log_number(double)")); + if u.contains("@perry_global_x") { + assert!( + u.contains("@perry_global_x = linkonce_odr global double 0.0") + || u.contains("@perry_global_x = external global double"), + "referencing unit must define or externally declare the global" + ); + } + // Declares are now scoped to what a unit references (the + // whole-module declaration list was a per-unit floor that + // splitting could not reduce). A unit that calls the helper must + // still declare it. + if u.contains("call void @js_console_log_number") { + assert!( + u.contains("declare void @js_console_log_number(double)"), + "a unit calling the helper must declare it" + ); + } assert!(u.contains("target triple = \"arm64-apple-macosx15.0.0\"")); } } @@ -976,7 +1189,12 @@ mod tests { m.declare_function("js_is_truthy", I32, &[DOUBLE]); for k in 0..2 { let f = m.define_function(format!("perry_fn_m__f{k}"), DOUBLE, vec![]); - f.create_block("entry").ret(DOUBLE, "0.0"); + let b = f.create_block("entry"); + // Reference the helper so the declare is genuinely needed: declares + // are scoped per unit now, and a test whose units never call the + // helper would assert nothing about its attribute group. + b.call(I32, "js_is_truthy", &[(DOUBLE, "0.0")]); + b.ret(DOUBLE, "0.0"); } let units = m.render_codegen_units(2); assert_eq!(units.len(), 2); diff --git a/crates/perry-codegen/src/statepoint_report.rs b/crates/perry-codegen/src/statepoint_report.rs new file mode 100644 index 0000000000..b1b59a4cfa --- /dev/null +++ b/crates/perry-codegen/src/statepoint_report.rs @@ -0,0 +1,303 @@ +//! Observational root-pressure report for the native-stack GC experiments. +//! +//! `PERRY_STATEPOINT_REPORT=1|text|json` records how many textual calls see +//! live roots, which ones can be omitted after the GC-effect audit, and how +//! much statepoint/stack-map metadata remains. Codegen never reads the data +//! back, so enabling the report cannot affect emitted IR. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::sync::{Mutex, OnceLock}; + +#[derive(Clone, Debug, Default, Eq, PartialEq, serde::Serialize)] +pub struct FunctionRecord { + function: String, + backend: String, + reserved_logical_slots: u32, + bound_native_slots: usize, + textual_calls: u64, + calls_without_live_roots: u64, + calls_with_live_roots: u64, + skipped_non_safepoints: u64, + statepoints: u64, + relocations: u64, + plain_stack_maps: u64, + stack_map_operands: u64, + statepoint_fallbacks: u64, + max_live_roots: usize, + live_roots_histogram: BTreeMap, + statepoints_by_callee: BTreeMap, + skipped_by_callee: BTreeMap, + fallbacks_by_callee: BTreeMap, +} + +impl FunctionRecord { + pub(crate) fn new( + function: &str, + backend: &str, + reserved_logical_slots: u32, + bound_native_slots: usize, + ) -> Self { + Self { + function: function.to_string(), + backend: backend.to_string(), + reserved_logical_slots, + bound_native_slots, + ..Self::default() + } + } + + pub(crate) fn note_call(&mut self, live_roots: usize) { + self.textual_calls += 1; + if live_roots == 0 { + self.calls_without_live_roots += 1; + } else { + self.calls_with_live_roots += 1; + } + } + + pub(crate) fn note_skipped(&mut self, callee: &str) { + self.skipped_non_safepoints += 1; + *self + .skipped_by_callee + .entry(callee.to_string()) + .or_default() += 1; + } + + fn note_emitted_roots(&mut self, live_roots: usize) { + self.max_live_roots = self.max_live_roots.max(live_roots); + *self.live_roots_histogram.entry(live_roots).or_default() += 1; + } + + pub(crate) fn note_statepoint(&mut self, callee: &str, live_roots: usize) { + self.statepoints += 1; + self.relocations += live_roots as u64; + self.note_emitted_roots(live_roots); + *self + .statepoints_by_callee + .entry(callee.to_string()) + .or_default() += 1; + } +} + +pub fn enabled() -> bool { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_STATEPOINT_REPORT").as_deref(), + Ok("1") | Ok("text") | Ok("json") + ) + }) +} + +static SINK: OnceLock>> = OnceLock::new(); + +fn sink() -> &'static Mutex> { + SINK.get_or_init(|| Mutex::new(Vec::new())) +} + +pub(crate) fn record(record: FunctionRecord) { + if enabled() { + if let Ok(mut records) = sink().lock() { + records.push(record); + } + } +} + +pub fn take_records() -> Vec { + let mut records = match sink().lock() { + Ok(mut records) => std::mem::take(&mut *records), + Err(_) => Vec::new(), + }; + records.sort_by(|a, b| { + (&a.function, &a.backend, a.reserved_logical_slots).cmp(&( + &b.function, + &b.backend, + b.reserved_logical_slots, + )) + }); + records.dedup(); + records +} + +#[derive(Default, serde::Serialize)] +struct Totals { + functions: usize, + reserved_logical_slots: u64, + bound_native_slots: u64, + textual_calls: u64, + calls_without_live_roots: u64, + calls_with_live_roots: u64, + skipped_non_safepoints: u64, + statepoints: u64, + relocations: u64, + plain_stack_maps: u64, + stack_map_operands: u64, + statepoint_fallbacks: u64, + max_live_roots: usize, + live_roots_histogram: BTreeMap, + statepoints_by_callee: BTreeMap, + skipped_by_callee: BTreeMap, + fallbacks_by_callee: BTreeMap, +} + +fn totals(records: &[FunctionRecord]) -> Totals { + let mut out = Totals { + functions: records.len(), + ..Totals::default() + }; + for record in records { + out.reserved_logical_slots += u64::from(record.reserved_logical_slots); + out.bound_native_slots += record.bound_native_slots as u64; + out.textual_calls += record.textual_calls; + out.calls_without_live_roots += record.calls_without_live_roots; + out.calls_with_live_roots += record.calls_with_live_roots; + out.skipped_non_safepoints += record.skipped_non_safepoints; + out.statepoints += record.statepoints; + out.relocations += record.relocations; + out.plain_stack_maps += record.plain_stack_maps; + out.stack_map_operands += record.stack_map_operands; + out.statepoint_fallbacks += record.statepoint_fallbacks; + out.max_live_roots = out.max_live_roots.max(record.max_live_roots); + for (width, count) in &record.live_roots_histogram { + *out.live_roots_histogram.entry(*width).or_default() += count; + } + for (callee, count) in &record.statepoints_by_callee { + *out.statepoints_by_callee.entry(callee.clone()).or_default() += count; + } + for (callee, count) in &record.skipped_by_callee { + *out.skipped_by_callee.entry(callee.clone()).or_default() += count; + } + for (callee, count) in &record.fallbacks_by_callee { + *out.fallbacks_by_callee.entry(callee.clone()).or_default() += count; + } + } + out +} + +fn render_ranked_map(out: &mut String, heading: &str, values: &BTreeMap) { + if values.is_empty() { + return; + } + let mut rows: Vec<_> = values.iter().collect(); + rows.sort_by(|(name_a, count_a), (name_b, count_b)| { + count_b.cmp(count_a).then_with(|| name_a.cmp(name_b)) + }); + let _ = writeln!(out, "{heading}"); + for (name, count) in rows.into_iter().take(25) { + let _ = writeln!(out, " {count:>6} {name}"); + } + out.push('\n'); +} + +pub fn render_text(records: &[FunctionRecord]) -> String { + let totals = totals(records); + let mut out = String::from( + "Perry native-stack GC report (--statepoint-report)\n\ + ==================================================\n\n", + ); + if records.is_empty() { + out.push_str( + "No native-stack lowering records were emitted. Enable PERRY_STATEPOINTS=1\n\ + or PERRY_RS4GC=1 and ensure codegen is not served from cache.\n", + ); + return out; + } + + let emitted = totals.statepoints + totals.plain_stack_maps; + let _ = writeln!( + out, + "{} function(s), {} bound native root slots ({} logical slots reserved)", + totals.functions, totals.bound_native_slots, totals.reserved_logical_slots + ); + let _ = writeln!( + out, + "{} textual calls: {} with live roots, {} without", + totals.textual_calls, totals.calls_with_live_roots, totals.calls_without_live_roots + ); + let _ = writeln!( + out, + "{} safepoints emitted: {} statepoints, {} plain stack maps", + emitted, totals.statepoints, totals.plain_stack_maps + ); + let _ = writeln!( + out, + "{} non-collecting calls skipped; {} statepoint parser fallback(s)", + totals.skipped_non_safepoints, totals.statepoint_fallbacks + ); + let _ = writeln!( + out, + "{} relocations, {} plain-map operands; maximum {} live roots at one safepoint\n", + totals.relocations, totals.stack_map_operands, totals.max_live_roots + ); + + if !totals.live_roots_histogram.is_empty() { + out.push_str("Live roots per emitted safepoint\n"); + for (width, count) in &totals.live_roots_histogram { + let _ = writeln!(out, " {width:>4} root(s): {count:>6} safepoint(s)"); + } + out.push('\n'); + } + render_ranked_map( + &mut out, + "Most frequent explicit statepoint callees", + &totals.statepoints_by_callee, + ); + render_ranked_map( + &mut out, + "Calls omitted by the GC-effect audit", + &totals.skipped_by_callee, + ); + render_ranked_map( + &mut out, + "Plain-map fallbacks in statepoint mode", + &totals.fallbacks_by_callee, + ); + out +} + +#[derive(serde::Serialize)] +struct JsonReport<'a> { + schema_version: u32, + totals: Totals, + functions: &'a [FunctionRecord], +} + +pub fn render_json(records: &[FunctionRecord]) -> String { + serde_json::to_string_pretty(&JsonReport { + schema_version: 1, + totals: totals(records), + functions: records, + }) + .unwrap_or_else(|error| format!("{{\"error\":\"{error}\"}}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_and_json_expose_root_pressure_and_fallbacks() { + let mut record = FunctionRecord::new("probe", "statepoint", 3, 2); + record.note_call(2); + record.note_statepoint("@may_collect", 2); + record.note_call(1); + record.note_skipped("@js_gc_temp_root_get"); + record.note_call(1); + + let text = render_text(std::slice::from_ref(&record)); + assert!(text.contains("2 bound native root slots")); + assert!(text.contains("1 non-collecting calls skipped")); + // The plain-map fallback is gone, so this can only ever report zero — + // which is the point: it is the report's evidence that no root was + // recorded in an unrecoverable location. + assert!(text.contains("0 statepoint parser fallback(s)")); + assert!(text.contains("@js_gc_temp_root_get")); + + let json = render_json(&[record]); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["schema_version"], 1); + assert_eq!(parsed["totals"]["relocations"], 2); + assert_eq!(parsed["totals"]["statepoint_fallbacks"], 0); + } +} diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 01a4091524..96a6d1839c 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1023,7 +1023,7 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( collector.stats.malloc_sweep_due = malloc_sweep_due; collector.stats.reset_blocks += crate::arena::copying_prepare_to_space(); - visit_mutable_root_slots(|slot| unsafe { + let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { let bits = slot.read(); if let Some(trace) = trace.as_mut() { let pointer_root = collector.ptrs.decode_bits(bits).is_some(); @@ -1046,6 +1046,8 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( } } }); + let mut root_sources = trace.as_mut().map(|trace| &mut trace.root_sources); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); let scanners: Vec = MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()); { @@ -1243,12 +1245,14 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( maybe_schedule_old_reclaim_after_copied_minor(); if std::env::var_os("PERRY_GC_DIAG").is_some() { eprintln!( - "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={}", + "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} trigger={:?} declared_safepoint={}", collector.stats.copied_objects, collector.stats.copied_bytes, collector.stats.promoted_objects, collector.stats.promoted_bytes, - freed_bytes + freed_bytes, + _trigger_kind, + super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) ); } Some(CopiedMinorFastPathOutcome { diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 5c79ac8864..8ce39ed4f0 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -113,6 +113,10 @@ pub fn gc_collect_minor() -> u64 { } pub(super) fn gc_collect_minor_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome { + // PERRY_GC_SAFEPOINT_ONLY: held for the whole collection so every + // consumer of the scan decision (root scan, copying eligibility, + // evacuation pinning, verifier) sees the same healed answer. + let _contract_heal = policy::contract_scan_heal_guard(); gc_drain_active_budgeted_cycle(); // Barriers-off ⇒ the remembered set is not being maintained, and a // minor's black-leafed old parents would hide live children. Route @@ -325,6 +329,10 @@ fn gc_collect_inner_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome } fn gc_collect_full_mark_sweep_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome { + // PERRY_GC_SAFEPOINT_ONLY: see gc_collect_minor_with_trigger. Manual + // gc() engages its own force_full_scan first, which this detects as + // already-Scan and no-ops. + let _contract_heal = policy::contract_scan_heal_guard(); gc_drain_active_budgeted_cycle(); GC_TRIGGER_BUMPED.with(|c| c.set(false)); GcCycleState::new_full(trigger).run_to_completion() @@ -709,6 +717,10 @@ pub fn gc_init() { #[no_mangle] pub extern "C" fn js_gc_init() { + // Parse LLVM stack-map metadata before the first collection. The parser + // allocates its immutable index once; root scans themselves must remain + // allocation-free while the collector owns the heap. + initialize_stack_maps(); // Windows: opt console stdout/stderr into VT/ANSI escape processing // once at program start so runtime-emitted escapes (console.clear, tty // cursor ops, color output keyed off isTTY) render instead of printing diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 295008b741..4c79ac080a 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -536,7 +536,7 @@ impl GcCollectionKind { } } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub(super) enum GcTriggerKind { ArenaBytes, MallocCount, @@ -681,6 +681,101 @@ thread_local! { /// last set — the baseline the deferral slack is measured from (#7024). /// Meaningless while `GC_SAFEPOINT_PENDING` is false. pub(super) static GC_SAFEPOINT_DEFER_ARENA_BASE: Cell = const { Cell::new(0) }; + /// True while a DECLARED safepoint drain is running: a loop back-edge + /// poll, the outermost microtask-pump moving minor, or an explicit + /// `gc()`. Consumed by the `PERRY_GC_SAFEPOINT_ONLY` contract assert in + /// the root-scan subphase. + pub(super) static GC_AT_DECLARED_SAFEPOINT: Cell = const { Cell::new(false) }; +} + +/// `PERRY_GC_SAFEPOINT_ONLY` — research contract for the native-root modes +/// (`exp/stackmap-viability`): a collection that skips the conservative stack +/// scan consumes only precise roots, and with native stack maps active those +/// roots exist only at mapped PCs — so such a collection may begin only at a +/// declared safepoint; anywhere else it must scan conservatively. Codegen +/// reads the same env to stop emitting statepoints around audited +/// allocate-but-never-reenter helpers; the enforcement in `cycle.rs` is what +/// turns the property from emergent (every possibly-collecting call happens +/// to be mapped) into enforced. +/// +/// `1`/`on`/`true` — HEAL: an undeclared precise-root cycle has the +/// conservative scan forced for that cycle (sound: the scan restores +/// liveness, and a conservatively-scanned cycle is non-moving). This is the +/// measuring mode: alloc-point full collections are legitimate today and +/// simply pay the scan. +/// `strict` — PANIC on any undeclared precise-root cycle. This is the gate +/// mode that proves the enforcement is live. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SafepointOnlyContract { + Off, + Heal, + Strict, +} + +pub(super) fn gc_safepoint_only_contract() -> SafepointOnlyContract { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init( + || match std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref() { + Ok("1") | Ok("on") | Ok("true") => SafepointOnlyContract::Heal, + Ok("strict") => SafepointOnlyContract::Strict, + _ => SafepointOnlyContract::Off, + }, + ) +} + +/// Contract enforcement chokepoint, called once at every synchronous +/// collection entry. When an undeclared precise-root collection is about to +/// begin, heal mode returns a scan-override guard that must be held for the +/// WHOLE collection: it flips the thread-local override that every consumer +/// of `conservative_stack_scan_decision()` reads — the root-scan subphase, +/// copying-minor eligibility, and the evacuation verifier alike. A previous +/// revision healed by overriding a local variable inside the root-scan +/// subphase only; copying-minor eligibility still read the global decision, +/// concluded there were no conservative roots to pin, and forced evacuation +/// moved objects that raw native-stack words still pointed at. +pub(super) fn contract_scan_heal_guard() -> Option { + if gc_safepoint_only_contract() == SafepointOnlyContract::Off { + return None; + } + if !super::roots::native_stack_maps_active() || GC_AT_DECLARED_SAFEPOINT.with(Cell::get) { + return None; + } + if matches!( + super::roots::conservative_stack_scan_decision(), + super::roots::ConservativeStackScanDecision::Scan + ) { + return None; + } + if gc_safepoint_only_contract() == SafepointOnlyContract::Strict { + panic!( + "PERRY_GC_SAFEPOINT_ONLY: precise-root collection began outside \ + a declared safepoint" + ); + } + Some(super::roots::ManualGcScanGuard::force_full_scan( + super::ConservativeScanSite::SafepointContractHeal, + )) +} + +/// RAII marker for a declared-safepoint drain. Nesting-safe: restores the +/// previous value so a poll firing inside a manual `gc()` cannot clear it. +pub(super) struct DeclaredSafepointGuard { + prev: bool, +} + +impl DeclaredSafepointGuard { + pub(super) fn enter() -> Self { + let prev = GC_AT_DECLARED_SAFEPOINT.with(|flag| flag.replace(true)); + Self { prev } + } +} + +impl Drop for DeclaredSafepointGuard { + fn drop(&mut self) { + let prev = self.prev; + GC_AT_DECLARED_SAFEPOINT.with(|flag| flag.set(prev)); + } } /// Committed arena bytes a deferred nursery trigger may allocate **past the @@ -1716,6 +1811,7 @@ pub(crate) fn gc_safepoint_moving_minor() { // We are handling this safepoint (collect or find nothing due): clear the // deferral flag set by the alloc-point arm (Phase 2/3). GC_SAFEPOINT_PENDING.with(|p| p.set(false)); + let _declared = DeclaredSafepointGuard::enter(); let kind = match gc_budgeted_due_trigger() { Some(BudgetedGcTrigger::ArenaBytes) => GcTriggerKind::ArenaBytes, Some(BudgetedGcTrigger::MallocCount) => GcTriggerKind::MallocCount, diff --git a/crates/perry-runtime/src/gc/root_words.rs b/crates/perry-runtime/src/gc/root_words.rs index 78b5016a3c..705a586bd9 100644 --- a/crates/perry-runtime/src/gc/root_words.rs +++ b/crates/perry-runtime/src/gc/root_words.rs @@ -143,13 +143,12 @@ pub(super) fn decode_root_word(bits: u64) -> Option { }) } -/// Mark the object referenced by a mutable-root slot word — shadow-stack -/// slots (`MutableRootSlotKind::ShadowStack`) and registered module-global -/// roots (`MutableRootSlotKind::GlobalRoot`) alike. +/// Mark the object referenced by a mutable-root slot word — shadow-stack, +/// native stack-map, and registered module-global slots alike. /// -/// Both slot kinds are rewritten by `rewrite_mutable_root_slots` through -/// `try_rewrite_value`, so both must be marked through the same decoder -/// (#6910). Marking is address-identical for the two forms once decoded — +/// All slot kinds are rewritten by `rewrite_mutable_root_slots` through +/// `try_rewrite_value`, so all must be marked through the same decoder +/// (#6910). Marking is address-identical for the forms once decoded — /// `try_mark_raw_root_addr` performs the same validation and mark that /// `try_mark_value` does after unwrapping a NaN box — so a single call /// covers them. diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index 91390de30e..2086639bb2 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -5,7 +5,11 @@ mod runtime_handles; mod scan_mode; mod scanner_shims; mod shadow_stack; +mod stack_maps; mod temp_roots; +pub(super) use stack_maps::initialize as initialize_stack_maps; +pub(super) use stack_maps::native_maps_active as native_stack_maps_active; +pub(super) use stack_maps::record_native_stack_walk_source; pub(super) use runtime_handles::{ new_runtime_handle_root_scan_state, scan_runtime_handle_roots_mut, @@ -1339,13 +1343,14 @@ pub(super) fn atomic_store_ordering( /// Which registry a mutable root slot came from. /// /// The kind selects a *telemetry bucket* only — it must never select a -/// different pointer decoding. Both kinds are marked by -/// `mark_mutable_root_bits` and rewritten by `try_rewrite_value`, and both +/// different pointer decoding. All kinds are marked by +/// `mark_mutable_root_bits` and rewritten by `try_rewrite_value`, and all /// therefore accept a heap reference either NaN-boxed or bare. That symmetry /// is the #6910 invariant; see `gc::root_words`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum MutableRootSlotKind { ShadowStack, + NativeStack, GlobalRoot, } @@ -1370,7 +1375,10 @@ impl MutableRootSlot { /// Visit every live shadow-stack slot. The visitor receives real /// mutable slot addresses so the same walk can support mark-only /// scanning and post-forwarding rewrites. -pub(super) fn visit_shadow_stack_root_slots(mut visit: impl FnMut(MutableRootSlot)) { +pub(super) fn visit_shadow_stack_root_slots( + mut visit: impl FnMut(MutableRootSlot), +) -> stack_maps::NativeStackWalkStats { + let native_stack_walk = stack_maps::visit_stack_map_root_slots(&mut visit); SHADOW.with(|cell| unsafe { let s = &mut *cell.get(); if s.len == 0 || s.ptr.is_null() { @@ -1412,6 +1420,7 @@ pub(super) fn visit_shadow_stack_root_slots(mut visit: impl FnMut(MutableRootSlo top = header.value as usize; } }); + native_stack_walk } /// Visit every registered module-global root slot. @@ -1432,9 +1441,12 @@ pub(super) fn visit_global_root_slots(mut visit: impl FnMut(MutableRootSlot)) { /// Visit the root slots whose storage is owned by this runtime and can /// therefore be rewritten after evacuation. -pub(super) fn visit_mutable_root_slots(mut visit: impl FnMut(MutableRootSlot)) { - visit_shadow_stack_root_slots(&mut visit); +pub(super) fn visit_mutable_root_slots( + mut visit: impl FnMut(MutableRootSlot), +) -> stack_maps::NativeStackWalkStats { + let native_stack_walk = visit_shadow_stack_root_slots(&mut visit); visit_global_root_slots(&mut visit); + native_stack_walk } #[derive(Default)] @@ -1459,7 +1471,7 @@ pub(super) fn mark_mutable_root_slots_step( } let mut seen = 0usize; let mut exhausted = true; - visit_shadow_stack_root_slots(|slot| unsafe { + let native_stack_walk = visit_shadow_stack_root_slots(|slot| unsafe { if seen < cursor.shadow_seen { seen += 1; return; @@ -1471,8 +1483,10 @@ pub(super) fn mark_mutable_root_slots_step( let bits = slot.read(); record_mutable_slot_scan_source(slot, bits, valid_ptrs, &mut root_sources); - if let Some(stats) = shadow_stats.as_mut() { - stats.record_scan(bits); + if matches!(slot.kind, MutableRootSlotKind::ShadowStack) { + if let Some(stats) = shadow_stats.as_mut() { + stats.record_scan(bits); + } } if bits != 0 { mark_mutable_root_bits(bits, valid_ptrs); @@ -1481,6 +1495,7 @@ pub(super) fn mark_mutable_root_slots_step( cursor.shadow_seen = seen; remaining -= 1; }); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); if !exhausted { return false; } @@ -1554,6 +1569,7 @@ pub(super) fn root_source_for_mutable_slot( ) -> &mut RootSourceSlotTraceStats { match kind { MutableRootSlotKind::ShadowStack => &mut sources.compiled_shadow, + MutableRootSlotKind::NativeStack => &mut sources.compiled_native, MutableRootSlotKind::GlobalRoot => &mut sources.module_globals, } } @@ -1593,7 +1609,7 @@ pub(super) fn mark_mutable_root_slots( mut shadow_stats: Option<&mut ShadowRootTraceStats>, mut root_sources: Option<&mut RootSourcesTraceStats>, ) { - visit_mutable_root_slots(|slot| unsafe { + let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { let bits = slot.read(); record_mutable_slot_scan_source(slot, bits, valid_ptrs, &mut root_sources); if matches!(slot.kind, MutableRootSlotKind::ShadowStack) { @@ -1606,6 +1622,7 @@ pub(super) fn mark_mutable_root_slots( } mark_mutable_root_bits(bits, valid_ptrs); }); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); } #[inline] diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs new file mode 100644 index 0000000000..77aca0824f --- /dev/null +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -0,0 +1,1282 @@ +//! Research precise-root backend for LLVM stack maps. +//! +//! The plain-map prototype places `llvm.experimental.stackmap` immediately +//! before mapped calls and records the address of each native root alloca. +//! The statepoint prototype instead records LLVM-owned spill slots for +//! `gc.relocate` values. Both are writable frame-register-relative locations +//! in the emitted stack-map section. +//! +//! This first implementation deliberately targets macOS, where the experiment +//! is being measured. It discovers the concatenated `__PERRY_GCMAP` section +//! in the main Mach-O image and uses the platform unwinder to recover the +//! frame-register value for each active generated frame. Unsupported targets +//! return no roots; neither native-stack experiment may be used for correctness +//! there. + +use super::{MutableRootSlot, MutableRootSlotKind}; +use crate::gc::telemetry::RootSourcesTraceStats; +use std::ffi::c_void; +use std::sync::OnceLock; + +/// Magic and version of the compact map the compiler emits +/// (`perry-codegen/src/gc_map.rs`). LLVM's own stack-map section is rewritten +/// at assembly time and never reaches the binary: >50% of it was the +/// statepoint constant preamble and base/derived duplicates that this parser +/// discarded anyway, and shipping it cost 3.9 MB on a real application. +const GC_MAP_MAGIC: &[u8; 4] = b"PGCM"; +const GC_MAP_VERSION: u8 = 3; +const MAX_SAFEPOINT_RETURN_DELTA: usize = 16; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct StackMapLocation { + dwarf_reg: u16, + offset: i32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct StackMapRecord { + pc: usize, + /// Start address of the containing function, from the map's function + /// table. Used to decode that function's prologue when an SP-relative + /// location needs the FP-to-SP offset (see `fp_to_sp_offset`). + function_address: usize, + /// The containing function's total frame size from the function table. + stack_size: u64, + /// Half-open range into `StackMapIndex::roots`. + /// + /// A range rather than an owned `Vec` because **77% of records have the + /// identical live set as the record before them** — consecutive safepoints + /// in a function usually share their roots — so the decoder points the + /// repeats at one copy instead of duplicating 154k entries. + roots_start: u32, + roots_len: u32, +} + +/// Parsed section plus the facts the fast walker's preconditions need. +/// +/// `chain_walkable` is decided once at parse time: the raw x29-chain walk can +/// recover only the frame pointer (register 29) directly, plus the body SP +/// (register 31) derived from the header's per-function stack size. Any other +/// register anywhere in the maps disables the fast path for the whole image +/// rather than risking a wrong base mid-walk. +#[derive(Debug, Default)] +struct StackMapIndex { + records: Vec, + /// Every root slot, referenced by `StackMapRecord`'s range. Shared between + /// records whose live sets are identical. + roots: Vec, + chain_walkable: bool, + min_pc: usize, + max_pc: usize, +} + +impl StackMapIndex { + fn locations(&self, record: &StackMapRecord) -> &[StackMapLocation] { + let start = record.roots_start as usize; + let end = start + record.roots_len as usize; + self.roots.get(start..end).unwrap_or(&[]) + } +} + +static STACK_MAPS: OnceLock = OnceLock::new(); + +const DWARF_REG_FP_AARCH64: u16 = 29; +const DWARF_REG_SP_AARCH64: u16 = 31; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WalkerMode { + /// x29-chain walk when `chain_walkable`, transparent unwinder fallback otherwise. + Fast, + /// Force the platform unwinder (bisection control). + Unwind, + /// Run both walks and panic unless they visit the identical slot set. + /// This is the only check that can catch a fast walk that silently skips + /// frames: forced-evacuation verification enumerates roots through the + /// same walker, so it cannot see a slot the walker never reached. + Verify, +} + +fn walker_mode() -> WalkerMode { + static MODE: OnceLock = OnceLock::new(); + *MODE.get_or_init(|| match std::env::var("PERRY_STACKMAP_WALKER").as_deref() { + Ok("unwind") => WalkerMode::Unwind, + Ok("verify") => WalkerMode::Verify, + _ => WalkerMode::Fast, + }) +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(in crate::gc) struct NativeStackWalkStats { + pub(in crate::gc) walks: usize, + pub(in crate::gc) frames_visited: usize, + pub(in crate::gc) records_matched: usize, + pub(in crate::gc) locations_visited: usize, + pub(in crate::gc) fp_walks: usize, + pub(in crate::gc) fallback_walks: usize, +} + +#[inline] +pub(in crate::gc) fn record_native_stack_walk_source( + stats: NativeStackWalkStats, + root_sources: &mut Option<&mut RootSourcesTraceStats>, +) { + if let Some(sources) = root_sources { + sources.native_stack_maps.record_walk( + stats.walks, + stats.frames_visited, + stats.records_matched, + stats.locations_visited, + stats.fp_walks, + stats.fallback_walks, + ); + } +} + +pub(in crate::gc) fn initialize() { + let _ = stack_maps(); +} + +/// Whether this image carries any native stack-map records — i.e. whether +/// precise frame roots depend on mapped PCs at all. Consumed by the +/// `PERRY_GC_SAFEPOINT_ONLY` contract assert. +pub(in crate::gc) fn native_maps_active() -> bool { + !stack_maps().records.is_empty() +} + +fn stack_maps() -> &'static StackMapIndex { + STACK_MAPS.get_or_init(|| { + // No section at all is the ordinary shadow-stack build: there are no + // native frame roots to find, and an empty index is the right answer. + let Some(section) = loaded_stack_map_section() else { + return StackMapIndex::default(); + }; + // A section that exists but does not decode is a different thing + // entirely, and it must never degrade to "no roots". The two failure + // shapes are indistinguishable downstream — both yield an empty index + // — but their consequences are not: with statepoints as the only root + // mechanism, an empty index means the collector frees live objects and + // corrupts the heap with no diagnostic at all. That is CLAUDE.md's + // fourth gate-failure mode (the gate runs, its subject never did), so + // fail loudly instead. In practice this can only mean a binary whose + // compiler and runtime disagree about the map format. + let Some((mut records, roots)) = parse_gc_map(section) else { + panic!( + "perry: the GC map section (__perry_gcmap / .perry_gcmap, {} bytes) is \ + present but could not be decoded — expected format {:?} v{}. This binary's \ + compiler and runtime disagree about the map layout; continuing would run \ + the collector with no roots and corrupt the heap silently.", + section.len(), + std::str::from_utf8(GC_MAP_MAGIC).unwrap_or("PGCM"), + GC_MAP_VERSION, + ); + }; + records.sort_unstable_by_key(|record| record.pc); + index_records(records, roots) + }) +} + +fn index_records(records: Vec, roots: Vec) -> StackMapIndex { + // SP-relative locations are admitted here and resolved per FRAME in the + // walker, which decodes the owning function's `add x29, sp, #imm` + // prologue to get the body SP (#7173). Deciding it here would mean + // dereferencing every function address at startup — unsafe for records + // whose addresses are not live code, and unnecessary because the walker + // already fails closed to the platform unwinder on any anomaly. + // The decoder only ever produces these two bases, but keep the check: it + // is what decides the fast walker is usable at all, and a format change + // that introduced a third base must disable the chain walk, not be + // trusted by it. + let chain_walkable = roots.iter().all(|location| { + matches!( + location.dwarf_reg, + DWARF_REG_FP_AARCH64 | DWARF_REG_SP_AARCH64 + ) + }); + let min_pc = records.first().map_or(usize::MAX, |record| record.pc); + let max_pc = records.last().map_or(0, |record| record.pc); + StackMapIndex { + records, + roots, + chain_walkable, + min_pc, + max_pc, + } +} + +/// Recover a function's frame-pointer-to-stack-pointer offset by decoding its +/// prologue (#7173). +/// +/// AArch64 prologues set the frame pointer with a single +/// `add x29, sp, #imm` after saving the `[x29, x30]` pair, so the body SP is +/// `fp - imm`. On Darwin that offset is a constant (the pair sits at the top +/// of the frame) but on Linux it varies per function with the callee-save +/// area laid out below the pair — measured 0x30 and 0x60 in adjacent +/// generated functions, which is why no `(fp, stack_size)` formula works +/// there and the fast chain previously fell back to the DWARF unwinder for +/// every collection (~22% of samples on a Pi 5). +/// +/// Instruction encoding: ADD (immediate, 64-bit, shift 0) with Rn = 31 (sp) +/// and Rd = 29 (fp) — `word & 0xFFC0_03FF == 0x9100_03FD`, immediate in bits +/// [21:10]. Scans a bounded prologue window and fails closed (`None`) if the +/// pattern is absent, in which case the caller uses the platform unwinder. +#[cfg(target_arch = "aarch64")] +fn fp_to_sp_offset(function_address: usize) -> Option { + const ADD_FP_SP_MASK: u32 = 0xFFC0_03FF; + const ADD_FP_SP_PATTERN: u32 = 0x9100_03FD; + const PROLOGUE_WINDOW_INSNS: usize = 24; + if function_address == 0 || function_address & 0x3 != 0 { + return None; + } + for i in 0..PROLOGUE_WINDOW_INSNS { + let word = unsafe { std::ptr::read((function_address + i * 4) as *const u32) }; + if word & ADD_FP_SP_MASK == ADD_FP_SP_PATTERN { + return Some(((word >> 10) & 0xFFF) as usize); + } + // `ret` ends the prologue window for a leaf that never sets up fp. + if word == 0xD65F_03C0 { + break; + } + } + None +} + +#[cfg(not(target_arch = "aarch64"))] +fn fp_to_sp_offset(_function_address: usize) -> Option { + None +} + +fn closest_record_pc(maps: &[StackMapRecord], ip: usize) -> Option { + let insertion = maps.partition_point(|record| record.pc < ip); + let before = insertion + .checked_sub(1) + .and_then(|idx| maps.get(idx)) + .map(|record| record.pc); + let at_or_after = maps.get(insertion).map(|record| record.pc); + match (before, at_or_after) { + (Some(before), Some(after)) => Some(if ip.abs_diff(before) <= ip.abs_diff(after) { + before + } else { + after + }), + (Some(before), None) => Some(before), + (None, Some(after)) => Some(after), + (None, None) => None, + } +} + +impl StackMapIndex { + /// The records describing the frame whose return address is `ip`: the + /// ±16-byte nearest-PC match, which can select several records at one PC + /// (plain maps sit just before the call, statepoints exactly at the + /// return address). + fn match_records(&self, ip: usize) -> &[StackMapRecord] { + let Some(candidate_pc) = closest_record_pc(&self.records, ip) else { + return &[]; + }; + if ip.abs_diff(candidate_pc) > MAX_SAFEPOINT_RETURN_DELTA { + return &[]; + } + let first = self + .records + .partition_point(|record| record.pc < candidate_pc); + let last = self + .records + .partition_point(|record| record.pc <= candidate_pc); + &self.records[first..last] + } +} + +pub(super) fn visit_stack_map_root_slots( + visit: &mut impl FnMut(MutableRootSlot), +) -> NativeStackWalkStats { + let index = stack_maps(); + if index.records.is_empty() { + return NativeStackWalkStats::default(); + } + match walker_mode() { + WalkerMode::Unwind => unwind::visit(index, visit), + WalkerMode::Fast => { + if index.chain_walkable { + if let Some(stats) = fp_chain::visit(index, visit) { + return stats; + } + } + let mut stats = unwind::visit(index, visit); + stats.fallback_walks = 1; + stats + } + WalkerMode::Verify => verify_visit(index, visit), + } +} + +/// Debug-only cross-check: the fast walk reads slot addresses without +/// mutating, then the unwinder performs the real visitation while recording +/// what it reached. Any set difference is a missed or invented frame and +/// panics immediately — this is the liveness gate for the fast walker itself. +fn verify_visit( + index: &StackMapIndex, + visit: &mut impl FnMut(MutableRootSlot), +) -> NativeStackWalkStats { + let mut fast_addresses: Vec = Vec::new(); + let fast_stats = fp_chain::visit(index, &mut |slot: MutableRootSlot| { + fast_addresses.push(slot.ptr as usize); + }); + let Some(fast_stats) = fast_stats else { + panic!( + "PERRY_STACKMAP_WALKER=verify: fast walk unavailable \ + (chain_walkable={}, anomaly or unsupported target)", + index.chain_walkable + ); + }; + let mut unwind_addresses: Vec = Vec::new(); + let mut stats = unwind::visit(index, &mut |slot: MutableRootSlot| { + unwind_addresses.push(slot.ptr as usize); + visit(slot); + }); + fast_addresses.sort_unstable(); + fast_addresses.dedup(); + unwind_addresses.sort_unstable(); + unwind_addresses.dedup(); + assert_eq!( + fast_addresses, + unwind_addresses, + "PERRY_STACKMAP_WALKER=verify: fast walk visited {} unique slots, \ + unwinder visited {}", + fast_addresses.len(), + unwind_addresses.len() + ); + stats.fp_walks = fast_stats.fp_walks; + stats +} + +/// Decode every concatenated compact map in the section. +/// +/// The linker concatenates one blob per object file, so this walks blob by +/// blob using each header's `total_len` rather than assuming a single map — +/// a decoder that reads only the first header silently drops every other +/// object's roots, which is invisible until a collection frees a live object. +fn parse_gc_map(bytes: &[u8]) -> Option<(Vec, Vec)> { + let mut records = Vec::new(); + let mut roots: Vec = Vec::new(); + let mut base = 0usize; + + while base + 16 <= bytes.len() { + if bytes.get(base..base + 4)? != GC_MAP_MAGIC { + // Linkers pad between input sections; a zero tail is the end. + if bytes[base..].iter().all(|byte| *byte == 0) { + break; + } + base += 1; + continue; + } + if read_u8(bytes, base + 4)? != GC_MAP_VERSION { + return None; + } + let function_count = read_u32(bytes, base + 8)? as usize; + let total_len = read_u32(bytes, base + 12)? as usize; + // A blob must at least cover its header and function table. Without + // this, a `total_len` of 0 leaves `base` unchanged — and because the + // magic still matches at that offset the resynchronisation path below + // is never reached, so the loop spins forever. This runs inside + // `OnceLock::get_or_init`, so that is a process hang at the first + // collection rather than the fail-closed panic in `stack_maps`. + if total_len < 16 + function_count.checked_mul(16)? { + return None; + } + let table = base.checked_add(16)?; + let stream_start = table.checked_add(function_count.checked_mul(16)?)?; + let blob_end = base.checked_add(total_len)?; + if blob_end > bytes.len() || stream_start > blob_end { + return None; + } + + // Instruction offsets are a fixed-width array ahead of the varint + // stream: at -O3 the compiler emits them as label differences the + // assembler evaluates, so their values cannot be varint-encoded at + // rewrite time. + // Not `unwrap_or(0)`: a failed read here means the function table is + // truncated, and treating that function as having zero records starts + // `cursor` at the wrong offset so every later varint decodes from + // misaligned bytes. A wrong live set is worse than no map. + let mut record_total: usize = 0; + for index in 0..function_count { + record_total = + record_total.checked_add(read_u32(bytes, table + index * 16 + 12)? as usize)?; + } + let offsets = stream_start; + let mut cursor = offsets.checked_add(record_total.checked_mul(4)?)?; + if cursor > blob_end { + return None; + } + let mut record_index = 0usize; + + for index in 0..function_count { + let entry = table + index * 16; + let function_address = read_u64(bytes, entry)? as usize; + let stack_size = u64::from(read_u32(bytes, entry + 8)?); + let record_count = read_u32(bytes, entry + 12)? as usize; + + let mut previous: Option<(u32, u32)> = None; + for _ in 0..record_count { + let instruction_offset = read_u32(bytes, offsets + record_index * 4)?; + record_index += 1; + + let (header, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + let range = if header & 1 == 1 { + // Repeat: this safepoint's live set is the previous one's. + previous? + } else { + let count = (header >> 1) as usize; + let start = u32::try_from(roots.len()).ok()?; + let mut last: Option = None; + for _ in 0..count { + let (value, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + // 2-bit base tag: 0 = FP, 1 = SP, 2 = explicit DWARF + // register in a following varint (LLVM uses x19 as a + // frame base in functions with dynamic allocation). + let dwarf_reg = match value & 3 { + 0 => DWARF_REG_FP_AARCH64, + 1 => DWARF_REG_SP_AARCH64, + 2 => { + let (reg, next) = read_varint(bytes, cursor, blob_end)?; + cursor = next; + u16::try_from(reg).ok()? + } + _ => return None, + }; + let delta = unzigzag((value >> 2) as u32); + let offset = match last { + None => delta, + Some(previous_offset) => previous_offset.wrapping_add(delta), + }; + last = Some(offset); + roots.push(StackMapLocation { dwarf_reg, offset }); + } + (start, u32::try_from(count).ok()?) + }; + previous = Some(range); + + records.push(StackMapRecord { + pc: function_address.checked_add(instruction_offset as usize)?, + function_address, + stack_size, + roots_start: range.0, + roots_len: range.1, + }); + } + } + + let next = align_up(blob_end, 8)?; + if next <= base { + return None; + } + base = next; + } + + Some((records, roots)) +} + +/// LEB128 read bounded by the blob it belongs to, so a corrupt length cannot +/// walk into the next blob or off the section. +fn read_varint(bytes: &[u8], mut at: usize, end: usize) -> Option<(u64, usize)> { + let mut value = 0u64; + let mut shift = 0u32; + loop { + if at >= end || shift > 63 { + return None; + } + let byte = *bytes.get(at)?; + at += 1; + value |= u64::from(byte & 0x7F) << shift; + if byte & 0x80 == 0 { + return Some((value, at)); + } + shift += 7; + } +} + +fn unzigzag(value: u32) -> i32 { + ((value >> 1) as i32) ^ -((value & 1) as i32) +} + +fn align_up(value: usize, alignment: usize) -> Option { + value + .checked_add(alignment.checked_sub(1)?) + .map(|value| value & !(alignment - 1)) +} + +fn read_u8(bytes: &[u8], offset: usize) -> Option { + bytes.get(offset).copied() +} + +/// ELF section headers store their counts and offsets as 16-bit fields, so +/// this is used only by `elf_section_vaddr`. Gated to Linux because the +/// compact GC map itself needs no 16-bit reads — deleting it as "orphaned" +/// after a macOS-only `cargo check` is what broke the Linux build. +#[cfg(target_os = "linux")] +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_le_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_le_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_le_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +#[cfg(target_os = "macos")] +fn loaded_stack_map_section() -> Option<&'static [u8]> { + use mach2::dyld::{_dyld_get_image_header, _dyld_get_image_vmaddr_slide}; + + const LC_SEGMENT_64: u32 = 0x19; + + #[repr(C)] + #[derive(Clone, Copy)] + struct MachHeader64 { + magic: u32, + cpu_type: i32, + cpu_subtype: i32, + file_type: u32, + command_count: u32, + commands_size: u32, + flags: u32, + reserved: u32, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct LoadCommand { + command: u32, + size: u32, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct SegmentCommand64 { + command: u32, + size: u32, + segment_name: [u8; 16], + vm_address: u64, + vm_size: u64, + file_offset: u64, + file_size: u64, + max_protection: i32, + initial_protection: i32, + section_count: u32, + flags: u32, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct Section64 { + section_name: [u8; 16], + segment_name: [u8; 16], + address: u64, + size: u64, + offset: u32, + alignment: u32, + relocation_offset: u32, + relocation_count: u32, + flags: u32, + reserved1: u32, + reserved2: u32, + reserved3: u32, + } + + fn fixed_name_matches(actual: &[u8; 16], expected: &[u8]) -> bool { + actual.get(..expected.len()) == Some(expected) + && actual.get(expected.len()).copied().unwrap_or(0) == 0 + } + + unsafe { + let raw_header = _dyld_get_image_header(0); + if raw_header.is_null() { + return None; + } + let header = &*(raw_header.cast::()); + let slide = _dyld_get_image_vmaddr_slide(0); + let mut command_ptr = raw_header + .cast::() + .add(std::mem::size_of::()); + for _ in 0..header.command_count { + let load = std::ptr::read_unaligned(command_ptr.cast::()); + if load.size < std::mem::size_of::() as u32 { + return None; + } + if load.command == LC_SEGMENT_64 { + let segment = std::ptr::read_unaligned(command_ptr.cast::()); + let mut section_ptr = command_ptr.add(std::mem::size_of::()); + for _ in 0..segment.section_count { + let section = std::ptr::read_unaligned(section_ptr.cast::()); + if fixed_name_matches(§ion.segment_name, b"__PERRY_GCMAP") + && fixed_name_matches(§ion.section_name, b"__perry_gcmap") + { + let address = (section.address as isize).checked_add(slide)? as usize; + let size = usize::try_from(section.size).ok()?; + if address == 0 || size == 0 { + return None; + } + return Some(std::slice::from_raw_parts(address as *const u8, size)); + } + section_ptr = section_ptr.add(std::mem::size_of::()); + } + } + command_ptr = command_ptr.add(load.size as usize); + } + } + None +} + +/// ELF (#7173): the `.perry_gcmap` section of the main executable. +/// +/// Linker-provided `__start_`/`__stop_` symbols would need weak linkage +/// (unstable in Rust) or `-rdynamic` (not guaranteed), so instead: read +/// `/proc/self/exe`'s section headers for `.perry_gcmap` (sh_addr, +/// sh_size) and add the main object's load bias from the first +/// `dl_iterate_phdr` callback. Runtime-verified gates for this path are +/// pending a Linux host — tracked in #7173; the parser, index, matching, +/// and verify machinery above are platform-independent already. +#[cfg(target_os = "linux")] +fn loaded_stack_map_section() -> Option<&'static [u8]> { + let bytes = std::fs::read("/proc/self/exe").ok()?; + let (addr, size) = elf_section_vaddr(&bytes, b".perry_gcmap")?; + let bias = main_object_load_bias()?; + let start = bias.checked_add(addr)?; + if start == 0 || size == 0 { + return None; + } + Some(unsafe { std::slice::from_raw_parts(start as *const u8, size) }) +} + +/// Minimal ELF64 section-header walk: returns (sh_addr, sh_size) for the +/// named section. Same defensive read style as the stack-map parser. +#[cfg(target_os = "linux")] +fn elf_section_vaddr(bytes: &[u8], name: &[u8]) -> Option<(usize, usize)> { + if bytes.get(..4)? != b"\x7fELF" || *bytes.get(4)? != 2 { + return None; // not ELF64 + } + let shoff = read_u64(bytes, 0x28)? as usize; + let shentsize = read_u16(bytes, 0x3A)? as usize; + let shnum = read_u16(bytes, 0x3C)? as usize; + let shstrndx = read_u16(bytes, 0x3E)? as usize; + let strtab_hdr = shoff.checked_add(shstrndx.checked_mul(shentsize)?)?; + let strtab_off = read_u64(bytes, strtab_hdr.checked_add(0x18)?)? as usize; + for i in 0..shnum { + let hdr = shoff.checked_add(i.checked_mul(shentsize)?)?; + let name_off = read_u32(bytes, hdr)? as usize; + let name_pos = strtab_off.checked_add(name_off)?; + let candidate = bytes.get(name_pos..name_pos.checked_add(name.len())?)?; + let terminator = bytes.get(name_pos + name.len()).copied().unwrap_or(1); + if candidate == name && terminator == 0 { + let addr = read_u64(bytes, hdr.checked_add(0x10)?)? as usize; + let size = read_u64(bytes, hdr.checked_add(0x20)?)? as usize; + return Some((addr, size)); + } + } + None +} + +/// Load bias of the main object: `dlpi_addr` of the first `dl_iterate_phdr` +/// callback (the executable itself on glibc and musl). +#[cfg(target_os = "linux")] +fn main_object_load_bias() -> Option { + #[repr(C)] + struct DlPhdrInfo { + dlpi_addr: usize, + dlpi_name: *const std::os::raw::c_char, + // remaining fields unused + } + unsafe extern "C" { + fn dl_iterate_phdr( + callback: unsafe extern "C" fn(*mut DlPhdrInfo, usize, *mut c_void) -> i32, + data: *mut c_void, + ) -> i32; + } + unsafe extern "C" fn first(info: *mut DlPhdrInfo, _size: usize, data: *mut c_void) -> i32 { + unsafe { + *data.cast::() = (*info).dlpi_addr; + } + 1 // stop after the first (main) object + } + let mut bias = usize::MAX; + unsafe { + dl_iterate_phdr(first, (&mut bias as *mut usize).cast::()); + } + (bias != usize::MAX).then_some(bias) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn loaded_stack_map_section() -> Option<&'static [u8]> { + None +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod unwind { + use super::*; + + #[repr(C)] + struct UnwindContext { + _private: [u8; 0], + } + + unsafe extern "C" { + fn _Unwind_Backtrace( + trace: unsafe extern "C" fn(*mut UnwindContext, *mut c_void) -> i32, + argument: *mut c_void, + ) -> i32; + fn _Unwind_GetIP(context: *mut UnwindContext) -> usize; + fn _Unwind_GetGR(context: *mut UnwindContext, register: i32) -> usize; + } + + struct WalkState<'a, F> { + index: &'a StackMapIndex, + visit: &'a mut F, + stats: NativeStackWalkStats, + } + + pub(super) fn visit( + index: &StackMapIndex, + visit: &mut F, + ) -> NativeStackWalkStats { + let mut state = WalkState { + index, + visit, + stats: NativeStackWalkStats { + walks: 1, + ..NativeStackWalkStats::default() + }, + }; + unsafe { + _Unwind_Backtrace( + walk_frame::, + (&mut state as *mut WalkState<'_, _>).cast::(), + ); + } + state.stats + } + + unsafe extern "C" fn walk_frame( + context: *mut UnwindContext, + argument: *mut c_void, + ) -> i32 { + let state = &mut *argument.cast::>(); + state.stats.frames_visited = state.stats.frames_visited.saturating_add(1); + let ip = _Unwind_GetIP(context); + let matched = state.index.match_records(ip); + if matched.is_empty() { + return 0; + } + state.stats.records_matched = state.stats.records_matched.saturating_add(matched.len()); + for record in matched { + for location in state.index.locations(record) { + state.stats.locations_visited = state.stats.locations_visited.saturating_add(1); + let base = _Unwind_GetGR(context, i32::from(location.dwarf_reg)); + let address = if location.offset < 0 { + base.checked_sub(location.offset.unsigned_abs() as usize) + } else { + base.checked_add(location.offset as usize) + }; + let Some(address) = address else { + continue; + }; + if address == 0 || address & (std::mem::align_of::() - 1) != 0 { + continue; + } + (state.visit)(MutableRootSlot { + kind: MutableRootSlotKind::NativeStack, + ptr: address as *mut u64, + }); + } + } + 0 + } +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +mod unwind { + use super::*; + + pub(super) fn visit( + _index: &StackMapIndex, + _visit: &mut impl FnMut(MutableRootSlot), + ) -> NativeStackWalkStats { + NativeStackWalkStats::default() + } +} + +/// Raw x29-chain walker. +/// +/// AArch64 prologues under `"frame-pointer"="non-leaf"` are +/// `stp x29, x30, [sp, #-16]!; mov x29, sp`, so every frame's x29 points at +/// a `[caller x29, return address]` pair. One hop is therefore two loads, +/// against a full unwind step (compact-unwind lookup plus register +/// recovery) — this is what turns the measured 350:1 frames-to-roots ratio +/// from a tax into noise. +/// +/// Fail-closed everywhere: a misaligned, non-increasing, or out-of-bounds +/// frame pointer abandons the walk with `None` and the caller re-runs the +/// whole scan through the platform unwinder. Slot visitation is idempotent +/// (a rewritten slot no longer points at a forwarded object), so a partial +/// fast walk followed by a full unwinder walk is safe. +#[cfg(all(any(target_os = "macos", target_os = "linux"), target_arch = "aarch64"))] +mod fp_chain { + use super::*; + + fn current_frame_pointer() -> usize { + let fp: usize; + unsafe { + core::arch::asm!("mov {fp}, x29", fp = out(reg) fp, options(nomem, nostack)); + } + fp + } + + #[cfg(target_os = "macos")] + fn stack_top() -> usize { + unsafe extern "C" { + fn pthread_self() -> usize; + fn pthread_get_stackaddr_np(thread: usize) -> *mut c_void; + } + unsafe { pthread_get_stackaddr_np(pthread_self()) as usize } + } + + /// Linux (#7173): stack bounds via pthread attrs — the returned address + /// is the LOW end, so the exclusive top is addr + size. Runtime gates + /// pending a Linux host; a failure here returns 0 and the caller falls + /// back to the platform unwinder (fail-closed like every other anomaly). + #[cfg(target_os = "linux")] + fn stack_top() -> usize { + unsafe extern "C" { + fn pthread_self() -> usize; + fn pthread_getattr_np(thread: usize, attr: *mut u8) -> i32; + fn pthread_attr_getstack( + attr: *const u8, + stackaddr: *mut *mut c_void, + stacksize: *mut usize, + ) -> i32; + fn pthread_attr_destroy(attr: *mut u8) -> i32; + } + // pthread_attr_t is at most 64 bytes on glibc/musl for the supported + // targets; over-allocate defensively. + let mut attr = [0u8; 128]; + let mut addr: *mut c_void = std::ptr::null_mut(); + let mut size: usize = 0; + unsafe { + if pthread_getattr_np(pthread_self(), attr.as_mut_ptr()) != 0 { + return 0; + } + let ok = pthread_attr_getstack(attr.as_ptr(), &mut addr, &mut size) == 0; + pthread_attr_destroy(attr.as_mut_ptr()); + if !ok { + return 0; + } + } + (addr as usize).saturating_add(size) + } + + pub(super) fn visit( + index: &StackMapIndex, + visit: &mut F, + ) -> Option { + if !index.chain_walkable { + return None; + } + let top = stack_top(); + if top == 0 { + return None; + } + let mut stats = NativeStackWalkStats { + walks: 1, + fp_walks: 1, + ..NativeStackWalkStats::default() + }; + let low_pc = index.min_pc.saturating_sub(MAX_SAFEPOINT_RETURN_DELTA); + let high_pc = index.max_pc.saturating_add(MAX_SAFEPOINT_RETURN_DELTA); + let mut fp = current_frame_pointer(); + while fp != 0 { + if fp & 0xF != 0 || fp.checked_add(16)? > top { + return None; + } + let return_address = unsafe { *((fp + 8) as *const usize) }; + let caller_fp = unsafe { *(fp as *const usize) }; + stats.frames_visited = stats.frames_visited.saturating_add(1); + if return_address == 0 { + break; + } + if return_address >= low_pc && return_address <= high_pc { + let matched = index.match_records(return_address); + { + if !matched.is_empty() { + // The record describes the caller's frame; its + // locations are relative to the caller's own x29, + // which is exactly the saved word we just read. + // + // It gets the SAME validation `fp` gets at the top of + // the loop, and it gets it BEFORE the root loop rather + // than after. Every FP-relative root is based on this + // word, and `fp_to_sp_offset` subtracts from it for the + // SP-relative ones; downstream the only filters are + // non-zero and 8-byte alignment, so an unvalidated + // `caller_fp` lets a corrupt frame produce addresses + // outside the stack that the collector then reads and + // rewrites. Fail closed to the platform unwinder. + if caller_fp == 0 + || caller_fp & 0xF != 0 + || caller_fp <= fp + || caller_fp.checked_add(16)? > top + { + return None; + } + stats.records_matched = stats.records_matched.saturating_add(matched.len()); + for record in matched { + // Body SP = fp - (prologue's `add x29, sp, #imm`). + // `chain_walkable` proved this decodes for every + // SP-relative record in the image (#7173). + let sp = fp_to_sp_offset(record.function_address) + .and_then(|off| caller_fp.checked_sub(off)); + for location in index.locations(record) { + stats.locations_visited = stats.locations_visited.saturating_add(1); + let base = if location.dwarf_reg == DWARF_REG_FP_AARCH64 { + Some(caller_fp) + } else { + sp + }; + let Some(base) = base else { + return None; + }; + let address = if location.offset < 0 { + base.checked_sub(location.offset.unsigned_abs() as usize) + } else { + base.checked_add(location.offset as usize) + }; + let Some(address) = address else { + continue; + }; + if address == 0 || address & (std::mem::align_of::() - 1) != 0 + { + continue; + } + visit(MutableRootSlot { + kind: MutableRootSlotKind::NativeStack, + ptr: address as *mut u64, + }); + } + } + } + } + } + if caller_fp != 0 && caller_fp <= fp { + return None; + } + fp = caller_fp; + } + Some(stats) + } +} + +#[cfg(not(all(any(target_os = "macos", target_os = "linux"), target_arch = "aarch64")))] +mod fp_chain { + use super::*; + + pub(super) fn visit( + _index: &StackMapIndex, + _visit: &mut impl FnMut(MutableRootSlot), + ) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn push_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8 & 0x7F) | 0x80); + value >>= 7; + } + out.push(value as u8); + } + + fn zigzag(value: i32) -> u64 { + ((value << 1) ^ (value >> 31)) as u32 as u64 + } + + /// Build one compact blob, mirroring `perry-codegen/src/gc_map.rs`. + /// `records` is `(instruction_offset, roots)`, roots as `(dwarf_reg, offset)`; + /// an empty root slice with `repeat` set encodes the repeat flag. + fn one_map(function: u64, records: &[(u32, Vec<(u16, i32)>, bool)]) -> Vec { + let mut offsets = Vec::new(); + let mut stream = Vec::new(); + for (instruction_offset, roots, repeat) in records { + offsets.extend_from_slice(&instruction_offset.to_le_bytes()); + if *repeat { + push_varint(&mut stream, 1); + continue; + } + push_varint(&mut stream, (roots.len() as u64) << 1); + let mut last: Option = None; + for (reg, offset) in roots { + let tag = match *reg { + DWARF_REG_FP_AARCH64 => 0u64, + DWARF_REG_SP_AARCH64 => 1, + _ => 2, + }; + let delta = match last { + None => *offset, + Some(previous) => offset.wrapping_sub(previous), + }; + push_varint(&mut stream, (zigzag(delta) << 2) | tag); + if tag == 2 { + push_varint(&mut stream, u64::from(*reg)); + } + last = Some(*offset); + } + } + + let total_len = 16 + 16 + offsets.len() + stream.len(); + let mut bytes = Vec::new(); + bytes.extend_from_slice(GC_MAP_MAGIC); + bytes.push(GC_MAP_VERSION); + bytes.extend_from_slice(&[0, 0, 0]); + bytes.extend_from_slice(&1u32.to_le_bytes()); + bytes.extend_from_slice(&(total_len as u32).to_le_bytes()); + bytes.extend_from_slice(&function.to_le_bytes()); + bytes.extend_from_slice(&32u32.to_le_bytes()); + bytes.extend_from_slice(&(records.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&offsets); + bytes.extend_from_slice(&stream); + while bytes.len() % 8 != 0 { + bytes.push(0); + } + bytes + } + + fn simple(function: u64, offset: u32, frame_offset: i32) -> Vec { + one_map(function, &[(offset, vec![(29, frame_offset)], false)]) + } + + #[test] + fn decodes_frame_location() { + let bytes = simple(0x1000, 0x10, -8); + let (records, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].pc, 0x1010); + assert_eq!(records[0].function_address, 0x1000); + assert_eq!(records[0].stack_size, 32); + assert_eq!( + roots, + vec![StackMapLocation { + dwarf_reg: 29, + offset: -8, + }] + ); + } + + #[test] + fn decodes_linker_concatenated_input_sections() { + let mut bytes = simple(0x1000, 0x10, -8); + bytes.extend_from_slice(&simple(0x2000, 0x20, -16)); + let (records, _) = parse_gc_map(&bytes).expect("concatenated maps"); + assert_eq!(records.len(), 2); + assert_eq!(records[0].pc, 0x1010); + assert_eq!(records[1].pc, 0x2020); + } + + #[test] + fn repeated_live_sets_share_one_copy() { + // Three safepoints, the last two repeating the first's live set: the + // whole point of the format, and the reason the in-memory index does + // not hold 154k duplicated entries on a real application. + let bytes = one_map( + 0x1000, + &[ + (0x10, vec![(29, -8), (29, -16)], false), + (0x20, vec![], true), + (0x30, vec![], true), + ], + ); + let (records, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!(records.len(), 3); + assert_eq!(roots.len(), 2, "the repeats must not append new roots"); + for record in &records { + assert_eq!(record.roots_start, 0); + assert_eq!(record.roots_len, 2); + } + } + + #[test] + fn decodes_negative_and_ascending_root_offsets() { + let bytes = one_map(0x1000, &[(0, vec![(29, -64), (29, -8), (31, 24)], false)]); + let (_, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!( + roots, + vec![ + StackMapLocation { + dwarf_reg: 29, + offset: -64 + }, + StackMapLocation { + dwarf_reg: 29, + offset: -8 + }, + StackMapLocation { + dwarf_reg: 31, + offset: 24 + }, + ] + ); + } + + #[test] + fn decodes_an_explicit_base_register() { + // LLVM uses x19 as a frame base pointer in functions with dynamic + // stack allocation — 66 root slots in one real module. A single FP/SP + // bit cannot express that, which is what forced the 2-bit base tag. + let bytes = one_map(0x1000, &[(0x10, vec![(19, -40), (29, -8)], false)]); + let (_, roots) = parse_gc_map(&bytes).expect("valid map"); + assert_eq!( + roots, + vec![ + StackMapLocation { + dwarf_reg: 19, + offset: -40 + }, + StackMapLocation { + dwarf_reg: 29, + offset: -8 + }, + ] + ); + } + + #[test] + fn an_explicit_base_register_disables_the_fast_walk() { + // The x29-chain walker can only recover FP and SP; anything else must + // fall back to the platform unwinder, which can. + let index = index_records( + vec![StackMapRecord { + pc: 0x1000, + function_address: 0x1000, + stack_size: 64, + roots_start: 0, + roots_len: 1, + }], + vec![StackMapLocation { + dwarf_reg: 19, + offset: -40, + }], + ); + assert!(!index.chain_walkable); + } + + #[test] + fn rejects_a_blob_whose_length_cannot_advance_the_cursor() { + // `total_len` comes straight from the header. A zero (or too-small) + // value leaves `base` where it was, and because the magic still + // matches there the resync path never runs — the loop spins forever + // inside `OnceLock::get_or_init`, hanging the process at the first + // collection instead of failing closed. + let mut bytes = simple(0x1000, 0x10, -8); + bytes[12..16].copy_from_slice(&0u32.to_le_bytes()); + assert!( + parse_gc_map(&bytes).is_none(), + "a blob that cannot advance the cursor must be rejected, not looped on" + ); + + // Long enough to look plausible, still short of header + function table. + let mut bytes = simple(0x1000, 0x10, -8); + bytes[12..16].copy_from_slice(&20u32.to_le_bytes()); + assert!(parse_gc_map(&bytes).is_none()); + } + + #[test] + fn rejects_a_truncated_function_table() { + // The record counts size the fixed-width offset array; a short read + // there must not be rounded down to zero, or every later varint is + // decoded from the wrong offset. + let bytes = simple(0x1000, 0x10, -8); + let truncated = &bytes[..20]; + assert!(parse_gc_map(truncated).is_none()); + } + + #[test] + fn rejects_truncated_or_wrong_version_sections() { + assert!(parse_gc_map(&[]).is_none() || parse_gc_map(&[]).unwrap().0.is_empty()); + let mut bytes = simple(0x1000, 0x10, -8); + bytes[4] = GC_MAP_VERSION + 1; + assert!( + parse_gc_map(&bytes).is_none(), + "an unknown version must not be guessed at" + ); + // A total_len that runs past the section must fail rather than read on. + let mut bytes = simple(0x1000, 0x10, -8); + let len = bytes.len(); + bytes[12..16].copy_from_slice(&((len as u32) + 64).to_le_bytes()); + assert!(parse_gc_map(&bytes).is_none()); + } + + #[test] + fn chain_walkable_index_accepts_fp_and_sp_locations_only() { + let rec = |pc: usize| StackMapRecord { + pc, + function_address: pc, + stack_size: 160, + roots_start: 0, + roots_len: 1, + }; + // FP and SP are both walkable: SP resolves per frame by decoding the + // owning function's prologue (#7173). + let walkable = index_records( + vec![rec(0x1000), rec(0x2000)], + vec![ + StackMapLocation { + dwarf_reg: DWARF_REG_FP_AARCH64, + offset: -8, + }, + StackMapLocation { + dwarf_reg: DWARF_REG_SP_AARCH64, + offset: -8, + }, + ], + ); + assert!(walkable.chain_walkable); + assert_eq!(walkable.min_pc, 0x1000); + assert_eq!(walkable.max_pc, 0x2000); + // Any other register disqualifies the whole image. + assert!( + !index_records( + vec![rec(0x1000)], + vec![StackMapLocation { + dwarf_reg: 1, + offset: -8 + }], + ) + .chain_walkable, + "a non-FP/SP register must disable the fast walk" + ); + } + + #[test] + fn matches_plain_maps_before_and_statepoints_after_unwinder_ips() { + let rec = |pc: usize| StackMapRecord { + pc, + function_address: pc, + stack_size: 32, + roots_start: 0, + roots_len: 0, + }; + let maps = vec![rec(0x1000), rec(0x1020)]; + assert_eq!(closest_record_pc(&maps, 0x1004), Some(0x1000)); + assert_eq!(closest_record_pc(&maps, 0x101c), Some(0x1020)); + assert_eq!(closest_record_pc(&maps, 0x1020), Some(0x1020)); + } +} diff --git a/crates/perry-runtime/src/gc/scan_fallback.rs b/crates/perry-runtime/src/gc/scan_fallback.rs index d83347be7a..345a400c9f 100644 --- a/crates/perry-runtime/src/gc/scan_fallback.rs +++ b/crates/perry-runtime/src/gc/scan_fallback.rs @@ -72,6 +72,12 @@ pub(crate) enum ConservativeScanSite { ManualCollect, /// `js_gc_module_minor` — explicit `perry/gc` `minor()`. Explicit. ManualMinor, + /// `PERRY_GC_SAFEPOINT_ONLY` heal (#7174 research): a precise-root + /// collection began outside a declared safepoint, so the contract forces + /// the scan for that cycle rather than consuming roots that native + /// stack maps only describe at mapped PCs. Automatic, and research-mode + /// only — it cannot fire unless the contract env is set. + SafepointContractHeal, // ★ There is deliberately no `HostPressure` variant. `js_gc_memory_pressure` // used to force the scan unconditionally; after #7148 it either collects // with precise roots (empty shadow stack) or defers to a safepoint (a @@ -83,7 +89,7 @@ pub(crate) enum ConservativeScanSite { } impl ConservativeScanSite { - pub(crate) const COUNT: usize = 5; + pub(crate) const COUNT: usize = 6; const fn index(self) -> usize { match self { @@ -92,6 +98,7 @@ impl ConservativeScanSite { Self::EmergencyReclaim => 2, Self::ManualCollect => 3, Self::ManualMinor => 4, + Self::SafepointContractHeal => 5, } } @@ -102,6 +109,7 @@ impl ConservativeScanSite { Self::EmergencyReclaim => "emergency_reclaim", Self::ManualCollect => "manual_collect", Self::ManualMinor => "manual_minor", + Self::SafepointContractHeal => "safepoint_contract_heal", } } @@ -110,9 +118,10 @@ impl ConservativeScanSite { /// collections a program pays for without asking for them. pub(crate) const fn is_automatic(self) -> bool { match self { - Self::OldReclaimAllocPoint | Self::NurseryChurnSlackValve | Self::EmergencyReclaim => { - true - } + Self::OldReclaimAllocPoint + | Self::NurseryChurnSlackValve + | Self::EmergencyReclaim + | Self::SafepointContractHeal => true, Self::ManualCollect | Self::ManualMinor => false, } } @@ -124,6 +133,7 @@ impl ConservativeScanSite { Self::EmergencyReclaim, Self::ManualCollect, Self::ManualMinor, + Self::SafepointContractHeal, ]; } diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index abae571489..a2f346e5be 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -300,13 +300,45 @@ pub(super) struct NativeStackFallbackTraceStats { pub(super) compiled_frame_pinned_bytes: usize, } +#[derive(Clone, Copy, Default)] +pub(super) struct NativeStackMapTraceStats { + pub(super) walks: usize, + pub(super) frames_visited: usize, + pub(super) records_matched: usize, + pub(super) locations_visited: usize, + pub(super) fp_walks: usize, + pub(super) fallback_walks: usize, +} + +impl NativeStackMapTraceStats { + #[inline] + pub(super) fn record_walk( + &mut self, + walks: usize, + frames_visited: usize, + records_matched: usize, + locations_visited: usize, + fp_walks: usize, + fallback_walks: usize, + ) { + self.walks = self.walks.saturating_add(walks); + self.frames_visited = self.frames_visited.saturating_add(frames_visited); + self.records_matched = self.records_matched.saturating_add(records_matched); + self.locations_visited = self.locations_visited.saturating_add(locations_visited); + self.fp_walks = self.fp_walks.saturating_add(fp_walks); + self.fallback_walks = self.fallback_walks.saturating_add(fallback_walks); + } +} + #[derive(Clone, Copy, Default)] pub(super) struct RootSourcesTraceStats { pub(super) compiled_shadow: RootSourceSlotTraceStats, + pub(super) compiled_native: RootSourceSlotTraceStats, pub(super) module_globals: RootSourceSlotTraceStats, pub(super) runtime_handles: RootSourceSlotTraceStats, pub(super) runtime_mutable_scanners: RootSourceSlotTraceStats, pub(super) ffi_mutable_scanners: RootSourceSlotTraceStats, + pub(super) native_stack_maps: NativeStackMapTraceStats, pub(super) native_stack_fallback: NativeStackFallbackTraceStats, } @@ -1317,10 +1349,19 @@ pub(super) fn root_source_slot_json(stats: RootSourceSlotTraceStats) -> serde_js pub(super) fn root_sources_json(stats: RootSourcesTraceStats) -> serde_json::Value { serde_json::json!({ "compiled_shadow": root_source_slot_json(stats.compiled_shadow), + "compiled_native": root_source_slot_json(stats.compiled_native), "module_globals": root_source_slot_json(stats.module_globals), "runtime_handles": root_source_slot_json(stats.runtime_handles), "runtime_mutable_scanners": root_source_slot_json(stats.runtime_mutable_scanners), "ffi_mutable_scanners": root_source_slot_json(stats.ffi_mutable_scanners), + "native_stack_maps": { + "walks": stats.native_stack_maps.walks, + "frames_visited": stats.native_stack_maps.frames_visited, + "records_matched": stats.native_stack_maps.records_matched, + "locations_visited": stats.native_stack_maps.locations_visited, + "fp_walks": stats.native_stack_maps.fp_walks, + "fallback_walks": stats.native_stack_maps.fallback_walks, + }, "native_stack_fallback": { "decision": stats.native_stack_fallback.decision.as_str(), "scanned": stats.native_stack_fallback.scanned, diff --git a/crates/perry-runtime/src/gc/tests/copying.rs b/crates/perry-runtime/src/gc/tests/copying.rs index cffaf0074f..c35ed39488 100644 --- a/crates/perry-runtime/src/gc/tests/copying.rs +++ b/crates/perry-runtime/src/gc/tests/copying.rs @@ -770,6 +770,19 @@ fn root_source_active_shadow_frame_reports_precise_shadow_roots_only() { ); } +#[test] +fn root_source_native_stack_slot_has_a_distinct_telemetry_bucket() { + let mut sources = RootSourcesTraceStats::default(); + root_source_for_mutable_slot(&mut sources, MutableRootSlotKind::NativeStack) + .record_scan(true, true); + root_source_for_mutable_slot(&mut sources, MutableRootSlotKind::NativeStack).record_rewrite(); + + assert_eq!(sources.compiled_native.slots_scanned, 1); + assert_eq!(sources.compiled_native.pointer_roots, 1); + assert_eq!(sources.compiled_native.rewritten_slots, 1); + assert_eq!(sources.compiled_shadow.slots_scanned, 0); +} + #[test] fn test_copied_minor_eligibility_empty_rust_copy_only_scanner_falls_back() { let _guard = CopyingNurseryTestGuard::new(0); diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index 81b5c1a7c6..4f2f0aa79e 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -243,6 +243,19 @@ fn root_heavy_workload_reports_root_sources_and_budgeted_progression() { >= u64::from(roots), "shadow-root telemetry should classify the roots as pointers" ); + assert!( + event["root_sources"]["compiled_native"].is_object(), + "native stack-map roots need their own source bucket" + ); + assert!( + event["root_sources"]["native_stack_maps"]["frames_visited"].is_number(), + "native stack-map telemetry should expose unwinder work" + ); + assert!( + event["root_sources"]["native_stack_maps"]["fp_walks"].is_number() + && event["root_sources"]["native_stack_maps"]["fallback_walks"].is_number(), + "native stack-map telemetry should expose which walker ran" + ); let live_after = (js_shadow_slot_get(0) & POINTER_MASK) as *const crate::StringHeader; unsafe { diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index ef15f56b32..7df3ca728b 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -892,7 +892,7 @@ pub(super) fn rewrite_mutable_root_slots_with_sources( mut shadow_stats: Option<&mut ShadowRootTraceStats>, mut root_sources: Option<&mut RootSourcesTraceStats>, ) { - visit_mutable_root_slots(|slot| unsafe { + let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { let bits = slot.read(); record_mutable_slot_scan_source(slot, bits, valid_ptrs, &mut root_sources); if bits == 0 { @@ -908,6 +908,7 @@ pub(super) fn rewrite_mutable_root_slots_with_sources( } } }); + record_native_stack_walk_source(native_stack_walk, &mut root_sources); } pub(super) fn rewrite_mutable_registered_roots(valid_ptrs: &ValidPointerSet) { @@ -948,6 +949,7 @@ pub(super) fn verify_mutable_root_slots(valid_ptrs: &ValidPointerSet) { if let Some(new_bits) = try_rewrite_value(bits, valid_ptrs) { let surface = match slot.kind { MutableRootSlotKind::ShadowStack => "shadow stack roots", + MutableRootSlotKind::NativeStack => "native stack-map roots", MutableRootSlotKind::GlobalRoot => "global roots", }; panic_stale_forwarded_reference(surface, slot.ptr as usize, bits, new_bits); diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index d40c6e259c..d917284cc5 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -38,6 +38,9 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_LLVM_INPROCESS", "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", + "PERRY_STATEPOINTS", + "PERRY_RS4GC", + "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 1c333afd95..3b1c3a492b 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -230,6 +230,7 @@ fn stable_type_key(ty: &perry_hir::types::Type) -> String { /// at compile time but that aren't part of `CompileOptions`: /// `PERRY_DEBUG_INIT`, `PERRY_DEBUG_SYMBOLS`, `PERRY_LLVM_CLANG`, /// `PERRY_WRITE_BARRIERS`, `PERRY_SHADOW_STACK`, +/// `PERRY_STATEPOINTS`, /// `PERRY_DISABLE_BUFFER_FAST_PATH`, `PERRY_VERIFY_NATIVE_REGIONS`, /// `PERRY_UNBOXED_OBJECT_FIELDS`, and `PERRY_TARGET_CPU`. See the env-var /// block at the bottom of this function for the rationale. @@ -759,6 +760,10 @@ fn compute_object_cache_key_with_env( // calls at heap-store sites (codegen.rs / expr.rs). // - PERRY_SHADOW_STACK=0/off/false suppresses generated frame/slot // roots at function entry and pointer local stores. + // - (historical) PERRY_STACK_MAPS lowered precise roots to plain LLVM stackmap records; deleted, statepoint fallback keeps the lowering internal. PERRY_STATEPOINTS=1 lowers roots to native-frame + // stack maps instead of the runtime shadow stack. + // - PERRY_STATEPOINTS=1 replaces supported calls with LLVM statepoint + // relocation sequences and uses native stack maps for the remainder. // - PERRY_DISABLE_BUFFER_FAST_PATH=1 overrides CompileOptions and // changes Buffer/Uint8Array lowering. // - PERRY_VERIFY_NATIVE_REGIONS=1 overrides CompileOptions and must @@ -801,6 +806,18 @@ fn compute_object_cache_key_with_env( "env_shadow_stack", env_var("PERRY_SHADOW_STACK").as_deref().unwrap_or(""), ); + h.field( + "env_statepoints", + env_var("PERRY_STATEPOINTS").as_deref().unwrap_or(""), + ); + h.field("env_rs4gc", env_var("PERRY_RS4GC").as_deref().unwrap_or("")); + // Explicit-safepoint contract: flips audited AllocNoReentry helpers + // between statepoint and plain call. Two arms sharing a cached object + // would make the contract's metadata reduction unmeasurable. + h.field( + "env_gc_safepoint_only", + env_var("PERRY_GC_SAFEPOINT_ONLY").as_deref().unwrap_or(""), + ); // #7088: flips the shadow-slot store between an inline sequence and the // `js_shadow_slot_*` calls. Two arms that shared a cached object would // silently measure the same code. diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index d36aa631fa..180df97796 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -584,6 +584,9 @@ fn key_changes_with_codegen_env_vars() { "PERRY_LLVM_INPROCESS", "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", + "PERRY_STATEPOINTS", + "PERRY_RS4GC", + "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", "PERRY_UNBOXED_OBJECT_FIELDS", diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 677ef4563b..4ee2318cf1 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -223,6 +223,31 @@ pub fn run_with_parse_cache( std::env::set_var("PERRY_NO_CACHE", "1"); } + // Native-stack GC root-pressure report. Like `--opt-report`, this is + // observational and must be enabled before rayon starts module codegen. + // Cache reuse is disabled because cached objects bypass the lowering that + // records each function. + let statepoint_report_format = args.statepoint_report.or_else(|| { + match std::env::var("PERRY_STATEPOINT_REPORT").as_deref() { + Ok("json") => Some(StatepointReportFormat::Json), + Ok("1") | Ok("text") => Some(StatepointReportFormat::Text), + _ => None, + } + }); + if let Some(fmt) = statepoint_report_format { + std::env::set_var( + "PERRY_STATEPOINT_REPORT", + match fmt { + StatepointReportFormat::Json => "json", + StatepointReportFormat::Text => "text", + }, + ); + std::env::set_var("PERRY_NO_CACHE", "1"); + // `perry dev` reuses the process; discard records from its previous + // build before starting this one. + let _ = perry_codegen::statepoint_report::take_records(); + } + // Canonicalize the input path first so its `.parent()` is an absolute directory. // Without this, a bare filename like `perry demo.ts` produced `Path::new("").parent()` // → fallback `"."`, and the walk-up loops below (package.json + perry.toml discovery) @@ -4704,6 +4729,15 @@ pub fn run_with_parse_cache( eprintln!("{rendered}"); } + if let Some(fmt) = statepoint_report_format { + let records = perry_codegen::statepoint_report::take_records(); + let rendered = match fmt { + StatepointReportFormat::Json => perry_codegen::statepoint_report::render_json(&records), + StatepointReportFormat::Text => perry_codegen::statepoint_report::render_text(&records), + }; + eprintln!("{rendered}"); + } + // #835 + #846: fold the codegen-side FFI provenance registry into // ctx so the well-known flip and `needs_stdlib` decisions below see // the symbols codegen actually emitted, not just the modules the diff --git a/crates/perry/src/commands/compile/types.rs b/crates/perry/src/commands/compile/types.rs index 94b3ced08e..0f266205a2 100644 --- a/crates/perry/src/commands/compile/types.rs +++ b/crates/perry/src/commands/compile/types.rs @@ -494,6 +494,18 @@ pub struct CompileArgs { /// that codegen actually executes and has something to report. #[arg(long, value_enum, num_args = 0..=1, default_missing_value = "text")] pub opt_report: Option, + + /// Report native-stack GC root pressure for the stack-map/statepoint + /// research backends. Shows calls with live roots, audited calls that + /// cannot collect, statepoint relocation counts, plain stack-map + /// fallbacks, and the live-root-width distribution. + /// + /// Useful with `PERRY_STATEPOINTS=1`. + /// `--statepoint-report=json` emits a stable machine-readable schema. + /// Observational only; cache reuse is disabled for the reporting run so + /// codegen executes and produces records. + #[arg(long, value_enum, num_args = 0..=1, default_missing_value = "text")] + pub statepoint_report: Option, } /// Output format for `--opt-report`. @@ -505,6 +517,15 @@ pub enum OptReportFormat { Json, } +/// Output format for `--statepoint-report`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum StatepointReportFormat { + /// Human-readable root-pressure summary and ranked callees. + Text, + /// Stable JSON schema for tooling. + Json, +} + /// Information about a JavaScript module that will be interpreted at runtime #[derive(Debug, Clone)] pub struct JsModule { diff --git a/crates/perry/src/commands/dev.rs b/crates/perry/src/commands/dev.rs index 62a92d78ff..e2da75f66e 100644 --- a/crates/perry/src/commands/dev.rs +++ b/crates/perry/src/commands/dev.rs @@ -314,6 +314,7 @@ fn build_once( disable_buffer_fast_path: false, explain_lowering: false, opt_report: None, + statepoint_report: None, emit_attest: false, emit_sandbox: false, lockdown: false, diff --git a/crates/perry/src/commands/run/mod.rs b/crates/perry/src/commands/run/mod.rs index 6669976afc..dcfe5d7a53 100644 --- a/crates/perry/src/commands/run/mod.rs +++ b/crates/perry/src/commands/run/mod.rs @@ -226,6 +226,7 @@ pub fn run(args: RunArgs, format: OutputFormat, use_color: bool, verbose: u8) -> disable_buffer_fast_path: false, explain_lowering: false, opt_report: None, + statepoint_report: None, emit_attest: false, emit_sandbox: false, lockdown: false, diff --git a/docs/src/cli/flags.md b/docs/src/cli/flags.md index 537c7b72ca..46a9f3615b 100644 --- a/docs/src/cli/flags.md +++ b/docs/src/cli/flags.md @@ -103,6 +103,7 @@ accept either the `$perryfs/` virtual path or the embed-relative key. | `--no-codegen` | Skip the `package.json` `perry.codegen` build-time steps (also `PERRY_SKIP_CODEGEN=1`). See [Project Configuration](../getting-started/project-config.md) | | `--keep-intermediates` | Keep `.o` and `.asm` intermediate files | | `--opt-report[=json]` | Report which values Perry could **not** statically type, why, and whether you can fix it. Text by default; `--opt-report=json` emits a stable schema for tooling. Also settable via `PERRY_OPT_REPORT=1` | +| `--statepoint-report[=json]` | Report native-stack GC root pressure: calls with live roots, audited non-collecting calls omitted, relocations, plain-map fallbacks, and live-root widths. Research-only; requires `PERRY_STATEPOINTS=1` or `PERRY_RS4GC=1` (the plain stack-map mode it also named is gone) | The `--trace`/`--focus` pair localizes "compiled to the wrong thing" bugs: `perry compile foo.ts --trace hir,llvm --focus parseRow` dumps just the @@ -228,6 +229,7 @@ shrink less, proportionally. | `CI=true` | Auto-skip update checks (set by most CI systems) | | `RUST_LOG` | Debug logging level (`debug`, `info`, `trace`) | | `PERRY_OPT_REPORT` | `1`/`text` or `json` — same as `--opt-report[=json]`, for driving the report from an environment where adding a flag is awkward | +| `PERRY_STATEPOINT_REPORT` | `1`/`text` or `json` — same as `--statepoint-report[=json]`; observational root-pressure reporting for the native-stack GC experiments | ## Configuration Files diff --git a/docs/stack-map-gc-experiment.md b/docs/stack-map-gc-experiment.md new file mode 100644 index 0000000000..dfa031928d --- /dev/null +++ b/docs/stack-map-gc-experiment.md @@ -0,0 +1,235 @@ +# Stack-map GC experiment + +Date: 2026-07-31 + +Branch: `exp/stackmap-viability` + +Base commit: `e2557c1a985cb983ed00aafd1a2c1b31f1570b98` + +## Decision + +LLVM stack maps are technically viable for Perry's moving collector on +macOS/arm64, but this prototype does **not** justify replacing the shadow stack +yet. + +- Correctness is promising: all eight GC-ratchet probes match Node, forced + evacuation verification passes, and retained heap is byte-for-byte identical + to the shadow-stack arm. +- Runtime performance is effectively flat on ordinary workloads. The GC suite + measured a noisy 1.5% geometric-mean improvement, while an interleaved + deep-stack run was 1.5% slower. The host was heavily loaded, so neither is a + defensible headline win. +- Compilation time is flat. Root-heavy executables gain an approximately + 16 KiB Mach-O segment even when the stack-map payload is only 2–9 KiB. +- Root enumeration is not simpler overall. The heap-backed frame stack and its + TLS traffic can disappear, but they are replaced by LLVM ABI parsing, + linker-retention rules, native unwinding, call-site matching, compiler memory + barriers, and control-flow liveness analysis. +- The direction is strategically useful, but a production follow-up should + investigate `gc.statepoint`/`gc.relocate`, not promote this plain + `llvm.experimental.stackmap` prototype. + +The prototype remains opt-in with `PERRY_STACK_MAPS=1`; the default +shadow-stack path is unchanged. + +## What was built + +The experiment reuses Perry's existing precise-root discovery and slot +numbering, then changes the backend: + +1. Shadow-slot bind/clear operations remain temporary IR markers. +2. A final per-function pass resolves logical slots to their native allocas. +3. A conservative CFG dataflow pass computes roots that may be live at each + call. A join uses union, so it may retain a stale value but cannot omit a + live root. +4. Each call with live roots gets an `llvm.experimental.stackmap` carrying the + addresses of those allocas. LLVM emits them as writable `Direct` + frame-register-relative locations. +5. Empty memory-clobbering inline assembly brackets mapped calls. It generates + no instructions, but makes the collector's otherwise invisible slot rewrite + observable to LLVM before and after the call. +6. The runtime parses LLVM stack-map v3 records from the main Mach-O image, + unwinds active frames with `_Unwind_Backtrace`, finds the record immediately + preceding each return PC, and feeds its mutable slots into the existing GC + root visitor. +7. Mach-O module assembly marks each local `__LLVM_StackMaps` atom + `.no_dead_strip`; otherwise Perry's normal link removes the metadata. + +The parser accepts linker-concatenated stack-map blobs and handles LLVM's +independent alignment before and after the live-out list. Both details caused +real failures during the spike and now have regression coverage. + +LLVM documents this intrinsic and binary format as experimental and explicitly +separate from its GC statepoint machinery: +[Stack maps and patch points](https://llvm.org/docs/StackMaps.html) and +[Garbage collection safepoints](https://llvm.org/docs/Statepoints.html). + +## Correctness results + +The final release artifacts passed: + +- all 8 GC-ratchet probes, with stdout identical to Node; +- all 8 probes again with `PERRY_GC_FORCE_EVACUATE=1` and + `PERRY_GC_VERIFY_EVACUATION=1`; +- the string-retention probe 100 consecutive times after it exposed the + parser/call-site issues; +- all 310 `perry-codegen` library tests; +- 1,570 `perry-runtime` library tests in single-threaded mode (3 ignored), + excluding the existing debug-only `extern "C"` malformed-pop test whose + intentional `debug_assert!` aborts a debug test process; +- 3 stack-map parser tests; +- 3 stack-map lowering/liveness tests; +- 46 object-cache tests, including `PERRY_STACK_MAPS` cache separation. + +Across the full ratchet, median retained heap and heap capacity were identical +for every probe. Promotions were identical. Six probes copied exactly the same +number of objects; the remaining differences were +3, -6, and +2 objects, with +the same final retention and freed bytes. + +The spike found three important correctness requirements: + +- A moving collector must map writable native slots, not merely record pointer + values. Passing alloca addresses produces LLVM `Direct` locations. +- Stack-map parsing must honor the pre-live-out alignment in the v3 format. + Missing it desynchronized any record with an odd number of locations. +- A shadow-slot clear is a liveness change, not a write to the program local. + Zeroing the native local corrupted a value used after its last GC-capable + call. Static per-call liveness is required. + +## Performance results + +Hardware was an Apple M1 Max on macOS 26.5. The GC runs reported load averages +between 19 and 45, so the numbers below are directional only. Each A/B used the +same final compiler and runtime archives, with caches and auto-optimization +disabled. Runtime pairs were interleaved where noted. + +### GC-ratchet wall time + +Three measured runs plus one warmup per mode: + +| Probe | Shadow | Stack map | Delta | +|---|---:|---:|---:| +| Nursery churn | 182.581 ms | 175.766 ms | -3.73% | +| Survivor promotion | 213.684 ms | 213.661 ms | -0.01% | +| Cross-generation writes | 210.161 ms | 203.122 ms | -3.35% | +| Dead after deep stack | 471.128 ms | 484.463 ms | +2.83% | +| Closure capture | 175.824 ms | 163.789 ms | -6.84% | +| String retention | 135.017 ms | 135.645 ms | +0.47% | +| Array grow/evacuate | 171.918 ms | 174.017 ms | +1.22% | +| Map/set side tables | 459.315 ms | 449.043 ms | -2.24% | + +Geometric mean: stack maps were 1.50% faster. This is smaller than the +cross-run noise expected under the recorded host load. An additional +11-pair interleaved run measured: + +- deep active stack: stack maps **1.49% slower**; +- string retention: stack maps **0.07% faster**. + +The deep-stack result is consistent with native unwinding costing more than a +linear walk over the compact shadow buffer. + +### Broader runtime samples + +Eleven interleaved runs per binary: + +| Workload | Shadow | Stack map | Delta | +|---|---:|---:|---:| +| Process startup | 5.351 ms | 5.238 ms | -2.10% | +| Method calls | 93.295 ms | 93.383 ms | +0.09% | +| Function calls | 111.325 ms | 111.055 ms | -0.24% | +| GC pressure | 38.958 ms | 38.874 ms | -0.22% | +| JSON roundtrip | 362.536 ms | 363.680 ms | +0.32% | + +The substantive workloads are flat. Startup's 0.11 ms difference is below +what this host can resolve. + +### Compile time and size + +Five uncached compilations per cell: + +| Workload | Shadow | Stack map | Delta | +|---|---:|---:|---:| +| Startup | 449.71 ms | 449.76 ms | +0.01% | +| Method calls | 472.72 ms | 471.16 ms | -0.33% | +| Function calls | 457.69 ms | 457.73 ms | +0.01% | +| GC pressure | 465.83 ms | 460.47 ms | -1.15% | +| JSON roundtrip | 486.94 ms | 489.84 ms | +0.59% | + +The stack-map section measured 2,192 bytes for method calls, 5,512 bytes for GC +pressure, and 8,568 bytes for JSON roundtrip. On these Mach-O executables the +new segment rounded the file-size increase to roughly 16 KiB. Programs with no +mapped roots emitted no section and had identical file size. + +The modified runtime archive is 18,656 bytes larger than the exact baseline +(30,313,640 versus 30,294,984 bytes). + +## Is the GC simpler? + +Only in a narrow sense. + +The stack-map scanner is 451 lines in this spike versus 752 lines for the +current shadow-stack runtime. A completed replacement could also delete much +of the 563-line inline shadow-slot emitter and remove frame push/pop, TLS buffer +growth, longjmp depth restoration, and slot mirroring. + +The total system is not yet simpler: + +- the compiler gained a textual IR lowering and CFG liveness analysis; +- the linker needs platform-specific metadata retention; +- the runtime depends on an experimental LLVM binary contract and platform + unwinder behavior; +- moving-GC relocation needs writable allocas plus compiler memory fences; +- diagnostics must distinguish missing metadata from a genuinely rootless + frame; +- target support moves from ordinary generated calls to per-object-format + section discovery and per-architecture DWARF register handling. + +This trades locally optimized, explicit machinery for more cross-layer +machinery. It could become simpler after statepoints make relocation and +liveness first-class, but plain stack maps do not reach that point. + +## Is this better positioned? + +Potentially: + +- per-safepoint liveness can be more precise than a mutable activation-wide + registry; +- no shadow-frame push/pop or TLS slot mutation is required on the common + path; +- native frame metadata aligns Perry with established AOT/JIT GC techniques; +- exceptions and non-local exits naturally remove unwound frames from the + root set. + +The current prototype is not yet a platform: + +- scanning is implemented only for Mach-O/macOS; +- unknown calls remain conservatively instrumented; an audited call-effect + table now omits runtime helpers proven unable to enter Perry's collector; +- the runtime assumes the matching stack-map PC is within 16 bytes of the + unwound return PC; +- active roots must remain in addressable allocas; +- parameter/`this` bindings still need a fully audited incremental-mark + transition barrier; +- async signals, foreign callbacks, tail calls, `setjmp`/`longjmp`, code + splitting, and non-Apple object formats need dedicated end-to-end tests; +- `PERRY_STACK_MAPS=1` is currently unsafe on unsupported targets because the + runtime scanner intentionally returns no roots there. + +## Recommended next experiment + +The explicit statepoint follow-up is now recorded in +[`statepoint-gc-experiment.md`](statepoint-gc-experiment.md). It validates +relocation correctness but finds no all-around performance win and no +whole-system simplification. + +Do not replace the default shadow stack from this branch. If this direction +continues after the representation work: + +1. Add an explicit safepoint capability table so pure calls do not receive + metadata or optimization fences. +2. Close the incremental parameter/`this` barrier gap and add a probe that + enters a new rooted activation during an in-flight incremental cycle. +3. Fail compilation on unsupported targets before expanding ELF/Windows + section discovery and unwind/register support. +4. Re-run the A/B suite on an idle host with the repository's standard 11-run + methodology and profile both mutator root updates and GC root scanning. diff --git a/docs/statepoint-bridge-probe.ll b/docs/statepoint-bridge-probe.ll new file mode 100644 index 0000000000..1f8b2f2d63 --- /dev/null +++ b/docs/statepoint-bridge-probe.ll @@ -0,0 +1,36 @@ +; Minimal Perry-shaped statepoint probe. +; +; A live NaN-boxed value is carried as an addrspace(1) pointer only across +; the safepoint. The runtime may rewrite its spill slot, gc.relocate reloads +; it, and ptrtoint restores Perry's existing i64 representation. + +target triple = "arm64-apple-macosx15.0.0" + +declare i64 @may_collect(i64) +declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) +declare i64 @llvm.experimental.gc.result.i64(token) +declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1(token, i32 immarg, i32 immarg) + +define i64 @statepoint_bridge_probe(i64 %bits, i64 %arg) gc "statepoint-example" { +entry: + %root = inttoptr i64 %bits to ptr addrspace(1) + %statepoint = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( + i64 1, + i32 0, + ptr elementtype(i64 (i64)) @may_collect, + i32 1, + i32 0, + i64 %arg, + i32 0, + i32 0 + ) ["gc-live"(ptr addrspace(1) %root)] + %result = call i64 @llvm.experimental.gc.result.i64(token %statepoint) + %root.relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %statepoint, + i32 0, + i32 0 + ) + %bits.relocated = ptrtoint ptr addrspace(1) %root.relocated to i64 + %combined = xor i64 %result, %bits.relocated + ret i64 %combined +} diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md new file mode 100644 index 0000000000..10d8fe3fc3 --- /dev/null +++ b/docs/statepoint-gc-experiment.md @@ -0,0 +1,1049 @@ +# Explicit statepoint GC experiment + +Date: 2026-07-31 + +Branch: `exp/stackmap-viability` + +Base commit: `e2557c1a985cb983ed00aafd1a2c1b31f1570b98` + +## Decision + +The explicit `gc.statepoint` bridge is correct enough to validate the +mechanism, but it is not currently a performance win and does not yet make +Perry's GC simpler. Keep the shadow stack as the default. + +- All eight GC-ratchet probes pass normally and with forced evacuation plus + evacuation verification. +- The full suite emits 1,080 statepoints and 1,562 relocations. It has zero + plain-stack-map fallbacks at GC-relevant calls. +- Runtime is effectively flat versus the shadow stack: -0.27% geometric mean + on a heavily loaded host. It is 1.42% slower than the plain-stack-map arm. +- Uncached compilation is 2.12% slower than shadow-stack compilation. +- Statepoint stack-map payload is 2.01x the plain-map payload across the suite. +- Relocation is better expressed: LLVM now owns the call/result/relocated-value + relationship, and the compiler memory barriers from the plain-map prototype + disappear. +- The whole system remains more complex because native unwinding, stack-map + parsing, textual call rewriting, root liveness, fallbacks, and + platform-specific metadata retention are still required. + +The prototype remains opt-in with `PERRY_STATEPOINTS=1`. The default +shadow-stack path is unchanged. + +## Follow-up: root pressure and audited safepoints + +The first prototype treated almost every textual call with live roots as a +safepoint. That was correct but needlessly pessimistic. The follow-up adds an +audited GC-call-effect table whose only claim is whether a helper can enter +Perry's collector. Unknown calls remain safepoints. + +This is intentionally separate from LLVM memory effects. Temporary-root +bookkeeping, write barriers, layout notes, feedback counters, and refcount +writes mutate memory, but they do not run a Perry collection and therefore do +not need stack-map metadata. + +`--statepoint-report[=json]` makes the resulting root pressure visible. It +reports per-function logical/bound root slots, calls with live roots, audited +non-collecting calls, statepoints, relocations, plain-map fallbacks, live-root +widths, and callee frequencies. It is observational and disables cache reuse +for the reporting run: + +```sh +PERRY_STATEPOINTS=1 perry compile app.ts --statepoint-report +``` + +On `benchmarks/app-patterns/kernels/batch.ts`, the audit changed: + +| Metric | Before | After | Change | +|---|---:|---:|---:| +| Statepoints | 442 | 219 | -50.5% | +| Relocations | 867 | 403 | -53.5% | +| `__llvm_stackmaps` | 54,968 B | 26,432 B | -51.9% | +| Plain-map fallbacks | 0 | 0 | unchanged | + +The report found 223 calls with live roots that cannot collect. The largest +groups were typed-feedback bookkeeping, class-field guards, temporary-root +push/get/truncate, layout notes, write barriers, and property-observation +records. + +Across the eight GC probes: + +| Probe | Statepoints before | Statepoints after | Relocations after | Calls skipped | +|---|---:|---:|---:|---:| +| Nursery churn | 152 | 65 | 91 | 88 | +| Survivor promotion | 165 | 79 | 132 | 88 | +| Cross-generation writes | 168 | 74 | 95 | 96 | +| Dead after deep stack | 119 | 55 | 59 | 65 | +| Closure capture | 146 | 69 | 81 | 78 | +| String retention | 92 | 45 | 46 | 50 | +| Array grow/evacuate | 100 | 49 | 49 | 52 | +| Map/set side tables | 138 | 66 | 124 | 74 | +| **Total** | **1,080** | **502** | **677** | **591** | + +The same audit applies to the plain-map backend. Its eight-probe metadata +payload is now 26,816 bytes; explicit statepoints use 53,952 bytes. Statepoint +metadata therefore remains 2.01x plain maps even after both shrink by roughly +half. Reducing the number of maybe-pointer roots through representation +promotion remains the larger shared lever. + +All eight probes pass in shadow, plain-map, and statepoint modes with forced +evacuation and relocation verification: 24/24 mode/probe comparisons match +Node's probe/checksum output. + +### Follow-up runtime and compile measurements + +Each runtime cell below is the median of seven executions on the same host and +full-feature runtime artifact. Host variance was high, so the numbers are +directional: + +| Probe | Shadow | Plain stack map | Statepoint | Statepoint vs plain | +|---|---:|---:|---:|---:| +| Nursery churn | 196.698 ms | 194.860 ms | 195.178 ms | +0.16% | +| Survivor promotion | 234.184 ms | 235.600 ms | 233.294 ms | -0.98% | +| Cross-generation writes | 383.002 ms | 330.386 ms | 341.206 ms | +3.27% | +| Dead after deep stack | 1,029.803 ms | 996.121 ms | 1,196.806 ms | +20.15% | +| Closure capture | 230.904 ms | 212.415 ms | 188.984 ms | -11.03% | +| String retention | 151.553 ms | 173.651 ms | 162.272 ms | -6.55% | +| Array grow/evacuate | 251.063 ms | 267.094 ms | 237.260 ms | -11.17% | +| Map/set side tables | 577.092 ms | 563.401 ms | 601.368 ms | +6.74% | + +Geometric means put plain maps at -1.17% versus shadow, statepoints at -1.54% +versus shadow, and statepoints at -0.38% versus plain maps. That small aggregate +statepoint lead is below the noise floor; the deep-stack regression remains a +clear negative signal. + +For uncached `batch.ts` compilation, seven-run medians were 910.3 ms shadow, +946.0 ms plain maps (+3.92%), and 958.3 ms statepoints (+5.28%). + +### Follow-up: x29-chain fast walker (2026-07-31, after rebase onto #7114) + +The deep-stack telemetry below (36,458 frames unwound for 104 root +locations) identified `_Unwind_Backtrace` as the walker bottleneck: full +compact-unwind register recovery on every native frame. The branch now walks +the raw x29 chain instead — two loads per frame — enabled by two facts: + +1. Generated functions now carry `"frame-pointer"="non-leaf"`. Textual-IR + input gets no frame-pointer default from the clang driver, which is why + generated code previously saved x29 without establishing a chain. +2. Statepoint spills are SP-relative (`Indirect [R#31 + N]`) on AArch64 + regardless of frame-pointer attributes, but the stack-map header records + each function's frame size, and LLVM's AArch64 frame keeps the + `[x29, x30]` pair at the top of the frame — so the body SP is always + `fp + 16 - stack_size` from the same chain loads. + +The fast path is fail-closed twice over. At parse time, any location that is +not FP-relative or sized-SP-relative marks the whole image not chain-walkable. +At walk time, a misaligned, non-increasing, or out-of-bounds frame pointer +abandons the walk and re-runs the platform unwinder (slot visits are +idempotent, so a partial fast walk followed by a full unwinder walk is safe). +`PERRY_STACKMAP_WALKER=unwind` forces the old walker as a bisection control; +`PERRY_STACKMAP_WALKER=verify` runs both and panics unless they visit the +identical slot set. Verify exists because forced-evacuation verification +enumerates roots through the same walker and therefore cannot catch a walker +that silently skips frames — this is CLAUDE.md gate-failure mode 4 applied to +the walker itself. Telemetry gains `fp_walks`/`fallback_walks` so any run can +prove which walker executed. + +Results (loaded host, directional): 24/24 correctness matrix, 16/16 +verify-mode probe runs (fast walk engaged and byte-identical to the unwinder +everywhere), and the deep-stack statepoint probe improves 4.8% end-to-end +against the unwinder walker on the same binary with interleaved reps. + +A finding for the mode decision fell out of the register census: plain-map +mode emits `Register R#1` locations — the root slot's address materialized in +a caller-saved register at the map point. No parser can soundly use that +location (the register is clobbered by the callee and unrecoverable at GC +time), so those roots are structurally invisible to the collector. LLVM's +stackmap intrinsic offers no way to force the address into memory; statepoint +spill slots cannot exhibit the problem. Plain maps are therefore unsound by +construction at a small but nonzero rate (3 of 60 locations on the deep-stack +probe), which strengthens the case for deleting the plain-map arm once +statepoints match it on the walker-sensitive workloads. + +### Native-root and unwinder telemetry + +Native stack-map roots now have their own `root_sources.compiled_native` +telemetry bucket instead of being incorrectly charged to +`compiled_shadow`. `root_sources.native_stack_maps` also records walks, frames +visited, records matched, and locations visited. + +The forced-evacuation deep-stack probe reported 105 walks, 36,458 frames +visited, 36,139 records matched, and only 104 root locations visited, with a +maximum of 694 frames in one cycle. The walker is therefore a justified +optimization target, but a direct frame-pointer walker is not yet a safe +substitution: current generated AArch64 code saves `x29` without consistently +establishing an `x29` frame chain, and Rust/runtime frames have no matching +contract. A fast path first needs an explicit frame-pointer/unwind ABI for +generated and intervening runtime frames, plus fallback and cross-architecture +tests. + +### Follow-up: the explicit-safepoint collection contract (PERRY_GC_SAFEPOINT_ONLY) + +The prerequisite that gated this experiment — the #7114 temp-root +correctness fix — landed on main during the first prototype session, so the +contract experiment ran after rebasing onto it. + +**The contract.** A collection that skips the conservative stack scan +consumes only precise roots; with native stack maps active, precise frame +roots exist only at mapped PCs. Therefore such a collection may only begin +at a declared safepoint (a loop back-edge poll or the outermost +microtask-pump boundary) — anywhere else it must scan conservatively. The +runtime already routes moving minors to those safepoints (the #7024 +deferral machinery), so today the property is *emergent*: it holds because +every possibly-collecting call happens to be mapped. The contract makes it +*enforced* — a thread-local declared-safepoint flag plus a check at the +root-scan subphase — and enforcement is what makes it sound to stop mapping +call sites. + +Two enforcement levels: `PERRY_GC_SAFEPOINT_ONLY=1` (heal — an undeclared +precise-root cycle gets the conservative scan forced for that cycle, which +restores liveness and keeps it non-moving) and `=strict` (panic — the gate +mode that proves the enforcement is live, per the four-ways-a-gate-cannot- +fail rule). Manual `gc()` and the alloc-point slack valve force the scan +already and are exempt by construction. (An earlier revision also drained +non-nursery triggers at every allocating loop back-edge; that turned churn +loops into per-iteration collection work — O(n²) — and was deleted. The heal +alone is sufficient: undeclared full collections simply pay one conservative +scan.) + +**What it unmaps.** A new audited `GcCallEffect::AllocNoReentry` class: +helpers that may allocate (arming a trigger) but never collect synchronously +and never re-enter generated JS. Under the contract their call sites need no +statepoint — any trigger they arm either defers to a declared safepoint or +collects behind the forced scan. First audited set: singleton closure +allocation, class-object allocation, `js_array_push_f64`, `js_array_length`, +`js_array_slice_values`. + +**Measured results (loaded host, correctness-grade).** All gates green at +`4e3d5c70e`: 16/16 probe cells (forced evacuation + walker-verify under the +contract), strict-mode enforcement fired on the deliberately unsound +configuration (`PERRY_GC_SCAVENGE=1` + polls off — a precise-root minor at +an unmapped alloc point aborts with the contract panic), and max RSS is +unchanged by deferral (27/27 MB and 36/36 MB on the two churn-heaviest +probes). The audited five-helper set removes 7.8% of `batch.ts` statepoints +(217 → 200) and 7.5% of relocations. Getting the contract here found and +fixed three implementation bugs, each caught by a gate: enforcement that +missed the copying-minor path, a heal that overrode a local decision while +copying eligibility read the global one (real memory corruption under +forced evacuation), and a per-poll trigger drain that turned churn loops +into O(n²) collection work. All three fixes deleted code. + +**The census result that bounds the idea.** On `batch.ts`, 217 statepoints +break down as roughly 85 property-access diamonds (getter re-entry possible +— must stay mapped), ~40 coercion/setter/throw paths (re-entry — stay), ~10 +generated-to-generated calls and polls (stay by definition), and only ~25-30 +pure-allocation sites the contract can unmap. **Re-entry, not allocation, is +what bounds the contract's reach on object-heavy code.** Deleting the +property-access calls is representation selection's job (`Ptr`); the +contract unmaps what allocation traffic remains. The two campaigns compose +rather than compete. + +### Work deliberately left gated + +This follow-up does not alter temporary-root semantics, collection scheduling, +or the conservative native-stack fallback. The representation plan makes the +temp-root correctness work a prerequisite for an explicit-only collection +contract and conservative-scanner removal. Doing that here would overlap the +other agent's work and make failures impossible to attribute. Once that +prerequisite lands, the next experiment is to assert that moving collections +occur only at declared safepoints and then measure whether the conservative +scanner can be deleted. + +## Quiet-host matrix (2026-08-01, reserved Mac mini) + +First measurement of this experiment not taken on a loaded host: Apple M1 +(4P+4E), macOS 26.5.1, the gc-ratchet pinned-baseline platform, reserved +with baseline load 1.4–1.9 from release infrastructure only (recorded +per-rep). Artifacts shipped SHA-pinned from `4e3d5c70e` (no cargo on the +host); all four arms hash-distinct per probe; 8×4 forced-evacuation preflight +plus walker-verify and the strict-enforcement gate all green there before +any timing. 11 interleaved reps, rotated arm order, `/usr/bin/time -l`. +Caveat: 10 ms timer granularity puts ±1 quantum (≈2–9% on these probe +durations) on any single cell; medians were stable across reps. + +Runtime, median seconds (spread), delta vs shadow: + +| Probe | Shadow | Plain map | Statepoint | Contract | +|---|---:|---:|---:|---:| +| Nursery churn | 0.160 | 0.160 (+0.0%) | 0.160 (+0.0%) | 0.160 (+0.0%) | +| Survivor promotion | 0.190 | 0.180 (−5.3%) | 0.190 (+0.0%) | 0.190 (+0.0%) | +| Cross-gen writes | 0.190 | 0.180 (−5.3%) | 0.180 (−5.3%) | 0.190 (+0.0%) | +| **Dead after deep stack** | 0.430 | 0.410 (−4.7%) | **0.410 (−4.7%)** | 0.410 (−4.7%) | +| Closure capture | 0.140 | 0.130 (−7.1%) | 0.130 (−7.1%) | 0.130 (−7.1%) | +| String retention | 0.110 | 0.110 (+0.0%) | 0.120 (+9.1%, one quantum) | 0.120 | +| Array grow/evacuate | 0.150 | 0.150 (+0.0%) | 0.150 (+0.0%) | 0.150 (+0.0%) | +| Map/set side tables | 0.430 | 0.430 (+0.0%) | 0.430 (+0.0%) | 0.430 (+0.0%) | + +Geometric means vs shadow: plain maps −2.83%, statepoints −1.10%, +contract −0.43%. + +**The deep-stack weakness is closed.** The probe that was ~20% slower on +the loaded host is now 4.7% *faster* than shadow, and the attribution is +exact: the same statepoint binary with `PERRY_STACKMAP_WALKER=unwind` runs +at 0.430 — precise shadow parity — so the x29-chain walker is the entire +difference. + +Max RSS: every cell within ±0.8% of shadow (ratchet-comparable platform). +Uncached `batch.ts` compile: shadow 0.590 s, statepoints 0.570 s (−3.4%) — +the loaded-host "+5.3% slower to compile" claim did not survive quiet +measurement and is withdrawn. + +Metadata (`__llvm_stackmaps`, summed over the eight probes): plain maps +42,936 B, statepoints 81,104 B (1.89×), contract 73,952 B (−8.8% vs +statepoint). Generated `__text` is ~5.8 KB smaller across the eight +binaries in the native arms (probe code is small; the shadow-stack text +delta scales with generated code, per #7108's 13.3% on a real app). + +Standing conclusion after this matrix: on wall-clock, RSS, and compile +time, statepoints are at worst tied with the shadow stack on this +hardware; metadata remains the only losing axis, and it is the axis +repsel promotion shrinks. Small-hardware and Linux numbers still require +the ELF scanner port. + +## Post-matrix follow-through (2026-08-01, `897e0f53b`) + +Two changes landed after the matrix, both gate-verified (16/16 forced +evacuation + walker-verify, strict-enforcement gate fires, RSS flat): + +1. **The plain-map user mode is deleted** per the GC knob kill-policy: + after the quiet matrix it was a losing configuration (statepoints match + it within timer quantization), and it is structurally unsound — LLVM's + stackmap intrinsic can record a root slot's address as `Register R#N`, + caller-saved and unrecoverable at collection time, so those roots are + invisible to the collector by construction. The lowering survives only + as statepoint mode's internal `try`/setjmp fallback; shrinking that + fallback set is the remaining correctness work for the backend. +2. **Noreturn call sites carry no metadata** (`GcCallEffect::NeverReturns`): + every `js_throw*` helper funnels into `exception::js_throw` (`-> !`), so + control never returns, no relocation is ever consumed, and the frame's + roots are dead past the call. Sound in any mode; deeper frames carry + their own records. + +Metadata trajectory on `batch.ts` statepoints, all without any +representation-selection improvement: 442 (first prototype) → 217 +(call-effect audit) → 198 (noreturn elision) → 181 (contract) → **172 after +the second audit round — −61% total**. The remaining big step is the property-access diamonds +(~85 sites), which fall to repsel `Ptr`. + +## The compact per-function experiment — a measured NEGATIVE result + +The per-function precision model was then built and disproven +(implementation at `bd066d62b`, deleted from the tip afterward — an unsound +mode must not survive as a configuration a future bisect will trust). + +**The thesis**: one entry stackmap per generated function recording every +root alloca as a stable Direct location; calls carry only memory barriers; +a `__perry_gen_end` sentinel object linked last bounds the generated region +so the runtime can match frames by region instead of per-safepoint PCs. +**The size result was real**: 424–680 B of metadata per probe binary versus +5.3–8.9 KB for statepoints — 10–13× — with `__text` mostly smaller too. + +**The correctness result kills it.** A ten-line churn loop +(object escapes into a ring, two field reads) deterministically computes +wrong values. The forensics chain, recorded because each step eliminated a +plausible-but-wrong theory: retention-clear lowering (no effect), +callee-saved register clobbers at barriers (no effect, bit-identical +failure), dead-slot zeroing before every GC-capable call (no effect, +bit-identical), and finally disabling the walker's visits entirely — +**still bit-identical corruption**, proving the stack-map machinery was +never the vector. The corrupted fields contain forwarding-stub and +header-age-bit patterns: the mutator reads from-space through a stale +pointer that lives in optimized SSA, not in any root slot. The same module +compiles to **79 `gc.relocate`s** under the statepoint backend — each one a +place where LLVM held a heap-derived value whose post-collection identity +only relocation semantics can restore. `asm "~{memory}"` constrains memory +ordering, not dataflow; no barrier discipline reaches values the optimizer +carries in registers and rematerializes. + +## Real-app remeasurement (test-drizzle-pg, 133 modules, 2026-08-01) + +#7108's size model, re-taken as a direct measurement on the same +application with every in-branch reduction live: + +| Arm | file | `__text` | `__llvm_stackmaps` | +|---|---:|---:|---:| +| shadow (default) | 28,474,576 | 20,376,748 | 0 | +| statepoint | 32,206,720 | 20,227,252 | 4,025,336 | +| statepoint + contract | 32,008,560 | 20,226,728 | 3,832,384 | +| + second audit round | 31,925,984 | 20,223,400 | **3,757,520** | + +Two model corrections, one in each direction. The metadata came in at +**3.83 MB — below the refined model band's 4.5 MB floor** (the audit, +noreturn elision, and contract compose better on real dependency code than +the all-roots-live worst case assumed). But the text actually recovered is +**150 KB, not the 439 KB** #7108 reported — that figure measured +`PERRY_SHADOW_STACK=0` (rooting fully off) as the floor, while real +statepoint codegen keeps spill/reload work. Net file-size cost of the best +native arm on a real app: **+3.53 MB (+12.4%) versus shadow — a ~25× +imbalance that no audited elision closes.** The contract's real-app effect +is −4.8% metadata (probe-scale was −8.8%; dependency code has +proportionally fewer audited-helper sites). + +**Verdict as of 2026-08-01 (superseded below):** the shadow stack was the +three-axis optimum — wall-clock tied within timer quantization, RSS tied, +file-size won by 3.5 MB on a real application. The statepoint backend was +correctness-superior (the forgot-to-root class is structurally impossible), +speed-competitive, and 59% leaner in metadata than its own first prototype, +but carried a 25× metadata imbalance that no audited elision closed. + +That conclusion assumed the metadata's *content* was the cost. It was not. + +## The file-size axis was the wire format, not the roots (2026-08-03) + +Re-examined with `scripts/stackmap_anatomy.py`, which breaks the section +down by structural component and asserts it parsed 100% of the bytes. + +**First correction: generated code is already smaller under statepoints.** +On `test-drizzle-pg` the RS4GC arm's `__text` is 20,128,708 against shadow's +20,376,748 — **248 KB better**. The entire loss is `__llvm_stackmaps`. + +**Second correction: most of that section is provably dead weight.** + +| component | share of section | +|---|---:| +| `Constant` location slots (3 per record: CC / Flags / NumDeopt) | 40.6% | +| duplicate base/derived location slots | 13.3% | +| record headers (incl. an 8-byte patchpoint ID never patched) | 18.0% | +| inter-record padding | 11.3% | + +`stack_maps.rs` **already discarded the constants and collapsed the +base/derived pair at parse time**. Over half the section was shipped in the +binary and thrown away at startup — redundancy, not a tradeoff. LLVM's +stack map is a JIT-patching wire format; an AOT collector needs +`{dwarf_reg, offset}` per distinct root and nothing else. + +**Compaction measured on drizzle** (4,214,384 B, 124 concatenated maps, +1,717 functions, 33,406 records, 154,020 distinct roots): + +| encoding | bytes | ratio | +|---|---:|---:| +| flat varint (drop constants + duplicate pairs) | 387,199 | 10.9× | +| + roots sorted and delta-encoded | 286,258 | 14.7× | +| + "same live set as previous record" flag | 132,418 | 31.8× | +| **shipped**: as above, but offsets fixed-width | **224,832** | **18.7×** | + +The third row is the big one and it is a fact about real programs, not a +coding trick: **77% of records have exactly the live set of the record +before them**, because consecutive safepoints in a function share their +roots. That same fact shrinks the in-memory index — the decoder points +repeats at one copy instead of materialising 154k entries — so it is an RSS +win as well as a file-size one. + +The fourth row is what actually ships, and the difference is a constraint +rather than a choice: **at `-O3` LLVM emits each record's instruction offset +as a label difference** (`.long Ltmp9-_main`) that only the assembler can +evaluate, so those offsets cannot be delta-varint-encoded at rewrite time. +They go in a fixed-width `u32` array instead, costing ~4 bytes per record. +Recovering the last 92 KB would mean assembling twice — once to learn the +numbers the assembler just computed, once to emit them — which is more +machinery than the bytes are worth. + +### Measured, not projected (2026-08-03) + +Built with one compiler, identical flags, and a **clean object cache per +arm** (a clean-cache rebuild reproduced the cached shadow figure to within +8 bytes, so nothing here is a stale-artifact reading): + +| arm | total | `__text` | `__perry_gcmap` | vs shadow | +|---|---:|---:|---:|---:| +| shadow (default) | 28,737,536 | 20,646,900 | 0 | — | +| statepoint + compact | 28,688,464 | 20,497,296 | 227,275 | **−49,072** | +| RS4GC + compact | 28,605,912 | 20,409,232 | 224,126 | **−131,624** | + +The emitted map came in at 227,275 B against the 224,832 B the encoder model +predicted — within 1%. Metadata fell from 4,214,384 B to 227,275 B (18.5×), +and `__text` is 149,604 B smaller than shadow's on the same build. + +**The file-size axis is flipped.** The statepoint backend now leads on +**all three axes** — wall-clock −0.93%, RSS flat, and size −131,624 B on the +RS4GC arm — where it previously lost size by 3.5 MB. + +Both arms pass the full gate: 8/8 probes byte-match the pinned Node oracle +normally *and* under `PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 +PERRY_STACKMAP_WALKER=verify`, which is the check that can actually fail if +the new format decoded to a smaller root set — a map that lost roots would +corrupt the heap under forced evacuation rather than merely print something +different. The gate also asserts its subject was live: `__llvm_stackmaps` +absent **and** `__perry_gcmap` non-empty, before any output is compared. +Its first run correctly reported 0/8 because the rewrite had silently not +run at all. + +**Compile-time cost, stated honestly:** 11.95s vs 10.43s for the whole +application (+14.6%), covering statepoint lowering plus the assembly round +trip. That is not one of the three axes being optimised, and it buys the +axis that was losing. + +### Why the rewrite happens on assembly + +`clang -S` prints the stack map as ordinary directives with the function +addresses as **symbol names in plain text** (`.quad _main`), so one text +parser replaces Mach-O *and* ELF relocation parsing, `llvm-objcopy`, and a +second link pass. Two facts settled this empirically rather than by taste: + +- the stack map's function-address fields are **external symbol + relocations** (`otool -r`: `extern 1` at offsets 16 and 40), so a + separately assembled table can reference them by name and the linker + resolves it — no relink needed; +- `-S` costs nothing: it takes the **same 0.04s as `-c`** (codegen is the + cost, printing text is free), and `llvm-mc` assembles in 0.02s. Use + `llvm-mc`, not `clang -c file.s` — the latter is 0.13s of driver overhead. + +### Two traps that cost time here + +- **On-disk stack-map addresses look like garbage** (`0x00300000000008b0`) + because they are **dyld chained-fixup entries**, not addresses: bits 0-35 + are the target, bits 51-62 the chain delta to the next fixup. dyld + resolves them at load. Do not conclude records are mismatching from an + on-disk read. +- **The section is a concatenation of one map per object file**, not a + single map. A parser that reads only the first header silently + under-counts, and nothing downstream notices. Assert that parse coverage + equals the section size. +- **`.no_dead_strip` names the block's label from outside the block.** + Removing the label without retargeting that directive leaves an undefined + symbol — and the directive is also the only thing keeping a section + nothing references from being stripped, which would leave the collector + with no roots at all. + +### The transfer question, also measured: can the audit shrink the SHADOW stack? + +The audited call-effect facts apply to shadow bookkeeping in principle — a +function whose every call is provably non-collecting needs no shadow frame +at all, by the same soundness argument the statepoint elisions passed gates +with. Census on the real app's traced IR (instrument validated against +#7108's function totals): **118 of 1,535 shadow-framed functions qualify +(7.7%), covering only 4.0% of shadow-op IR lines.** The elidable functions +are small leaves; the cost lives in large functions with genuine collecting +calls. Whole-frame elision would recover well under 1% of generated text — +recorded here as measured-and-not-pursued (finer per-region elision is +complexity the win does not justify). The shadow stack's remaining text +cost is, as the repsel campaign already measured from the other side, +bookkeeping for values that cannot yet be proven non-pointers — one more +place every road converges on representation selection. + +## RS4GC pipeline slice (#7174) — running, fully gated, 2026-08-01 + +`PERRY_RS4GC=1` exists and passes the complete gate matrix: 8/8 probes +under forced evacuation + verification and 8/8 under walker-verify, with +`RewriteStatepointsForGC` inserting every statepoint, relocation, and +downstream-use rewrite over surgically-retyped `ptr addrspace(1)` root +SSA. Requirements established empirically: + +- **Version-matched toolchain**: LLVM 22 `opt` output is unparseable by + Apple clang 21 — `PERRY_LLVM_CLANG` must point at the same LLVM's clang. +- **Statepoint placement must precede cast-merging optimization.** + `default` before RS4GC fails 3/8 probes: GVN/CSE merges the per-site + `ptrtoint`/`bitcast` chains across future statepoint sites, recreating + the stale-double hazard. `function(mem2reg)` alone is the sound + pre-pass; clang's full optimization AFTER statepoint insertion is safe + by construction. This is the same design law again, now stated + positively: relocation semantics must be present before the optimizer + is allowed to move heap-derived values. +- **Two vacuous-green runs preceded the real one**: the fail-closed + surgery bail (recognizer knew only the unit-test `alloca i64` idiom; + real roots are `alloca double`) silently routed every function to the + explicit bridge. Caught by record-count comparison (200 vs 55), not by + any probe — assert surgery liveness before believing an RS4GC A/B. + +Probe-scale metadata trails the bridge (probe 01: 8,072 B vs 5,320 B — +the zero-live statepoint record constant dominates small functions), but +the real-app measurement flips the hierarchy: on `test-drizzle-pg`, +RS4GC `__text` is 20,128,708 — **248 KB below shadow and 98 KB below the +explicit bridge** — and metadata is 3,875,416 B, within 3.1% of the +audited bridge's 3,757,520. Total file: 31,957,792, the smallest native +arm measured. SSA liveness pruning compensates for the record constant at +scale exactly as predicted. Leaf-attribute transfer verified by record +count (200 → 103 on probe 01). Runtime (8 probes × 9 interleaved reps, loaded host, arms share load — +directional): shadow 218.1 ms geo-mean, bridge+contract 216.3 ms (−0.83%), +RS4GC 216.1 ms (−0.93%) — the fastest arm measured. Max RSS: flat with +shadow on the four churn-heaviest probes (27/27, 36/36, 37/38, 25/25 MB). +RS4GC is therefore already the preferred +native backend on every axis measured except probe-scale metadata, while +carrying the structural correctness model — the explicit bridge becomes a +deletion candidate once RS4GC grows has_try coverage and a leaner +zero-live-record story (upstream pass option recorded on #7174). + +## The repsel-erasure projection, measured — and corrected + +Both this campaign and the representation-selection plan share the +assumption that repsel promotion erases native-root metadata ("each value +proven non-pointer deletes its records"; the repsel plan itself notes "the +GC currency was not measured at all"). First measurement, using the #7133 +knob-scoping fixes: `batch.ts` under statepoints with all landed +promotions on versus `PERRY_CANONICAL_{I32,U32,STR}_LOCALS=0` is +**byte-identical** — 24,752 B of metadata, 198 statepoints, 363 +relocations, 33 root slots, unchanged. + +The correction this forces: the landed promotion classes remove *calls +and guards* (the performance currency), but they promote values the +rooter's type analysis already classified non-pointer — so they delete +zero roots. The metadata-erasure currency is paid only by promotions in +the maybe-pointer population: untyped locals, temporaries, and dependency +JS — exactly where repsel coverage is currently weakest (the +`__esModule` barrier, minified slot reuse). The "repsel erases the 25× +gap" projection therefore needs either the Track E/F work (types made +load-bearing, dependency-JS recovery) or `Ptr`-class promotions +feeding a typed-slot story, not wider scalar coverage. Recorded so +neither campaign builds on the uncorrected assumption. + +## Linux verification and the small-hardware numbers (#7173, 2026-08-01) + +Runtime verification on two granted Linux hosts, cross-built from macOS +(perry `--target linux`/`linux-aarch64` `--no-link` for ELF probe objects; +`cargo zigbuild` archives with `-Cforce-frame-pointers=yes`; `zig cc` link +with `-lunwind`): + +- **x86-64 (Ubuntu, idle prod webserver): 8/8** forced-evacuation probes + byte-matched to the pinned oracles, first run — first execution of the + ELF section discovery and Linux unwinder path. +- **aarch64 (Raspberry Pi 5): 8/8 after one caught defect.** The + verify-walker mode fired exactly as designed: the Darwin SP + reconstruction (`SP = FP + 16 − stack_size`) is wrong on aarch64-Linux + (frame pair at the bottom, not the top) — fast walk and unwinder + disagreed by the layout delta on one slot. Fix: SP-relative locations + disqualify the fast chain off-Darwin — and the follow-up disassembly + proved this permanent, not provisional: generated prologues set + `x29 = sp + 0x30` and `x29 = sp + 0x60` for stack sizes 128 and 192 — + the offset varies per function with the callee-save area below the pair, + so no `(FP, stack_size)` formula exists on aarch64-Linux. The unwinder + is the sound Linux path; a Linux fast chain requires FP-relative spills + from LLVM (upstream) or per-function side metadata (the compact-section + ghost). A silent-fallback foot-gun was also found: + an unrecognized `--target` value compiles for HOST (Mach-O out of + `linux-arm64`); the accepted spelling is `linux-aarch64`. + +**Small-hardware timing (Pi 5, load ≤0.1, 9 interleaved reps)**: shadow +469.2 ms geo-mean, statepoints 538.2 ms (**+14.7%**; deep-stack +23%, +string-retention +35%, array-grow +32%). The M1 parity does NOT transfer +to narrow cores. + +**Decomposition — the delta is COLLECTOR-SIDE, and specifically the +unwinder.** Re-running every probe with collections suppressed +(`PERRY_GC_HEAP_LIMIT` beyond the workload) leaves the deltas essentially +unchanged (string-retention +35.0% suppressed vs +35.7% normal; +deep-stack +22.9% vs +22.8%), and `PERRY_GC_DIAG` shows *identical cycle +counts per probe across arms* — so it is neither mutator codegen cost nor +collection-frequency skew. Wait: suppression leaving the delta intact +would normally implicate the mutator — but the `perf` profiles resolve +it. The statepoint arm's top symbols are dominated by +`libunwind::CFI_Parser::parseCIE`, `getEncodedP`, `getULEB128`, +`findFDE` (8.7% + 6.0% + 4.1% + 3.0% on string-retention alone); the +shadow arm has none. The GC still runs its fixed cycle count under +suppression (the limit raises the trigger, it does not disable the +collector), and every one of those cycles walks the stack with the +platform unwinder because the fast chain is disqualified on Linux. **The +measured cost is DWARF CFI parsing per collection, not the statepoint +model.** + +That is a configuration cost with two known remedies (an indexed +walker, or the Linux fast chain via upstream FP-relative spills), and it +means the Pi number must NOT be read as "statepoints are 15% slower on +small hardware." An attempt to confirm by relinking against libgcc's +unwinder instead of zig's bundled libunwind produced segfaulting +binaries (a hand-rolled link line missing the working recipe's flags) — +the implementation-vs-model split is therefore *measured to be unwinder +walking* but the specific unwinder's contribution remains unquantified. + +Consequence for the campaign verdict: shadow retains three-axis +optimality including small hardware, and a future default-flip needs a +Pi-class gate — but the gap's cause is a named, fixable walker cost. + +## Real-application scale (Claude Code 2.1.112, 2026-08-01) + +The campaign's origin target — the real 13 MB minified `@anthropic-ai/claude-code` +bundle — compiles and runs natively under the shadow stack (204,103,064 B +binary, `__text` 149.4 MB, 115.1 MB RSS, `--version` correct, 82 min). + +**The explicit statepoint bridge cannot compile it**, and the reason is a +scaling defect worth recording precisely: + +- Statepoint lowering roughly doubles module IR: **1,083 MB, 16,748 + functions (~66 KB/fn)** for a 13 MB input. +- `clang -c` rejects the oversized unit outright: *"file … is too large for + Clang to process."* +- **Adding codegen units does not fix it.** `decide_codegen_units` sizes by + *callable count* (`ceil(fns / 6000)`), never by IR bytes; and + `render_codegen_units` replicates **all shared string constants and + globals into every unit**. At `PERRY_CODEGEN_UNITS=16` each unit still + rendered ~370–436 MB, and unit 10/16 failed the same way — while total + emitted IR ballooned past 6 GB (which also exhausted the disk mid-run + and killed an earlier attempt). + +Two independent fixes fall out, both mode-agnostic wins: size codegen units +by estimated IR bytes rather than callable count, and emit shared +strings/globals **once** with external declarations in sibling units +instead of replicating them. The second is what makes splitting actually +scale for string-heavy minified bundles. + +Fairness note for anyone extending this: unit count changes cross-unit +inlining scope, so a statepoint arm forced to N units must be compared +against a shadow arm at the *same* N, not against the auto-chosen count. +The `-Os` downgrade (`module IR > 6 MB`) is NOT a confound — it applies to +both arms, since shadow IR for the same program cannot be smaller than the +statepoint arm's 1,083 MB. + +## Pi 5 re-measurement after the prologue-decode walker + main rebase (2026-08-02) + +The small-hardware regression is **closed and inverted**. Same host (Pi 5, +aarch64 Linux, load <1), same method (9 interleaved reps, per-probe +medians), both arms cross-built from one tree and one runtime archive: + +| Probe | before: shadow / statepoint | after: shadow / statepoint | +|---|---:|---:| +| Nursery churn | 385.7 / 402.0 (+4.2%) | 392.4 / 391.9 (−0.1%) | +| Survivor promotion | 448.4 / 446.7 (−0.4%) | 286.1 / 277.1 (−3.1%) | +| Cross-gen writes | 428.4 / 461.0 (+7.6%) | 374.8 / 367.4 (−2.0%) | +| Dead after deep stack | 932.1 / 1145.0 (+22.8%) | 1039.2 / 1017.6 (−2.1%) | +| Closure capture | 334.6 / 361.6 (+8.0%) | 449.8 / 436.3 (−3.0%) | +| String retention | 275.7 / 374.1 (+35.7%) | 240.9 / 240.5 (−0.2%) | +| Array grow/evacuate | 362.8 / 483.1 (+33.1%) | 226.9 / 225.6 (−0.6%) | +| Map/set side tables | 978.9 / 1139.9 (+16.4%) | 1070.3 / 1040.8 (−2.8%) | +| **geometric mean** | **+14.72%** | **−1.74%** | + +Correctness first, as always: 8/8 under forced evacuation + verification +and 8/8 under `PERRY_STACKMAP_WALKER=verify` (fast x29 walk and the +platform unwinder visit the identical slot set) — the prologue-decoded SP +is right on the architecture where the constant-based approach was proven +impossible. + +**Attribution, stated honestly: the walker is NOT the cause of the +improvement.** A direct A/B on the two worst probes — same binary, fast +chain versus `PERRY_STACKMAP_WALKER=unwind` — is a dead heat (0.24 s vs +0.24 s; 1.01 s vs 1.01 s). The DWARF CFI parsing that `perf` measured at +~22% of samples is simply no longer hot. The other variable between the +two runs is the rebase onto main's 64 commits of GC work (root-store +dominance #7192, from-space protection and zeal #7196, and #7148's precise +safepoint drains replacing conservative-scan fallbacks), which plausibly +reduced how often the native stack is walked at all. Shadow itself got +faster on the same probes (469.2 → 429.2 ms geo), which is consistent with +that explanation and inconsistent with "the walker fixed it". + +So: the prologue decode is *correct and verified* and removes a real +fallback, but the measured win belongs to main's GC hardening. Both are +recorded rather than conflated. + +★ One instrument failure worth recording: a first attempt to count GC +cycles reported **0 cycles for both arms**, which would have made the whole +comparison vacuous. It was a grep pattern that no longer matched main's +changed diagnostic format — raw output shows 81.9 MB freed across 79 arena +blocks. Never trust a count without looking at what produced it. + +**Conclusion, stated as the design law this branch keeps re-deriving:** +*with an optimizing compiler between the source and the safepoint, root +metadata without relocation semantics is unsound — per-call plain maps +merely made the window small enough for probes to pass, and per-function +compact maps made it wide enough to fail in ten lines.* This upgrades +#7108's argument ("only statepoint describes the frame during the call") +from analysis to demonstration, and it means the metadata floor for a +sound non-statepoint scheme does not exist: the choice is statepoint-style +relocation (per-safepoint records, ~2× plain maps, the measured −59% +trajectory) or the shadow stack. The compact 10–13× is only reachable via +`RewriteStatepointsForGC`-style managed-pointer SSA — the toolchain +decision #7108 costed — or repsel shrinking the recorded set. + +## Which statepoint design this tests + +This is the explicit bridge, not LLVM's `RewriteStatepointsForGC` pipeline. +Perry emits the three intrinsics directly: + +1. Load each live NaN-boxed `i64` root from its existing native alloca and + temporarily convert the bits to `ptr addrspace(1)`. +2. Replace the original call with `llvm.experimental.gc.statepoint`. +3. Recover the call's scalar return through `llvm.experimental.gc.result`. +4. Recover every live root through `llvm.experimental.gc.relocate`, convert it + back to `i64`, and store it to the original alloca. + +The runtime collector executes inside the statepoint's callee. It unwinds to +the generated caller, finds LLVM's `Indirect` spill locations in +`__LLVM_STACKMAPS`, and rewrites those words during evacuation. The generated +caller then reloads the rewritten words through `gc.relocate`. + +The small standalone version is +[`statepoint-bridge-probe.ll`](statepoint-bridge-probe.ll). + +This choice intentionally avoids colliding with the representation experiment. +A full `RewriteStatepointsForGC` integration wants managed pointers to be +identifiable throughout SSA and expects a compiler pass to discover and +rewrite safepoints. Perry currently carries GC-capable values as NaN-boxed +`i64` words, so the bridge changes their representation only across one call. + +## Why LLVM calls it experimental + +The `llvm.experimental.*` prefix means LLVM does not promise a permanently +stable IR or binary interface across releases. It does not mean that the +mechanism is an abandoned toy or that production-oriented runtimes cannot use +it. For Perry it creates an engineering requirement: pin and test supported +LLVM versions, verify emitted IR, and treat upgrades as an ABI migration. + +The relevant upstream contracts are +[Garbage collection safepoints](https://llvm.org/docs/Statepoints.html) and +[Stack maps and patch points](https://llvm.org/docs/StackMaps.html). + +## Implementation + +The prototype reuses the precise-root discovery and conservative per-call CFG +liveness built for the plain-stack-map experiment. + +- Functions with roots receive `gc "statepoint-example"`. +- Ordinary direct calls with scalar arguments and scalar/void results are + rewritten explicitly. +- LLVM intrinsics and compiler-only inline assembly are not safepoints. +- Unsupported call forms retain the plain `llvm.experimental.stackmap` + fallback. +- A function containing Perry's setjmp-based `try` lowering uses the plain-map + backend for the whole function. +- The runtime parser accepts plain-map `Direct` alloca addresses and statepoint + `Indirect` spill locations. It deduplicates identical base/derived locations + before visiting roots. +- The module retains each Mach-O `__LLVM_STACKMAPS` atom with + `.no_dead_strip`. +- `PERRY_STATEPOINTS` participates in both build and object cache keys. + +When `PERRY_STATEPOINTS=1` and `PERRY_STACK_MAPS=1` are both present, +statepoints take precedence in eligible functions. + +## Correctness and coverage + +Final release artifacts passed: + +- all 8 GC-ratchet probes with stdout identical to Node; +- all 8 probes with `PERRY_GC_FORCE_EVACUATE=1` and + `PERRY_GC_VERIFY_EVACUATION=1`; +- LLVM 22 verification of all eight generated modules; +- compilation of the minimal bridge with Apple clang 21.0.0 and LLVM 22.1.4; +- 314 `perry-codegen` library tests; +- 5 focused runtime stack-map/statepoint parser and call-site tests; +- the object-cache statepoint environment-key test. + +Final generated-IR coverage: + +| Probe | Statepoints | Relocations | Plain fallbacks | +|---|---:|---:|---:| +| Nursery churn | 152 | 227 | 0 | +| Survivor promotion | 165 | 296 | 0 | +| Cross-generation writes | 168 | 244 | 0 | +| Dead after deep stack | 119 | 135 | 0 | +| Closure capture | 146 | 191 | 0 | +| String retention | 92 | 95 | 0 | +| Array grow/evacuate | 100 | 100 | 0 | +| Map/set side tables | 138 | 274 | 0 | +| **Total** | **1,080** | **1,562** | **0** | + +The zero here describes these probes, not the backend's complete call-form +coverage. Indirect calls, aggregate signatures, unusual call-site attributes, +and setjmp functions can still take the deliberate plain-map fallback. + +Retained heap and heap capacity match the shadow-stack arm byte-for-byte. +Promotions, freed bytes, and cycle counts also match. Two probes have tiny +copy-accounting differences (+3 objects/+208 bytes and -152 bytes), with +identical final retention; the other six match all checked GC counters. + +One apparent intermittent relocation failure during development was a stale +`target/*/libperry_runtime.a`. Perry executables link the +`perry-runtime-static` package, not the `perry-runtime` rlib directly. +Rebuilding only the latter left the old scanner in generated binaries. The +final results rebuild both the compiler and static runtime archive. + +## Performance + +Hardware was an Apple M1 Max on macOS 26.5. The interleaved run reported load +averages of 22.71/25.83/27.77, so these results are directional and should not +be promoted to release claims. + +Each runtime cell is the median of 11 executions. Mode order was interleaved +and rotated after one warmup, and outputs were checked for equality before +timing. + +| Probe | Shadow | Plain stack map | Statepoint | Statepoint vs plain | +|---|---:|---:|---:|---:| +| Nursery churn | 182.026 ms | 177.968 ms | 175.573 ms | -1.35% | +| Survivor promotion | 216.247 ms | 208.667 ms | 208.192 ms | -0.23% | +| Cross-generation writes | 210.743 ms | 203.775 ms | 204.504 ms | +0.36% | +| Dead after deep stack | 460.249 ms | 470.665 ms | 501.690 ms | +6.59% | +| Closure capture | 173.719 ms | 163.076 ms | 162.902 ms | -0.11% | +| String retention | 130.773 ms | 133.166 ms | 136.794 ms | +2.72% | +| Array grow/evacuate | 171.915 ms | 175.039 ms | 172.519 ms | -1.44% | +| Map/set side tables | 460.879 ms | 444.031 ms | 466.627 ms | +5.09% | + +Geometric means versus shadow: + +- plain stack maps: -1.66%; +- statepoints: -0.27%; +- statepoints versus plain maps: +1.42%. + +The deep-stack result is the strongest negative signal. Both native-stack +backends pay for unwinding, while statepoints additionally materialize +relocation spill/reload state around a large number of calls. + +Three uncached compilations per probe measured a +1.47% geometric mean for +plain maps and +2.12% for statepoints versus shadow. Sequential RSS +measurements put statepoints at roughly +1.03% median RSS and +0.70% peak RSS, +but allocator and host noise make those figures less reliable than retained +heap. + +Statepoint metadata is materially larger: + +| Probe | Plain payload | Statepoint payload | +|---|---:|---:| +| Nursery churn | 7,104 B | 13,656 B | +| Survivor promotion | 8,224 B | 16,112 B | +| Cross-generation writes | 7,768 B | 15,040 B | +| Dead after deep stack | 5,128 B | 14,824 B | +| Closure capture | 9,464 B | 18,440 B | +| String retention | 6,016 B | 10,912 B | +| Array grow/evacuate | 5,840 B | 11,480 B | +| Map/set side tables | 7,288 B | 13,840 B | + +The total is 114,304 bytes versus 56,832 bytes, or 2.01x. Most executables are +about 16 KiB larger than shadow after Mach-O segment rounding; closure capture +crosses another segment boundary and is about 33 KiB larger. + +## Is the GC simpler? + +Relocation is simpler to reason about, but the GC system is not simpler yet. + +The improvement is real: a statepoint makes the original call result and every +post-call root explicit SSA results. Plain maps required empty inline assembly +memory barriers to stop LLVM from caching root values across a call whose +stack slots the compiler could not know the collector mutates. + +However, this bridge still needs: + +- Perry's root discovery, logical slots, and conservative CFG liveness; +- addressable root allocas and per-statepoint load/store bridges; +- the LLVM stack-map v3 parser and native unwinder; +- Mach-O section discovery and linker-retention directives; +- call-form parsing and plain-map fallbacks; +- target- and toolchain-specific verification. + +It removes generated shadow-frame push/pop and TLS slot mutation, but replaces +that local machinery with a wider compiler/linker/runtime contract. The +collector itself is nearly unchanged; only its root source changes. + +## Is Perry better positioned? + +Semantically, yes. Operationally, not enough yet to switch. + +Statepoints provide the right vocabulary for a future moving collector: +relocation is explicit, base/derived relationships have a representation, and +a later managed-pointer pipeline can keep roots in SSA rather than forcing +Perry to invent compiler barriers. + +This implementation is still a bridge with important liabilities: + +- arbitrary NaN-boxed bits temporarily masquerade as managed pointers; +- all ordinary calls with live roots are treated as potentially allocating; +- indirect and unusual calls fall back to plain maps; +- `try`/setjmp functions fall back wholesale; +- scanning is macOS/Mach-O-only; +- active-frame matching still uses a 16-byte nearest-PC tolerance; +- the intrinsic and metadata contracts require LLVM-version discipline; +- the current measurements show no all-around speedup. + +## Recommended next step + +Do not replace the shadow stack from this branch. Preserve the prototype as +evidence and wait for the representation work before choosing the production +path. + +After that work lands: + +1. Define a safepoint-capability table so only calls that can enter the + allocator become statepoints. +2. Represent genuine managed references directly instead of converting every + possible NaN-box root through `inttoptr`. +3. Compare direct explicit emission with `RewriteStatepointsForGC` on that + representation. +4. Remove plain-map fallbacks one call form at a time and fail closed on + unsupported targets. +5. Re-run the 11-way interleaved suite on an idle pinned host, with separate + profiles for mutator root maintenance, relocation reloads, unwinding, and + collector root scanning. + +## invoke-EH lands on main; try functions covered (2026-08-03) + +main replaced setjmp/longjmp exception lowering with `invoke`/`landingpad` +(#7302, PR #7305) and deleted `volatile_setjmp.rs` and `setjmp_abi.rs`. That +retires this branch's correctness blocker: a `longjmp` could jump past a +`gc.relocate`, which is why try-carrying functions were excluded from +statepoints and routed to the plain-stack-map lowering — itself unsound, since +LLVM may record a root slot's address in a caller-saved register that cannot +be recovered at collection time. + +The exclusion was not merely obsolete but **unrepresentable**: main deleted the +`has_try` field, so the compiler forced its removal. + +### The probe that had no equivalent + +Nothing in the suite contained a `try` — 0 of 8 probes — so the newly covered +case was exercised by nothing at all, and a green run said nothing about it. +`09_try_catch_roots.ts` closes that: objects allocated inside a `try` surviving +a collection inside the same `try`; locals live across a throw and read in the +`catch`; a throw crossing several frames so the rewritten roots sit in a +caller's frame; `finally` on both edges; and a rethrow caught one frame up. +Every survivor folds into the checksum, so a lost or stale root is a wrong +number rather than a crash. + +Its map is 1,116 bytes — the largest of any probe — which is the liveness +evidence that try-carrying functions now genuinely carry statepoint records. + +### Result, and a real limitation it exposed + +**Explicit statepoint bridge: 9/9**, byte-matching the pinned Node oracle +normally and under `PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 +PERRY_STACKMAP_WALKER=verify`. + +**RS4GC: 8/9 — it cannot compile a try-carrying function.** The LLVM verifier +rejects the module: + +``` +%lpad = landingpad { ptr, i32 } +token %r2.0.relocated = call coldcc ptr addrspace(1) + @llvm.experimental.gc.relocate.p1({ ptr, i32 } %lpad, i32 1, i32 0) +``` + +`gc.relocate`'s first operand must be a `token`. RS4GC emitted the relocates on +the unwind edge against the landing pad's `{ ptr, i32 }` result, because +`statepoint-example` expects a statepoint-invoke's unwind destination to carry +`landingpad token` rather than the Itanium form `try_stmt.rs` emits. + +This matters for the size ranking: **RS4GC is the leanest arm measured +(−131,624 B versus shadow) but cannot handle `try` yet, so the arm that is +actually complete today is the explicit bridge at −49,072 B.** Both still beat +the shadow stack on size; the RS4GC/EH interaction is the remaining work, and +it is an LLVM-convention problem rather than anything the compact map touches. + +### Correction: the size win does not survive the merge with main (2026-08-03) + +Re-measured on `test-drizzle-pg` after merging main, one compiler, clean +object cache per arm: + +| arm | total | `__text` | gcmap | unwind | eh_frame | vs shadow | +|---|---:|---:|---:|---:|---:|---:| +| shadow | 26,950,504 | 19,801,644 | 0 | 195,104 | 1,479,652 | — | +| bridge | 26,951,000 | 19,650,408 | 189,454 | 187,056 | 1,375,028 | **+496** | +| RS4GC | 27,000,568 | 19,562,088 | 220,936 | 187,064 | 1,374,892 | **+50,064** | + +Pre-merge the same measurement gave −49,072 (bridge) and −131,624 (RS4GC). +Main's own changes shrank every arm by ~1.7–1.8 MB, but shrank **shadow about +50 KB more than the statepoint arms**, which is the entire swing. + +What did not change is the reason to keep the compact map: statepoints still +generate less code (`__text` −151 KB for the bridge, −240 KB for RS4GC, plus +~105 KB less `__eh_frame`). Those savings are simply now cancelled by the +189–221 KB of remaining metadata. Without compaction that metadata is 4.2 MB +and the arm loses by ~4 MB, so the 18–19× is doing real work — it converted a +3.5 MB loss into a tie, not into a win. + +**Honest standing on the three axes**, post-merge: + +* **performance** — statepoints ahead (−0.93% RS4GC, measured earlier); +* **RSS** — tied; +* **file size** — the explicit bridge is *tied* with shadow (+496 B, 0.002%); + RS4GC is 50 KB behind despite the smallest `__text`, because its live-set is + larger and so is its map. + +Closing the last axis therefore needs the root set to shrink, not the encoding: +221 KB of map for 154k roots is already near this format's floor. That is the +repsel-promotion lever the earlier projection named, and it is still the +outstanding work. diff --git a/scripts/stackmap_anatomy.py b/scripts/stackmap_anatomy.py new file mode 100644 index 0000000000..371241b9a3 --- /dev/null +++ b/scripts/stackmap_anatomy.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Break an LLVM stackmap v3 section down into where its bytes actually go. + +Answers the only question that matters for file size: which structural +component dominates, and how much of it is redundant for a collector that +has no interior pointers. + +Usage: stackmap_anatomy.py # finds the section itself + stackmap_anatomy.py --raw +""" +import struct +import subprocess +import sys + +LOC_KIND = {1: "Register", 2: "Direct", 3: "Indirect", 4: "Constant", 5: "ConstIndex"} + + +def extract_section(path): + """Pull __LLVM_STACKMAPS (Mach-O) or .llvm_stackmaps (ELF) out of a binary.""" + with open(path, "rb") as fh: + head = fh.read(4) + if head[:4] == b"\x7fELF": + out = subprocess.run( + ["readelf", "-x", ".llvm_stackmaps", path], + capture_output=True, text=True).stdout + data = bytearray() + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0].startswith("0x"): + for word in parts[1:5]: + if all(c in "0123456789abcdefABCDEF" for c in word) and len(word) % 2 == 0: + data += bytes.fromhex(word) + return bytes(data) + # Mach-O: locate section by (segment, section) and slice the file. + out = subprocess.run(["otool", "-l", path], capture_output=True, text=True).stdout + lines = out.splitlines() + for i, line in enumerate(lines): + if "__llvm_stackmaps" in line: + off = size = None + for probe in lines[i:i + 14]: + p = probe.split() + if len(p) == 2 and p[0] == "offset": + off = int(p[1], 0) + if len(p) == 2 and p[0] == "size": + size = int(p[1], 0) + if off is not None and size is not None: + with open(path, "rb") as fh: + fh.seek(off) + return fh.read(size) + raise SystemExit(f"no stackmap section found in {path}") + + +def analyze(buf): + ver, _, _ = struct.unpack_from("12,} {100.0 * val / total:5.1f}%") + print() + print(" location kinds:") + for key, val in sorted(kinds.items(), key=lambda kv: -kv[1]): + print(f" {str(key):<16} {val:>12,} {100.0 * val / max(n_locs,1):5.1f}%") + print() + print(f" duplicate location slots within a record: {dup_locs:,} " + f"({100.0 * dup_locs / max(n_locs,1):.1f}% of locations, " + f"{dup_locs * 12:,} B = {100.0 * dup_locs * 12 / total:.1f}% of section)") + print(f" constant-kind locations: {const_locs:,} " + f"({const_locs * 12:,} B = {100.0 * const_locs * 12 / total:.1f}% of section)") + + return kinds + + +def varint_len(value): + """Bytes a LEB128 encoding of `value` occupies. + + Rejects negatives rather than returning 1 for them: Python ints are + unbounded, so a negative slips straight past `>= 0x80` and every negative + frame offset would be counted as a single byte, understating the very + encoding this script claims to measure exactly. + """ + if value < 0: + raise ValueError(f"varint_len expects a non-negative value, got {value}") + n = 1 + while value >= 0x80: + value >>= 7 + n += 1 + return n + + +def compact_size(buf): + """Exact byte count of the encoding the runtime would actually consume. + + The runtime already discards Constant locations and dedups the base/derived + pair at parse time (`stack_maps.rs`), so it keeps only {dwarf_reg, offset} + per distinct root. This measures shipping precisely that: + + header 16 B + per function 16 B (u64 relocated address, u32 stack size, u32 records) + per record varint(delta instruction offset) + varint(root count) + per root varint(zigzag(offset) << 1 | reg_is_sp) + """ + _ver, _, _ = struct.unpack_from("> 31)) & 0xFFFFFFFF + size += varint_len((zig << 1) | (1 if reg == 31 else 0)) + roots_kept += len(seen) + if (pos - rec_start) % 8: + pos += 8 - ((pos - rec_start) % 8) + (nlive,) = struct.unpack_from(" len(buf): + break + maps.append(buf[pos:pos + extent]) + pos += extent + while pos % 8: + pos += 1 + return maps + + +if __name__ == "__main__": + args = sys.argv[1:] + raw = open(args[1], "rb").read() if args and args[0] == "--raw" \ + else extract_section(args[0]) + maps = split_maps(raw) + print(f"section {len(raw):,} B = {len(maps)} concatenated map(s)\n") + merged = {"total": 0, "compact": 0, "roots": 0} + for chunk in maps: + merged["total"] += len(chunk) + csize, croots = compact_size(chunk) + # Only the first map pays a format header in the merged encoding. + merged["compact"] += csize + merged["roots"] += croots + if len(maps) == 1: + analyze(maps[0]) + else: + # Aggregate composition across every map. + agg = {} + for chunk in maps: + nfunc, nconst, nrec = struct.unpack_from(" {len(raw) / max(merged['compact'],1):.1f}x smaller, " + f"saves {len(raw) - merged['compact']:,} B")