Native-frame GC roots via LLVM statepoints, opt-in (#7173, #7174)

.github/workflows/gc-native-roots.yml

@@ -0,0 +1,80 @@
+# #7173: native-root (statepoint) GC verification on Linux.
+#
+# Runs the gc-ratchet probe matrix in statepoint mode under forced
+# evacuation + evacuation verification, byte-diffed against the pinned Node
+# oracle, natively on the Linux runner — the same matrix the branch runs on
+# macOS, webserver-class x86-64, and the Pi 5. Two liveness asserts keep
+# this from being a gate that cannot fail (CLAUDE.md's four ways):
+# the binary must carry a .perry_gcmap section (and no .llvm_stackmaps,
+# proving the compact rewrite ran), and at least
+# one probe must report a copying collection.
+name: gc-native-roots
+on:
+  # Must run where it can actually gate something. Branch-scoped triggers were
+  # right while this lived only on exp/stackmap-viability; on main that same
+  # filter would mean the job never runs at all — CLAUDE.md's second way a gate
+  # cannot fail. Cancellation is deliberately NOT set here: a `main` run that
+  # gets cancelled by the next merge is the third way.
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  statepoint-linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .node-version
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: gc-native-roots
+      - name: Build compiler and static runtime (perry-dev profile)
+        run: |
+          export RUSTFLAGS="-Cforce-frame-pointers=yes"
+          cargo build --profile perry-dev -p perry -p perry-runtime-static -p perry-stdlib-static
+      - name: Probe matrix, statepoint mode, forced evacuation
+        run: |
+          set -euo pipefail
+          export PERRY_RUNTIME_DIR="$PWD/target/perry-dev"
+          export PERRY_NO_AUTO_OPTIMIZE=1
+          pass=0
+          total=0
+          errs=""
+          for probe in benchmarks/gc_ratchet/probes/*.ts; do
+            total=$((total+1))
+            name=$(basename "$probe" .ts)
+            node --expose-gc --experimental-strip-types "$probe" > "/tmp/$name.oracle"
+            PERRY_STATEPOINTS=1 ./target/perry-dev/perry "$probe" -o "/tmp/$name"
+            # Liveness assert 1: the subject must exist. The compact map
+            # replaced LLVM's section, so assert BOTH facts — the new section
+            # is present AND the old one is gone. Checking only the former
+            # would still pass if compaction silently stopped running.
+            readelf -S "/tmp/$name" | grep -q "\.perry_gcmap" \
+              || { echo "::error::$name has no .perry_gcmap section — statepoint mode was not live"; exit 1; }
+            readelf -S "/tmp/$name" | grep -q "\.llvm_stackmaps" \
+              && { echo "::error::$name still carries .llvm_stackmaps — the compact rewrite did not run"; exit 1; }
+            PERRY_STATEPOINTS=1 PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 \
+              "/tmp/$name" > "/tmp/$name.out" 2> "/tmp/$name.err"
+            diff "/tmp/$name.oracle" "/tmp/$name.out" \
+              || { echo "::error::$name output diverged from the pinned oracle"; exit 1; }
+            errs="$errs /tmp/$name.err"
+            pass=$((pass+1))
+          done
+          # Derived from the glob, not hardcoded: a literal goes stale the
+          # moment a probe is added (it did — 09_try_catch_roots), and if it is
+          # ever lowered to match it silently stops asserting full coverage.
+          echo "statepoint forced-evacuation matrix: $pass/$total"
+          [ "$total" -gt 0 ] \
+            || { echo "::error::no probes matched — the matrix ran on nothing"; exit 1; }
+          [ "$pass" -eq "$total" ]
+          # Liveness assert 2: at least one probe actually collected (gcmetric
+          # lines go to stderr). Collected during the loop rather than globbed
+          # as /tmp/0*.err, which silently depends on every probe name starting
+          # with a zero.
+          grep -l "#gcmetric" $errs >/dev/null \
+            || { echo "::error::no probe emitted gc metrics — the collector never ran"; exit 1; }

benchmarks/gc_ratchet/probes/09_try_catch_roots.ts

@@ -0,0 +1,139 @@
+// GC ratchet probe: live roots held across a throw and a collection.
+//
+// This is the probe that had no equivalent while try/catch lowered to
+// setjmp/longjmp. Under that lowering a longjmp could jump past a
+// `gc.relocate`, so the relocated pointer was never written back and a local
+// could be left pointing at a moved object. Functions containing `try` were
+// therefore excluded from statepoints and routed to the plain-stack-map
+// lowering, which is itself unsound — LLVM may record a root slot's address in
+// a caller-saved register that cannot be recovered at collection time.
+//
+// With invoke/landingpad lowering (#7302) the unwind edge is explicit and
+// relocations exist on BOTH edges, so statepoints cover try-carrying functions
+// too. Nothing else in this suite has a `try` in it, so without this probe the
+// newly covered case is exercised by nothing at all.
+//
+// What it checks, specifically:
+//   * objects allocated INSIDE a try survive a collection that happens inside
+//     the same try, and read back correctly afterwards;
+//   * locals live ACROSS the throw — allocated before it, read in the catch —
+//     still hold their contents once the collection has moved things;
+//   * the same holds when the throw crosses a frame boundary (thrown deep,
+//     caught shallow) so the roots being rewritten are in a caller's frame;
+//   * `finally` runs on both the normal and unwinding edges.
+//
+// A lost or stale root shows up as a wrong checksum rather than a crash, which
+// is why every survivor is folded into the output.
+
+declare function gc(): void;
+
+const ROUNDS = 400;
+const PER_ROUND = 96;
+
+let escape: object[] | null = null;
+
+class Payload {
+  tag: number;
+  body: string;
+  constructor(tag: number) {
+    this.tag = tag;
+    this.body = "p" + tag;
+  }
+  value(): number {
+    return (this.tag + this.body.length) | 0;
+  }
+}
+
+// Thrown from the deepest frame so the unwind crosses several frames that hold
+// live roots of their own.
+function deep(level: number, seed: number): number {
+  if (level === 0) {
+    throw new Payload(seed);
+  }
+  const local = new Payload(seed + level);
+  const nested = deep(level - 1, seed);
+  // Unreachable, but keeps `local` live across the call in the eyes of any
+  // liveness analysis that is not lying to us.
+  return (local.value() + nested) | 0;
+}
+
+function roundTrip(seed: number): number {
+  // Live across the whole try/catch, including the collection.
+  const survivors: Payload[] = [];
+  let acc = 0;
+
+  try {
+    for (let i = 0; i < PER_ROUND; i++) {
+      survivors.push(new Payload(seed + i));
+    }
+    // Collect with everything above live and reachable only from this frame.
+    if ((seed & 15) === 0) {
+      escape = survivors.slice(0, 8);
+      gc();
+      escape = null;
+    }
+    acc = (acc + deep(6, seed)) | 0;
+  } catch (err) {
+    // The caught value must be the object that was thrown, after a collection
+    // that may have moved it.
+    const caught = err as Payload;
+    acc = (acc + caught.value()) | 0;
+    // Every survivor allocated before the throw must still be intact.
+    for (let i = 0; i < survivors.length; i++) {
+      acc = (acc + survivors[i].value()) | 0;
+    }
+  } finally {
+    // Runs on the unwinding edge; `survivors` must still be readable here.
+    acc = (acc + survivors.length) | 0;
+  }
+
+  return acc;
+}
+
+// Normal (non-throwing) exit through a try/finally, so the non-unwind edge of
+// the same lowering is covered too.
+function normalExit(seed: number): number {
+  const held = new Payload(seed);
+  try {
+    if ((seed & 31) === 0) {
+      gc();
+    }
+    return held.value();
+  } finally {
+    escape = null;
+  }
+}
+
+let checksum = 0;
+for (let r = 0; r < ROUNDS; r++) {
+  checksum = (checksum + roundTrip(r)) | 0;
+  checksum = (checksum + normalExit(r)) | 0;
+}
+
+// A rethrow that is caught one frame up, with roots live in both frames.
+function rethrower(seed: number): number {
+  const outer = new Payload(seed);
+  try {
+    try {
+      gc();
+      throw new Payload(seed + 1);
+    } catch (inner) {
+      throw new Payload((inner as Payload).tag + outer.tag);
+    }
+  } catch (final) {
+    return ((final as Payload).value() + outer.value()) | 0;
+  }
+}
+
+for (let r = 0; r < 32; r++) {
+  checksum = (checksum + rethrower(r)) | 0;
+}
+
+gc();
+const mu = process.memoryUsage();
+
+console.log("probe:09_try_catch_roots");
+console.log("checksum:" + checksum);
+console.error("#gcmetric heap_used_bytes=" + mu.heapUsed);
+console.error("#gcmetric heap_total_bytes=" + mu.heapTotal);
+console.error("#gcmetric rss_bytes=" + mu.rss);

changelog.d/7314-statepoint-native-roots.md

@@ -0,0 +1,75 @@
+### Native-frame GC roots via LLVM statepoints, opt-in (#7173, #7174)
+
+Adds a second precise-root mechanism alongside the shadow stack, selected by
+`PERRY_STATEPOINTS=1` (explicit bridge) or `PERRY_RS4GC=1` (LLVM's
+`RewriteStatepointsForGC` owns statepoint and relocation insertion). Either
+one activates native roots on its own — `native_stack_roots_enabled()` is
+`statepoints || rs4gc` — so `PERRY_RS4GC=1` does not require
+`PERRY_STATEPOINTS=1`. **The default path is unchanged**: with neither set
+nothing here runs, and the shadow stack remains the shipping root mechanism.
+
+The point of the mechanism is that the forgot-to-root bug class becomes
+structurally impossible — LLVM, not Perry, is responsible for knowing which
+values are live across a call and for rewriting them after a collection moves
+them.
+
+**Every root path fails closed.** The plain `llvm.experimental.stackmap`
+lowering is deleted outright rather than kept as a fallback: LLVM may record a
+root slot's address as `Register R#N`, caller-saved and unrecoverable at
+collection time, so a fallback to it silently loses roots. It survived in three
+places, all of which failed open:
+
+* `PreciseRootBackend::StackMap` was dead by construction (both sites setting
+  `stack_map_requested` are guarded by `native_stack_roots_enabled()`, which is
+  exactly `statepoints || rs4gc`);
+* the statepoint backend fell back for calls it could not parse — chiefly
+  **indirect** calls. That was a limitation of Perry's textual parser, not of
+  statepoints: `gc.statepoint` takes its callee as a `ptr` operand, so
+  `ptr elementtype(T) %fnptr` is as valid as `... @callee`. Indirect targets are
+  now statepoint-able and anything still unparseable is a hard compile error;
+* the compact-map rewriter fell back to keeping LLVM's section, which reads as
+  conservative and is not — the runtime reads only `__perry_gcmap`, so those
+  records sit unread and that module's roots go missing. Now a hard error.
+
+**The metadata is re-encoded rather than shipped as LLVM emits it.** Measured on
+`test-drizzle-pg`, `__llvm_stackmaps` was 4.21 MB, of which >50% was data the
+runtime already discarded at startup: three `Constant` slots per record
+(`gc.statepoint`'s calling-convention preamble) and a duplicate of every root
+(LLVM records base and derived; Perry has no interior pointers). Perry now
+rewrites that block at assembly time — where LLVM prints the function addresses
+as symbol names, so one text parser replaces Mach-O *and* ELF relocation
+parsing plus a second link pass — into a compact map: 4,214,384 B → 224,832 B
+(19.0× measured same-build; 18.5× once the conservative `js_throw`
+classification below is accounted for). The largest single lever is that **77% of records have the identical
+live set as the record before them**, so a repeat flag replaces the payload;
+that also lets the runtime share one copy per distinct set instead of
+materialising 154k entries.
+
+**Try/catch is covered.** Now that exception lowering uses `invoke`/`landingpad`
+(#7302), no jump can skip a `gc.relocate`, so try-carrying functions take
+statepoints like any other. Under RS4GC they additionally need
+`landingpad token` — RS4GC uses the landing pad *as* the relocate token — which
+is sound here only because the pad's value is dead; the retype refuses if the
+pad register is referenced anywhere.
+
+`benchmarks/gc_ratchet/probes/09_try_catch_roots.ts` is new and exists because
+nothing in the suite contained a `try` at all: objects allocated inside a `try`
+surviving a collection there, locals live across a throw and read in the
+`catch`, a throw crossing several frames so the rewritten roots sit in a
+caller's frame, `finally` on both edges, and a rethrow caught one frame up.
+
+**Measured on `test-drizzle-pg` (133 modules):** 23,301 safepoints, all
+statepoints, 0 plain stack maps, 0 parser fallbacks, 129,914 relocations.
+Binary size is a wash against the shadow stack (+496 B for the bridge,
++50,064 B for RS4GC): statepoints generate less code (`__text` −151 KB / −240 KB,
+plus ~105 KB less `__eh_frame`) and that is cancelled by the remaining
+189–221 KB of map. Runtime is −0.93% (RS4GC) and RSS is flat.
+
+Also lands three mode-independent codegen fixes that the work depended on:
+codegen-unit globals are emitted only into units that reference them and
+declarations are scoped to the unit that needs them (per-unit IR previously grew
+with unit *count*, which is why the 13 MB `@anthropic-ai/claude-code` bundle hit
+`clang: translation unit is too large` no matter how finely it was split —
+885 KB → 299 KB per unit on a 4-unit module), and codegen units now compile with
+bounded parallelism (`PERRY_CODEGEN_UNIT_JOBS`, default `parallelism/4` clamped
+to `[1,4]`) instead of one at a time.

crates/perry-codegen/src/codegen/helpers.rs

@@ -74,6 +74,67 @@ pub(super) fn shadow_stack_enabled() -> bool {
     })
 }
 
+/// Research-only moving-GC backend using LLVM's explicit statepoint
+/// relocation sequence (`PERRY_STATEPOINTS=1`).
+///
+/// The standalone plain-stack-map mode (`PERRY_STACK_MAPS`) was deleted per
+/// the GC knob kill-policy after the quiet-host matrix: statepoints matched
+/// it within timer quantization, and it is structurally unsound — LLVM's
+/// stackmap intrinsic can record a root slot's address as `Register R#N`
+/// (caller-saved, unrecoverable at collection time), making those roots
+/// invisible to the collector by construction. The plain-map LOWERING
+/// survives only as this mode's internal fallback for `try`/setjmp
+/// functions and unsupported call forms. The Register hazard exists there
+/// too, which is why shrinking the fallback set is the remaining
+/// correctness work for this backend, tracked in the experiment doc.
+pub(crate) fn statepoints_enabled() -> bool {
+    matches!(
+        std::env::var("PERRY_STATEPOINTS").as_deref(),
+        Ok("1") | Ok("on") | Ok("true")
+    )
+}
+
+/// `PERRY_RS4GC=1` — research pipeline for #7174: root allocas become
+/// `ptr addrspace(1)`, functions are tagged `gc "statepoint-example"`, and
+/// each module is piped through `opt -passes='function(mem2reg),
+/// rewrite-statepoints-for-gc'` before clang. LLVM then inserts every
+/// statepoint, relocation, and downstream-use rewrite itself — replacing the
+/// explicit bridge's hand emission and its conservative CFG-union liveness.
+/// Requires an `opt` binary (`PERRY_LLVM_OPT`, Homebrew LLVM, or PATH).
+pub(crate) fn rs4gc_enabled() -> bool {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        matches!(
+            std::env::var("PERRY_RS4GC").as_deref(),
+            Ok("1") | Ok("on") | Ok("true")
+        )
+    })
+}
+
+/// Whether precise roots should use a native-stack metadata backend rather
+/// than Perry's heap-backed shadow frame.
+pub(crate) fn native_stack_roots_enabled() -> bool {
+    statepoints_enabled() || rs4gc_enabled()
+}
+
+/// `PERRY_GC_SAFEPOINT_ONLY=1` — the explicit-safepoint collection contract
+/// (research, `exp/stackmap-viability`). The runtime enforces that a
+/// precise-root collection only begins at a declared safepoint; under that
+/// guarantee, audited allocate-but-never-reenter helpers
+/// (`GcCallEffect::AllocNoReentry`) need no statepoint. Participates in both
+/// build and object cache keys.
+pub(crate) fn gc_safepoint_only_contract_enabled() -> bool {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        matches!(
+            std::env::var("PERRY_GC_SAFEPOINT_ONLY").as_deref(),
+            Ok("1") | Ok("on") | Ok("true") | Ok("strict")
+        )
+    })
+}
+
 /// Inline shadow-slot store gate (#7088). Default ON.
 ///
 /// When enabled, a store to a GC-rooted local is emitted as an address

crates/perry-codegen/src/expr/shadow_slot.rs

@@ -206,6 +206,19 @@ pub(crate) fn emit_shadow_slot_bind_for_local(ctx: &mut FnCtx<'_>, local_id: u32
         return;
     };
     ctx.shadow_slots_bound.insert(slot_idx);
+    if crate::codegen::helpers::native_stack_roots_enabled() {
+        // Kept temporarily as a textual marker: LlFunction's final stack-map
+        // lowering records `slot_idx -> local_slot` and removes this call.
+        // The incremental root barrier remains real because the native slot
+        // can be updated after an in-flight cycle scanned this frame.
+        ctx.block().call_void(
+            "js_shadow_slot_bind",
+            &[(I32, &slot_idx.to_string()), (PTR, &local_slot)],
+        );
+        let value_bits = ctx.block().load(I64, &local_slot);
+        emit_persistent_shadow_root_barrier(ctx, &value_bits);
+        return;
+    }
     // #7088: the hot per-store root write. Emitted inline against this
     // activation's cached `ShadowStackState` pointer when it has one; falls
     // through to the call otherwise.