diff --git a/CLAUDE.md b/CLAUDE.md index 7320a237bb..fd42f01d0d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1366 +**Current Version:** 0.5.1367 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 37029da844..22c2fddb32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5547,7 +5547,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "base64", @@ -5607,14 +5607,14 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "serde", ] [[package]] name = "perry-audio-miniaudio" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "cc", "libc", @@ -5622,7 +5622,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "inkwell", @@ -5639,7 +5639,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-hir", @@ -5647,7 +5647,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-hir", @@ -5655,7 +5655,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-dispatch", @@ -5664,7 +5664,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-hir", @@ -5672,7 +5672,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "base64", @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-hir", @@ -5692,7 +5692,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "async-trait", @@ -5721,14 +5721,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "serde", "serde_json", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1366" +version = "0.5.1367" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5747,7 +5747,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "clap", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "block2", "objc2", @@ -5772,7 +5772,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "argon2", "perry-ffi", @@ -5780,7 +5780,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "reqwest", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "bcrypt", "perry-ffi", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "rusqlite", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "scraper", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "perry-runtime", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "chrono", "cron", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "chrono", "perry-ffi", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "rust_decimal", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "serde_json", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "rand 0.10.1", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "perry-runtime", @@ -5871,14 +5871,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "bytes", "http-body-util", @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "bytes", "lazy_static", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "bytes", "h2", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "lazy_static", "perry-ffi", @@ -5943,7 +5943,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "jsonwebtoken", @@ -5954,7 +5954,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "lru", "perry-ffi", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "chrono", "perry-ffi", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "bson", "futures-util", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "chrono", "perry-ffi", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "nanoid", "perry-ffi", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "bytes", "perry-ffi", @@ -6015,7 +6015,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "const-oid 0.9.6", "der 0.7.10", @@ -6034,7 +6034,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "lettre", "perry-ffi", @@ -6044,7 +6044,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "printpdf", @@ -6052,7 +6052,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "sqlx", @@ -6061,7 +6061,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "governor", "perry-ffi", @@ -6069,7 +6069,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "fast_image_resize", "image", @@ -6079,14 +6079,14 @@ dependencies = [ [[package]] name = "perry-ext-slugify" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-streams" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "lazy_static", "perry-ffi", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "perry-runtime", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "uuid", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ffi", "regex", @@ -6122,7 +6122,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "futures-util", "lazy_static", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "brotli", "flate2", @@ -6145,7 +6145,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "dashmap", "once_cell", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-api-manifest", @@ -6172,7 +6172,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-diagnostics", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "base64", @@ -6226,14 +6226,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6328,14 +6328,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "perry-hir", @@ -6344,14 +6344,14 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ui-model", ] [[package]] name = "perry-ui-android" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "itoa", @@ -6368,7 +6368,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "rand 0.10.1", "serde", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "cairo-rs 0.22.0", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "block2", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "block2", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1366" +version = "0.5.1367" [[package]] name = "perry-ui-test" @@ -6443,11 +6443,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1366" +version = "0.5.1367" [[package]] name = "perry-ui-tvos" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "block2", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "block2", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "block2", "libc", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "base64", "libc", @@ -6509,14 +6509,14 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "perry-ui-windows", ] [[package]] name = "perry-updater" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "anyhow", "base64", @@ -6532,7 +6532,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1366" +version = "0.5.1367" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 1c1b4e18b7..b842b8f410 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ codegen-units = 16 codegen-units = 16 [workspace.package] -version = "0.5.1366" +version = "0.5.1367" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/7646-gc-census-runs-membership.md b/changelog.d/7646-gc-census-runs-membership.md new file mode 100644 index 0000000000..da848a6135 --- /dev/null +++ b/changelog.d/7646-gc-census-runs-membership.md @@ -0,0 +1,24 @@ +**`perf(gc)`: the full collection's valid-pointer census kept a `BTreeSet` shadowing the run vector it was already building (#7592).** + +`ValidPointerSet` held two structures over the same addresses. `arena_runs` — the census walk's arena object starts, handed over by `ArenaObjectCursorBuilder::new(ArenaWalkOrder::Address)` in **ascending address order** — was documented as existing only for `enclosing_object`'s floor lookups. `lookup_set: BTreeSet` answered exact membership, fed by the *same* `push_arena` call. But the runs are sorted by construction, so a floor lookup that lands **on** the query already is the membership answer: the B-tree was buying nothing except one insert per live arena object. + +On `json_pipeline` 500k that shadow cost **245.5 ms of a 748.3 ms full collection** (the GC's own `phase_us.build_valid_pointer_set`) — 12.6% of the `build_out` phase, and the second-largest single leaf in the symbolicated profile. + +Membership for arena starts now comes from the runs; the B-tree keeps only malloc-tracked starts, which have no address order to exploit and are skipped entirely when empty. `lookup_count()` preserves `snapshot_for_tests`'s meaning (arena starts + malloc starts). + +**One part is load-bearing and is recorded so it is not "simplified" away.** A first cut searched `arena_runs: Vec>` directly, calling `run.first()` per probe. That arm moved `phase_us.trace_worklist` the *wrong* way — 388.8 → 493.1 ms, +99 ms over ~9.6M lookups — and netted only −6.3% on `build_out`, which reads like "the B-tree was buying something". It was not; it was buying an indirection. Mirroring each run's first key into one contiguous `arena_run_firsts` vector (~4k entries, 32 KB, L2-resident, against 33 MB of run storage) turned that +99 ms into −62 ms and took the change from −6.3% to −15.4%. + +Measured on the pinned quiet mini, interleaved A/B, 7 rounds, `PERRY_NO_AUTO_OPTIMIZE=1` with a pinned `PERRY_RUNTIME_DIR`: + +| | 200k | 500k | +|---|--:|--:| +| `build_out` phase | 731 → **620 ms (−15.2%)** | 1,956 → **1,654 ms (−15.4%)** | +| total wall | 1,155 → **1,041 ms (−9.9%)** | 3,008 → **2,703 ms (−10.1%)** | +| the full collection's pause | 294.4 → **182.8 ms (−37.9%)** | 750.3 → **458.3 ms (−38.9%)** | +| ↳ `build_valid_pointer_set` | 94.3 → **9.1 ms** | 247.0 → **23.4 ms** | +| ↳ `trace_worklist` | 152.9 → **129.2 ms** | 388.8 → **326.4 ms** | +| peak RSS | 461.7 → **446.2 MB (−3.3%)** | flat (±1%) | + +Output SHA-256 identical at both sizes. No policy, pacing, trigger, promotion or root-set behaviour is touched — the membership *set* is unchanged, only its storage and probe. `PERRY_GC_TRACE=1` on both arms at both sizes, comparing **every non-timing key of every `gc_cycle` event field by field: 0 differences** (same cycle count, kinds, triggers, `promoted_*`, `freed_bytes`, `copied_*`, `remembered_set`, `old_pages`, `layout_scans`, `root_sources`, `sweep`). + +Found while re-deriving the next `json_pipeline` lever after #7624 and #7633. The **larger** item on the same profile — the copying minor's eligibility preflight, a second full traversal of the young graph worth 21.8% of `build_out` — is filed as #7645 rather than taken here: it is a guard on the moving collector and needs a pin-site completeness gate, a sabotage test and a deliberate ratchet counter shift first. diff --git a/crates/perry-runtime/src/gc/tests/cycle_state.rs b/crates/perry-runtime/src/gc/tests/cycle_state.rs index 112519b8b3..69259cb541 100644 --- a/crates/perry-runtime/src/gc/tests/cycle_state.rs +++ b/crates/perry-runtime/src/gc/tests/cycle_state.rs @@ -272,6 +272,78 @@ fn build_valid_pointer_set_sliced_build_preserves_contains_and_enclosing_object( } } +/// #7646: arena membership now answers from the address-ordered census runs +/// rather than a shadow `BTreeSet`, which makes RUN BOUNDARIES load-bearing. +/// Runs seal every `VALID_POINTER_ARENA_RUN_CAPACITY` (1024) starts, so the +/// final run is partial and is only sealed by `finalize()`. The sliced-build +/// test above allocates 1100 strings — enough to cross the boundary — but +/// checks only the first 16, which all live in the FIRST run: it passes +/// unchanged if every later run is lost. +/// +/// This checks every start, both directions. +#[test] +fn valid_pointer_membership_spans_every_census_run_including_the_partial_one_7646() { + let _guard = CopyingNurseryTestGuard::new(0); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + + // > 2 full runs, so the last one is partial and cannot be sealed by the + // capacity check alone. + let arena_strings = (0..2600).map(|_| young_leaf()).collect::>(); + let (arena_object, fields) = unsafe { alloc_nursery_test_object(4) }; + let arena_object = arena_object as usize; + let interior = fields as usize; + + let valid_ptrs = ValidPointerSetBuilder::new().finish(); + + assert!( + valid_ptrs.arena_runs.len() >= 3, + "premise: the census must span several runs, got {}", + valid_ptrs.arena_runs.len() + ); + assert!( + valid_ptrs.current_arena_run.is_empty(), + "finalize() must seal the open run before the set escapes the builder; \ + {} starts would otherwise be invisible to membership", + valid_ptrs.current_arena_run.len() + ); + assert_eq!( + valid_ptrs.arena_run_firsts.len(), + valid_ptrs.arena_runs.len(), + "the fence mirror must stay index-aligned with the runs" + ); + for (index, run) in valid_ptrs.arena_runs.iter().enumerate() { + assert_eq!( + valid_ptrs.arena_run_firsts[index], run[0], + "fence {index} must equal its run's first key" + ); + } + + // Positive: EVERY censused start, not a prefix — a start in the last, + // partial run is the one a lost `finalize()` drops. + for (index, &ptr) in arena_strings.iter().enumerate() { + assert!( + valid_ptrs.contains(&ptr), + "arena start {index} of {} is censused but not reported as a member", + arena_strings.len() + ); + } + assert!(valid_ptrs.contains(&arena_object)); + + // Negative: membership must not become a SUPERSET. A floor lookup returns + // the greatest censused start <= ptr, so an interior pointer floors to its + // object and must still be rejected as a START — otherwise the conservative + // scan would begin tracing from the middle of an object. + assert!( + !valid_ptrs.contains(&interior), + "an interior pointer must not report as an object start" + ); + assert_eq!(valid_ptrs.enclosing_object(interior), Some(arena_object)); + assert!( + !valid_ptrs.contains(&(arena_object + 1)), + "an unaligned address inside an object must not report as a start" + ); +} + #[test] fn build_valid_pointer_set_finalize_is_separate_bounded_phase() { let _guard = CopyingNurseryTestGuard::new(0); diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index bef18bb800..aa7ffb450a 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -53,17 +53,36 @@ thread_local! { const VALID_POINTER_ARENA_RUN_CAPACITY: usize = 1024; pub(crate) struct ValidPointerSet { - /// Arena-only start pointers in address-ordered runs. `lookup_set` - /// handles exact pointer membership; these runs support - /// `enclosing_object` floor lookups for arena interior pointers without - /// a final heap-sized merge. + /// Arena-only start pointers in address-ordered runs — **the exact arena + /// membership set**, not merely an index for `enclosing_object`'s floor + /// lookups. `ArenaObjectCursorBuilder::new(ArenaWalkOrder::Address)` hands + /// the census headers over in ascending address order, so each run is + /// sorted by construction and a floor lookup that lands on the query IS + /// the membership answer. + /// + /// This used to be shadowed by a parallel `BTreeSet` over the same + /// addresses. That set cost one B-tree insert per live arena object with + /// nothing to show for it: the runs already held the same data in the same + /// order. On `json_pipeline` 500k the shadow cost **245.5 ms of a 748.3 ms + /// full collection** (`phase_us.build_valid_pointer_set`), 12.6% of the + /// `build_out` phase, and ~40 MB of transient peak heap (#7592). pub(super) arena_runs: Vec>, + /// `arena_runs[i].first()`, mirrored into one contiguous vector so the + /// run-level binary search reads 8-byte fences instead of chasing a + /// `Vec` header per probe. At 500k `json_pipeline` records this is ~4k + /// entries (32 KB, L2-resident) against 33 MB of run storage, and it is + /// what keeps the membership lookup competitive with the B-tree probe it + /// replaces (#7592). + pub(super) arena_run_firsts: Vec, pub(super) current_arena_run: Vec, - /// Exact pointer membership filled incrementally as arena and malloc - /// entries are discovered. A B-tree avoids hash-table rebuilds in tiny + /// Live count of pushed arena starts (sealed runs + the open one), kept so + /// `lookup_count` stays O(1). + pub(super) arena_count: usize, + /// Exact membership for **malloc-tracked** objects only, which have no + /// address order to exploit. A B-tree avoids hash-table rebuilds in tiny /// budget steps; insertion may split one fixed-size node but never /// rehashes all previously discovered pointers. - pub(super) lookup_set: std::collections::BTreeSet, + pub(super) malloc_lookup: std::collections::BTreeSet, // Min/max heap-pointer range across the valid set. Updated as entries // are inserted. The conservative stack scan calls `contains` once per // 8-byte stack word (~1024 calls per scanned KB of stack) and @@ -99,8 +118,10 @@ impl ValidPointerSet { pub(super) fn new() -> Self { Self { arena_runs: Vec::new(), + arena_run_firsts: Vec::new(), current_arena_run: Vec::with_capacity(VALID_POINTER_ARENA_RUN_CAPACITY), - lookup_set: std::collections::BTreeSet::new(), + arena_count: 0, + malloc_lookup: std::collections::BTreeSet::new(), range_min: usize::MAX, range_max: 0, tenured_nursery_bytes: 0, @@ -125,9 +146,9 @@ impl ValidPointerSet { debug_assert!(previous <= ptr); } - self.lookup_set.insert(ptr); self.record_pointer_range(ptr); self.current_arena_run.push(ptr); + self.arena_count += 1; if self.current_arena_run.len() >= VALID_POINTER_ARENA_RUN_CAPACITY { self.seal_current_arena_run(); } @@ -137,10 +158,15 @@ impl ValidPointerSet { if self.classifier_mode { return; // #6179: no exact census in classifier mode } - self.lookup_set.insert(ptr); + self.malloc_lookup.insert(ptr); self.record_pointer_range(ptr); } + /// Total censused entries (arena starts + malloc starts). + pub(super) fn lookup_count(&self) -> usize { + self.arena_count + self.malloc_lookup.len() + } + pub(super) fn record_tenured_nursery_bytes(&mut self, bytes: usize) { self.tenured_nursery_bytes += bytes; } @@ -166,6 +192,9 @@ impl ValidPointerSet { return; } let sealed = std::mem::take(&mut self.current_arena_run); + // Non-empty by the guard above, so the fence mirror stays index-aligned + // with `arena_runs` — `arena_run_firsts[i] == arena_runs[i][0]`. + self.arena_run_firsts.push(sealed[0]); self.arena_runs.push(sealed); } @@ -199,10 +228,13 @@ impl ValidPointerSet { // censused address range. return false; } - // Exact lookup. The B-tree insert path is bounded during - // `BuildValidPointerSet`, so a tiny GC step cannot trigger a - // heap-sized hash-table rebuild. - let exact = self.lookup_set.contains(ptr); + // Exact lookup. Arena starts answer from the address-ordered census + // runs (a floor lookup that lands ON the query is membership); only + // malloc-tracked starts, which have no usable order, need the B-tree. + // Arena first because arena hits dominate every workload that reaches + // here — a malloc pointer pays one extra run-level binary search. + let exact = self.arena_start_censused(*ptr) + || (!self.malloc_lookup.is_empty() && self.malloc_lookup.contains(ptr)); // #6179 differential verification (PERRY_GC_VERIFY_CLASSIFIER=1): // before the exact set can be replaced by page-metadata // classification on precise cycles, the classifier must be proven a @@ -253,10 +285,38 @@ impl ValidPointerSet { } } + /// Exact arena membership: the census runs are address-ordered, so `ptr` + /// was censused iff its floor is itself. + /// + /// **Load-bearing ordering requirement, which the `BTreeSet` this replaced + /// did not have.** The B-tree was complete after every `push_arena`, so a + /// mid-build query merely saw fewer entries. The runs are only complete + /// once `finalize()` has sealed `current_arena_run` — up to + /// `VALID_POINTER_ARENA_RUN_CAPACITY` censused starts are invisible before + /// that. A membership query on an unsealed set is therefore a FALSE + /// NEGATIVE, and a false negative here is not a missed optimisation: the + /// conservative scan drops the root, the object is swept live, and the + /// failure surfaces cycles later as `TypeError: value is not a function`. + /// + /// The builder's phase machine guarantees this today (`Finalize` precedes + /// `Done`, and the set escapes only through `finish()`), so the assert is + /// free in release. It exists so that a phase added after `Finalize`, or a + /// caller that queries a partially-built set, fails a test instead of + /// corrupting the heap. Verified to fire: skipping the seal in `finalize` + /// trips it with "4 censused starts are invisible to this lookup". + #[inline] + fn arena_start_censused(&self, ptr: usize) -> bool { + debug_assert!( + self.current_arena_run.is_empty(), + "arena membership queried before finalize() sealed the open run: \ + {} censused starts are invisible to this lookup", + self.current_arena_run.len() + ); + self.find_arena_floor(ptr) == Some(ptr) + } + fn find_arena_floor(&self, ptr: usize) -> Option { - let idx = self - .arena_runs - .partition_point(|run| run.first().copied().is_some_and(|first| first <= ptr)); + let idx = self.arena_run_firsts.partition_point(|&first| first <= ptr); if idx == 0 { return None; } @@ -470,7 +530,7 @@ impl ValidPointerSetBuilder { .map_or(0, crate::arena::ArenaObjectCursorBuilder::inspected_blocks), arena_run_count: self.set.arena_runs.len(), current_arena_run_len: self.set.current_arena_run.len(), - lookup_count: self.set.lookup_set.len(), + lookup_count: self.set.lookup_count(), malloc_index: self.malloc_index, } }