From 3f2c62a8d3f22070d3ada574f125fb2ac754a687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 10 Aug 2026 05:23:51 +0200 Subject: [PATCH 1/4] perf(gc): whole-block in-place promotion of a fully-live young generation (#7742) When a copying minor's nursery is (near-)entirely live, relabel its blocks as old-gen instead of evacuating them object by object. `retain.ts` 0.80 s -> 0.53 s, `retain_wide.ts` 1.33 -> 1.07, `deeplist` 0.30 -> 0.24 (RSS 117 -> 97 MB); promotion cost 243 ns/object -> 101 ns/object, derived from the trace's promoted-object count and pause. The decision is a measurement, not a guess: the young-survival ratio is bimodal across the GC benchmark set (1.000 on retain/deeplist, <= 0.004 on churn/push/cycles), and a PROMOTING cycle still traces, so it re-measures the ratio it will be judged by. A misprediction therefore costs one nursery of retained garbage, and a running 32 MB cap bounds the steady state. Full rationale, the pacing regression this had to fix first, and the three remembered-set passes a promoting cycle can prove empty: see the changeset. --- .../7744-gc-whole-block-in-place-promotion.md | 156 ++++++ crates/perry-runtime/src/arena/mod.rs | 26 +- crates/perry-runtime/src/arena/page_meta.rs | 129 ++++- crates/perry-runtime/src/arena/promote.rs | 475 ++++++++++++++++++ crates/perry-runtime/src/arena/tests.rs | 8 + crates/perry-runtime/src/gc/copying.rs | 209 +++++++- crates/perry-runtime/src/gc/layout.rs | 24 + crates/perry-runtime/src/gc/mod.rs | 5 + crates/perry-runtime/src/gc/policy.rs | 12 +- .../perry-runtime/src/gc/promote_in_place.rs | 258 ++++++++++ crates/perry-runtime/src/gc/telemetry.rs | 27 + crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../src/gc/tests/promote_in_place.rs | 207 ++++++++ scripts/addr_class_allowlist.txt | 1 + 14 files changed, 1514 insertions(+), 24 deletions(-) create mode 100644 changelog.d/7744-gc-whole-block-in-place-promotion.md create mode 100644 crates/perry-runtime/src/arena/promote.rs create mode 100644 crates/perry-runtime/src/gc/promote_in_place.rs create mode 100644 crates/perry-runtime/src/gc/tests/promote_in_place.rs diff --git a/changelog.d/7744-gc-whole-block-in-place-promotion.md b/changelog.d/7744-gc-whole-block-in-place-promotion.md new file mode 100644 index 0000000000..ee1501e915 --- /dev/null +++ b/changelog.d/7744-gc-whole-block-in-place-promotion.md @@ -0,0 +1,156 @@ +### perf(gc): whole-block in-place promotion of a fully-live young generation (#7742) + +When a copying minor's nursery is (near-)entirely live, the collector now +relabels its blocks as old-gen instead of evacuating them object by object. +`retain.ts` 0.80 s → **0.53 s**, `retain_wide.ts` 1.33 s → **1.07 s**, +`retain1` 0.38 → 0.29, `retain_wide1` 0.38 → 0.27, `deeplist` 0.30 → 0.24 +(peak RSS 117 MB → 97 MB). Promotion cost per object **243 ns → 101 ns**, +derived from the trace's own promoted-object count and pause, not estimated. +All measurements are best-of-5 wall clock on the pinned quiet M1 mini, outputs +byte-verified against `node --experimental-strip-types` before timing. + +#### Why the old path was pure overhead here + +`retain.ts` builds a 3M-element array of records and drops none of them. Its GC +trace measures a young-survival ratio of **1.000** on every copying minor, and +its four promoting minors moved 2,097,155 objects in 509.7 ms of pause. Every +one of those moves paid for a fresh `arena_alloc_gc_old`, a `memcpy`, a +`layout_transfer`, `old_page_account_promoted_object`, the `GcMoveHookKind` +hooks, a forwarding stub, and the rewrite of every slot that referred to it — to +put an object somewhere it had no reason to be. + +Whole-block promotion (V8 calls it page promotion) pays none of that. The block +changes generation; the bytes do not move, so nothing in the heap and nothing in +any address-keyed runtime side table needs rewriting. + +#### The mechanism (`crates/perry-runtime/src/arena/promote.rs`) + +Two halves around the existing trace: + +* **Before the trace**, `retag_young_for_in_place_promotion` flips every in-use + Eden and survivor block's page range to generation `Old`, space + `HeapSpace::PromotedYoung`. From that instant the barrier predicates + (`barrier_parent_needs_remembering`, `remembered_child_needs_tracking`) read + old-gen semantics, which is what makes the trace record the right + remembered-set edges; the distinct space is what still tells the copier "this + was young at cycle start, so it owes one field scan". The blocks stay in their + own arenas for the cycle, so no old-gen allocation can land in them mid-way. +* **Where the from-space reset would have run**, `finish_in_place_promotion` + walks each block linearly (one `size` hop per object, no hashing), stamps + `GC_FLAG_TENURED` on every header — `Old ⟹ TENURED` is what the generated + write barrier's fast path is gated on (#7511) — registers the live objects + into the old-gen page index in per-page bulk runs, hands the block to + `OLD_ARENA` leaving the usual tombstone behind, and retags the range to plain + `HeapSpace::Old`. + +Unmarked objects on a promoted block are dead and are neither swept nor indexed; +they stay as old-gen garbage until the next full mark-sweep finds them through +the ordinary linear old-arena walk. That retention is the technique's entire +footprint cost. + +#### The policy is a measurement, not a guess (`gc/promote_in_place.rs`) + +Block liveness is not knowable before the trace and Eden blocks are recycled at +offset 0, so per-block history means nothing. The decision is per cycle, from +the previous cycle's measured young-survival ratio. Measured across the GC +benchmark set the population is bimodal by three orders of magnitude: + +| workload | copying minors | young-survival ratio | +|---|--:|--:| +| `retain`, `retain1`, `retain_wide` | 5–7 | 0.999 – 1.000 | +| `deeplist` | 3 | 1.000 | +| `churn`, `churn_alloc`, `push_cls` | 105 | 0.000 – 0.004 | +| `push_num`, `cycles` | 16–18 | 0.000 | +| `tree`, `tree_wide`, `churn_read` | 0 | — no copying minor runs | + +so the 95% threshold is justified by footprint, not by classification: a +mispredicted cycle retains at most 5% of the young generation. What makes the +per-cycle decision sound rather than optimistic is that **a promoting cycle +still traces, so it measures too** — a workload that flips from live to garbage +pays one nursery of retained garbage and then the policy turns itself off. A +running 32 MB cap on promoted dead bytes bounds the steady state the per-cycle +correction does not cover, and resets when a full collection reclaims them. +Measured `in_place_dead_bytes` and `in_place_sparse_blocks` (blocks under 50% +live) are **zero** on every benchmark in the set. + +#### Three passes that a promoting cycle can prove are empty + +A remembered-set entry is only ever created for a nursery-generation child or a +malloc-registry child. After the retag the first population is empty by +construction — every in-use young block was taken — and when the malloc registry +was empty at cycle start so is the second. On such a cycle +`visit_slot_with_parent`'s re-decode-and-remember, +`rebuild_evacuated_old_to_young_remembered_set`, and +`restore_surviving_dirty_coverage` can only insert nothing, and are skipped; +`debug_assert_no_remembering_possible` re-derives the premise from the heap in +debug builds. This is where most of the win is: it took `retain`'s promoting +minors from 42.8/86.5/107.8/156.2 ms to 23.4/46.7/57.6/84.0 ms. + +#### Pacing: the part that made the first cut a 1.52 s regression + +A copying minor recycles Eden's blocks, so Eden's free capacity survives the +collection and the arena-bytes trigger's runway is "that capacity + the step". +Promotion hands the blocks away. The first working version therefore collapsed +Eden's high-water from 57 MB to 18 MB on `retain`, doubled the collection count, +and bought a second 690 ms full collection — 0.81 s → **1.52 s** from a change +that had made every individual promotion cheaper. The capacity is now returned +to the next trigger as one-shot **headroom** rather than as re-reserved blocks; +re-reserving also fixes the pacing but maps memory the program may never reach +(measured: `retain_wide` peak RSS 470 MB against baseline's 447 MB). + +#### Also here + +`CopyingPointerSet::classify_arena` resolved the page-generation range twice per +visited slot — once for the user pointer, once for the header 8 bytes below it. +It now resolves once and answers the second from the range base, which is also +the guard that keeps a garbage candidate at the very start of a range from +becoming a read of the unmapped page below. `classify_heap_space` self-samples +on a retain profile: 201 → 108. + +#### Escape hatch and instruments + +`PERRY_GC_PROMOTE_IN_PLACE=0|off|false` reverts to object-by-object evacuation; +both states of the parse are asserted (`promote_in_place_knob_parses_both_states`), +per the GC knob kill-policy. `PERRY_GC_FORCE_EVACUATE` and `PERRY_GC_ZEAL` +disable the path outright — they exist to make objects MOVE, and a promoting +cycle moves nothing, so leaving it on would quietly stop exercising their +subject. The trace gains `in_place_promotion`, `in_place_promoted_objects`, +`in_place_promoted_blocks`, `in_place_dead_bytes`, `in_place_sparse_blocks`, +`young_survival_permille` and `remembering_skipped`, and the end-to-end test +gates on the promoted-object count rather than on "nothing threw". + +#### Validation + +`gc-handoff/apps/iso_miss.ts` prints `checksum 437840 misses 0` plain, under +`PERRY_GC_PROTECT_FROMSPACE=1 PERRY_GC_PROTECT_FROMSPACE_DEPTH=800`, and under +`PERRY_GC_VERIFY_EVACUATION=1`. It is an interpreter, so it takes the ordinary +copying path on all 50 of its minors and retires 50 quarantine sets — i.e. the +canary's instrument coverage is exactly what it was, and it does **not** cover +the new path. What does cover it: `PERRY_GC_FROMSPACE_SCAN_ABORT=1 ./n_retain` +(4 promoting cycles, up to 2,111,418 objects / 21,085,616 words scanned per +cycle) reports `missing_rewrites=0 dangling=0 never_dirty=0 lost_dirty=0 +dirty_but_missed=0` on every cycle — those last three are precisely the +"remembered set is missing an edge" counters that the skipped passes could have +broken. `retain`, `retain_wide` and `deeplist` also produce correct output under +`PERRY_GC_VERIFY_EVACUATION=1` and under the from-space quarantine. + +`cargo test -p perry-runtime --release`: 1982 passed, 0 failed. + +`gc_ratchet` (7 repeats, `shared_ci`): **every gated semantic cell — +correctness, heap bytes, cycle counts, copied/promoted/freed — is identical to +`origin/main` measured on the same host, including `12_large_live_set`.** The +six cells the profile reports as REGRESSION are reported identically by +`origin/main` itself, so they are pre-existing drift between the pinned artifact +and current `main`. The flip side of that identity: none of the 13 ratchet +probes runs a promoting cycle, so the ratchet does not yet cover this path. + +No protected GC benchmark regressed: `churn` 0.41, `churn_alloc` 0.37, +`push_cls` 0.35, `push_num` 0.14, `churn_read` 0.02, `cycles` 0.19, +`deeplist` 0.24, `tree` 1.63, `tree_wide` 2.10 s. Peak RSS `retain` +326 → 316 MB, `retain_wide` 447 → 446 MB. + +`retain` remains 4.4× Node (0.53 vs 0.12) and `retain_wide` 7.1× (1.07 vs 0.15). +The residue is no longer promotion bookkeeping: on a `retain_live_big` profile +the remaining GC time is the remembered-set scan over the array's dirty element +slots (26%) and one full collection that reclaims nothing (28%). #7742 stays +open for those. diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 9015ba1120..9e6e55d08a 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -12,6 +12,9 @@ mod allocators; mod block; mod inline; mod page_meta; +/// #7742: whole-block in-place promotion of a (near-)fully-live young +/// generation, in place of object-by-object evacuation. +mod promote; /// #7154 tooling: from-space quarantine + poison + `mprotect` so a stale /// pointer faults at the instruction that used it. Default-off. mod quarantine; @@ -91,6 +94,14 @@ pub(crate) use reset::{ }; pub use reset::{arena_reset_all_blocks_to_zero, arena_reset_empty_blocks}; +// promote.rs (#7742 whole-block in-place promotion) +#[cfg(debug_assertions)] +pub(crate) use promote::young_in_use_bytes_after_retag; +pub(crate) use promote::{ + finish_in_place_promotion, retag_young_for_in_place_promotion, InPlacePromotion, + InPlacePromotionStats, +}; + // quarantine.rs (#7154 from-space protection; default-off) pub(crate) use quarantine::{copying_quarantine_from_spaces_and_flip, protect_fromspace_enabled}; #[cfg(test)] @@ -111,13 +122,14 @@ pub(crate) use stats::{old_gen_in_use_bytes_recomputed, old_gen_in_use_bytes_res // page_meta.rs (public + pub(crate) classification/page-meta API) pub(crate) use page_meta::{ - classify_heap_generation, classify_heap_space, generation_page_for_addr, - old_arena_page_index_remove_object, old_arena_source_blocks_for_pages, - old_arena_walk_objects_on_pages, old_object_page_overlaps, old_page_account_dirty_slot, - old_page_account_promoted_object, old_page_account_swept_object, old_page_clear_dirty, - old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, - old_pages_reset_sweep_accounting, unregister_old_object_pages, HeapGeneration, HeapSpace, - OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, + classify_heap_generation, classify_heap_space, classify_heap_space_in_range, + generation_page_for_addr, old_arena_page_index_remove_object, + old_arena_source_blocks_for_pages, old_arena_walk_objects_on_pages, old_object_page_overlaps, + old_page_account_dirty_slot, old_page_account_promoted_object, old_page_account_swept_object, + old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, + old_pages_begin_gc_cycle, old_pages_reset_sweep_accounting, unregister_old_object_pages, + HeapGeneration, HeapSpace, OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, + OldPageSummary, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index ae9c7cee9f..0905fca734 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -25,6 +25,17 @@ pub(crate) enum HeapSpace { Survivor1, Longlived, Old, + /// A young block that a copying minor decided to promote **whole, in + /// place** (`arena/promote.rs`, #7742). Its generation is already `Old` — + /// the write barrier, `remembered_child_needs_tracking` and + /// `barrier_parent_needs_remembering` all see old-gen semantics from the + /// instant the retag lands — but the space stays distinguishable for the + /// duration of that one cycle so the copier can tell "must still be + /// traced once, because it was young when the cycle began" from a + /// genuinely old object it may skip. The finish walk retags it to + /// [`HeapSpace::Old`] before the mutator runs again, so no mutator-visible + /// classification ever observes this variant. + PromotedYoung, } impl HeapSpace { @@ -474,6 +485,100 @@ pub(crate) fn register_block_space( invalidate_generation_cache(); } +/// Change the generation/space a block already registered at `base..base+size` +/// reports, **without** disturbing the old-page metadata that +/// [`unregister_block_generation`] would tear down. +/// +/// Whole-block promotion (#7742) needs exactly this: the block's bytes do not +/// move, only their generation label does, and the sequence +/// `unregister_block_generation` + `register_block_space` is *not* equivalent — +/// the unregister half drops every `OLD_GEN_PAGE_META` / `OLD_GEN_PAGE_OBJECTS` +/// entry on those pages, which for the second (PromotedYoung → Old) retag would +/// throw away the object index the finish walk just built. +/// +/// Only ranges whose `base`/`end` match exactly are retagged, so a block that +/// shares a 1 MiB generation class with its neighbours cannot relabel them. +pub(crate) fn retag_block_space( + base: usize, + size: usize, + generation: HeapGeneration, + space: HeapSpace, +) { + if base == 0 || size == 0 || matches!(generation, HeapGeneration::Unknown) { + return; + } + let end = base + size; + let first_key = generation_class_key_for_addr(base); + let last_key = generation_class_key_for_addr(end - 1); + PAGE_GENERATIONS.with(|pages| { + let mut pages = pages.borrow_mut(); + for key in first_key..=last_key { + let Some(slot) = pages.get_mut(&key) else { + continue; + }; + match slot { + PageGenerationSlot::Single(range) => { + if range.base == base && range.end == end { + range.generation = generation; + range.space = space; + } + } + PageGenerationSlot::Multiple(ranges) => { + for range in ranges.iter_mut() { + if range.base == base && range.end == end { + range.generation = generation; + range.space = space; + } + } + } + } + } + }); + if matches!(generation, HeapGeneration::Old) { + register_old_block_pages(base, size); + } + invalidate_generation_cache(); +} + +/// Bulk registration of a run of freshly-promoted old-gen objects that share +/// one 4 KiB page, in **address order**, onto a page whose object list this +/// promotion is the sole author of. +/// +/// This is the whole-block-promotion twin of +/// [`flush_deferred_old_page_registrations_batch`]. It exists because that path +/// still pays, per object, an `entry(page)` hash lookup, a `contains` scan of +/// the page's pre-batch list and a `refresh_policy_bits` recompute — costs that +/// are amortisable when the caller knows, as here, that it is filling one page +/// once from a linear walk. `headers` is appended wholesale and the meta is +/// updated once for the whole run. +/// +/// `bytes` is the number of bytes of the run that fall inside `page` (an object +/// straddling a page boundary contributes its overlap to each page it touches), +/// matching `update_old_page_meta_for_object`'s accounting exactly. +pub(crate) fn register_promoted_page_run(page: usize, headers: &[usize], bytes: usize) { + if headers.is_empty() { + return; + } + OLD_GEN_PAGE_OBJECTS.with(|index| { + let mut index = index.borrow_mut(); + index + .entry(page) + .or_insert_with(Vec::new) + .extend_from_slice(headers); + }); + OLD_GEN_PAGE_META.with(|meta| { + let mut meta = meta.borrow_mut(); + let page_meta = meta + .entry(page) + .or_insert_with(|| OldPageMeta::zero_for_page(page)); + page_meta.allocated_bytes = page_meta.allocated_bytes.saturating_add(bytes); + page_meta.object_count = page_meta.object_count.saturating_add(headers.len()); + page_meta.live_bytes = page_meta.live_bytes.saturating_add(bytes); + page_meta.live_object_count = page_meta.live_object_count.saturating_add(headers.len()); + page_meta.refresh_policy_bits(); + }); +} + pub(crate) fn unregister_block_generation(base: usize, size: usize) { if base == 0 || size == 0 { return; @@ -571,13 +676,29 @@ fn classify_heap_generation_uncached(addr: usize, key: usize) -> HeapGeneration #[inline] pub(crate) fn classify_heap_space(addr: usize) -> HeapSpace { + classify_heap_space_in_range(addr).map_or(HeapSpace::Unknown, |(space, _)| space) +} + +/// [`classify_heap_space`] plus the base of the registered range `addr` fell +/// in. +/// +/// The base is what lets a caller answer a SECOND classification — the one for +/// `addr - GC_HEADER_SIZE`, which every object-classifying path needs — with a +/// bounds compare instead of a second map lookup. A header is on the same +/// registered range as its user pointer for every real object (a block always +/// begins with a header, so a user pointer is never within `GC_HEADER_SIZE` of +/// a range base); the base is precisely the guard that keeps a *garbage* +/// candidate address at the very start of a range from turning into a read of +/// the unmapped page below it (#7742). +#[inline] +pub(crate) fn classify_heap_space_in_range(addr: usize) -> Option<(HeapSpace, usize)> { if addr == 0 { - return HeapSpace::Unknown; + return None; } let key = generation_class_key_for_addr(addr); // SAFETY: thread-local, single-threaded, and the borrow ends here. if let Some(range) = unsafe { (*hot_page_generation_cache()).lookup(key, addr) } { - return range.space; + return Some((range.space, range.base)); } let found = { @@ -587,9 +708,9 @@ pub(crate) fn classify_heap_space(addr: usize) -> HeapSpace { if let Some(range) = found { // SAFETY: as above. unsafe { (*hot_page_generation_cache()).insert(key, range) }; - range.space + Some((range.space, range.base)) } else { - HeapSpace::Unknown + None } } diff --git a/crates/perry-runtime/src/arena/promote.rs b/crates/perry-runtime/src/arena/promote.rs new file mode 100644 index 0000000000..0d13752d12 --- /dev/null +++ b/crates/perry-runtime/src/arena/promote.rs @@ -0,0 +1,475 @@ +//! Whole-block in-place promotion of the young generation (#7742). +//! +//! # Why +//! +//! The copying minor relocates every survivor object by object. Each move +//! costs a fresh old-gen allocation, a `memcpy`, a `layout_transfer`, per-object +//! old-page accounting, the `GcMoveHookKind` hooks, and a forwarding stub — and +//! then every referring slot has to be rewritten to the new address. That price +//! is worth paying when most of the nursery is garbage: the survivors are few +//! and compaction buys back the whole block. +//! +//! It is worth nothing at all when the nursery is *fully live*. `retain.ts` +//! (3M records, none ever dropped) measured **1.000** young-survival on every +//! copying minor, so every one of its 2.1M promotions paid ~240 ns of +//! bookkeeping to move an object that had no reason to move. +//! +//! Whole-block promotion is the standard answer (V8 calls it page promotion): +//! when a block is (near-)entirely live, relabel the block as old-gen instead +//! of evacuating it object by object. Nothing moves, so there is no copy, no +//! allocation, no layout transfer, no move hook, no forwarding pointer, and — +//! because addresses are unchanged — **no slot anywhere in the heap or in any +//! address-keyed runtime side table needs rewriting**. +//! +//! # Shape of the operation +//! +//! Promotion is decided *per cycle*, not per block, because a block's liveness +//! is not knowable before the trace and block identity does not survive a reset +//! (Eden blocks are recycled at offset 0). What IS knowable is the previous +//! cycle's measured young-survival ratio, and that ratio is re-measured on +//! every cycle including the promoting ones — see `gc::promote_in_place`. So a +//! misprediction costs at most one nursery's worth of retained garbage before +//! the policy corrects itself. +//! +//! The two halves: +//! +//! * [`retag_young_for_in_place_promotion`] runs *before* the trace. It flips +//! every in-use Eden and active-survivor block's page range to generation +//! `Old`, space [`HeapSpace::PromotedYoung`]. From that instant the barrier +//! predicates (`barrier_parent_needs_remembering`, +//! `remembered_child_needs_tracking`) treat those objects as old — which is +//! what makes the trace record the right remembered-set edges — while +//! `classify_arena` can still tell they were young at cycle start and so must +//! be traced once. The blocks stay in their own arenas for the duration, so +//! no old-gen allocation can land in them mid-cycle. +//! * [`finish_in_place_promotion`] runs where the from-space reset would have. +//! It walks each block linearly (cheap: one `size` hop per object, no +//! hashing), stamps `GC_FLAG_TENURED` on every header so the `Old ⟹ TENURED` +//! invariant the generated write barrier's fast path relies on (#7511) still +//! holds, registers the **marked** (live) objects into the old-gen page index +//! in per-page bulk runs, moves the block into `OLD_ARENA` leaving a +//! tombstone behind, and retags the range to plain [`HeapSpace::Old`]. +//! +//! # What is deliberately NOT done +//! +//! Unmarked objects on a promoted block are dead. They are neither swept nor +//! registered — they stay as old-gen garbage until the next full mark-sweep, +//! which finds them through the ordinary linear old-arena walk. That retention +//! is the entire footprint cost of the technique, it is bounded by +//! `(1 − liveness) × young bytes` per cycle, and the policy layer both caps its +//! running total and reports it in the GC trace. + +use super::page_meta::{ + generation_page_base, register_promoted_page_run, retag_block_space, GENERATION_PAGE_SIZE, +}; +use super::*; + +/// One block captured by [`retag_young_for_in_place_promotion`], to be finished +/// by [`finish_in_place_promotion`]. +#[derive(Clone, Copy, Debug)] +struct PromotedBlock { + /// Which arena the block still lives in, so the finish walk can take it out. + source: PromotionSource, + /// Index into that arena's `blocks`. + index: usize, + base: usize, + size: usize, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PromotionSource { + Eden, + Survivor(usize), +} + +/// The set of young blocks this cycle promotes in place. Created before the +/// trace, consumed after it. +#[derive(Default)] +pub(crate) struct InPlacePromotion { + blocks: Vec, + /// Reserved bytes the young generation is about to lose to old-gen. + /// + /// This is NOT bookkeeping trivia — it is the whole of the pacing contract. + /// The arena-bytes trigger fires on `arena_total_bytes()`, and the runway + /// between two collections is therefore "Eden's free capacity + the trigger + /// step". A copying minor recycles Eden's blocks, so that capacity survives + /// the collection and the runway stays wide. Promotion HANDS THOSE BLOCKS + /// AWAY, so without giving the capacity back the runway collapses to the + /// bare step. Measured on `retain.ts`: Eden's high-water fell 57 MB → 18 MB, + /// collections went 6 → 12, and the extra pressure bought a second full + /// collection at 690 ms — a 0.81 s → 1.52 s regression from a change that + /// made every individual promotion cheaper. + /// + /// The capacity is given back as trigger HEADROOM + /// (`gc::note_promoted_young_capacity`), not as eagerly mapped blocks. + /// Re-reserving the blocks up front also fixes the pacing, but it maps + /// memory the program may never reach: measured on `retain_wide.ts`, the + /// eager form ratcheted Eden to 89 MB and left peak RSS at 470 MB against + /// the baseline's 447 MB. Headroom costs nothing until the mutator actually + /// allocates into it. + reserved_bytes: usize, +} + +impl InPlacePromotion { + pub(crate) fn is_empty(&self) -> bool { + self.blocks.is_empty() + } + + pub(crate) fn block_count(&self) -> usize { + self.blocks.len() + } + + /// Young capacity this promotion hands to old-gen — see + /// [`InPlacePromotion::reserved_bytes`]. + pub(crate) fn reserved_bytes(&self) -> usize { + self.reserved_bytes + } +} + +/// Per-block liveness, produced by the finish walk. This is the measurement the +/// promotion policy is calibrated against — `PERRY_GC_DIAG=1` prints it. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct InPlacePromotionStats { + pub(crate) blocks: usize, + pub(crate) objects: usize, + pub(crate) live_objects: usize, + pub(crate) bytes: usize, + pub(crate) live_bytes: usize, + /// Blocks whose live fraction was below 50% — the shape that would have + /// been better served by evacuation. Zero on every benchmark measured. + pub(crate) sparse_blocks: usize, +} + +/// Retag every in-use young block as old-gen `PromotedYoung`. +/// +/// Returns the blocks captured. An empty result means there was nothing to +/// promote and the caller must fall back to the ordinary copying path (in +/// particular it must NOT skip the from-space reset). +pub(crate) fn retag_young_for_in_place_promotion() -> InPlacePromotion { + sync_inline_arena_state(); + let mut promotion = InPlacePromotion::default(); + + ARENA.with(|arena| { + let arena = unsafe { &*arena.get() }; + collect_blocks(&arena.blocks, PromotionSource::Eden, &mut promotion); + }); + // BOTH semispaces, not just the active one. The inactive space is to-space + // and is empty at cycle start, so this normally captures nothing — but + // taking every in-use young block unconditionally is what makes "after the + // retag, no address in the heap classifies as `Nursery`" true BY + // CONSTRUCTION rather than by an argument about semispace bookkeeping. Two + // provable no-op eliminations downstream rest on exactly that claim. + for idx in [0usize, 1usize] { + with_survivor_arena(idx, |arena| { + collect_blocks( + &arena.blocks, + PromotionSource::Survivor(idx), + &mut promotion, + ); + }); + } + + for block in &promotion.blocks { + retag_block_space( + block.base, + block.size, + HeapGeneration::Old, + HeapSpace::PromotedYoung, + ); + } + promotion +} + +/// Bytes still in use in blocks that classify as young generation. The premise +/// `gc::copying::debug_assert_no_remembering_possible` re-derives: after +/// [`retag_young_for_in_place_promotion`] this must be zero. +#[cfg(debug_assertions)] +pub(crate) fn young_in_use_bytes_after_retag() -> usize { + let mut total = 0usize; + let mut count = |blocks: &[ArenaBlock]| { + for block in blocks { + if block.data.is_null() || block.offset == 0 { + continue; + } + if matches!( + super::page_meta::classify_heap_space(block.data as usize), + HeapSpace::NurseryEden | HeapSpace::Survivor0 | HeapSpace::Survivor1 + ) { + total = total.saturating_add(block.offset); + } + } + }; + ARENA.with(|arena| count(&unsafe { &*arena.get() }.blocks)); + for idx in [0usize, 1usize] { + with_survivor_arena(idx, |arena| count(&arena.blocks)); + } + total +} + +fn collect_blocks(blocks: &[ArenaBlock], source: PromotionSource, out: &mut InPlacePromotion) { + for (index, block) in blocks.iter().enumerate() { + if block.data.is_null() || block.offset == 0 { + continue; + } + out.blocks.push(PromotedBlock { + source, + index, + base: block.data as usize, + size: block.size, + }); + out.reserved_bytes = out.reserved_bytes.saturating_add(block.size); + } +} + +/// Finish the promotion: stamp headers, index the live objects, hand the blocks +/// to `OLD_ARENA`, and leave both young regions empty. +/// +/// MUST run before `CopyingNurseryCollector::clear_marks` — liveness here is +/// `GC_FLAG_MARKED | GC_FLAG_PINNED` on the still-marked headers. +pub(crate) fn finish_in_place_promotion(promotion: InPlacePromotion) -> InPlacePromotionStats { + let mut stats = InPlacePromotionStats { + blocks: promotion.blocks.len(), + ..InPlacePromotionStats::default() + }; + if promotion.blocks.is_empty() { + return stats; + } + + // #7624: the page-objects index must be complete before this adds to it, + // for the same reason every other reader flushes — a pending registration + // for one of these pages would otherwise be folded in afterwards and land + // behind the run this walk appends. + super::page_meta::flush_deferred_old_page_registrations(); + + let mut moved_blocks: Vec = Vec::with_capacity(promotion.blocks.len()); + for block in &promotion.blocks { + let taken = take_block(*block); + let Some(taken) = taken else { continue }; + let (objects, live_objects, live_bytes) = stamp_and_index_block(&taken); + stats.objects += objects; + stats.live_objects += live_objects; + stats.bytes += taken.offset; + stats.live_bytes += live_bytes; + if taken.offset > 0 && live_bytes * 2 < taken.offset { + stats.sparse_blocks += 1; + } + moved_blocks.push(taken); + } + + // Hand the blocks to old-gen. The page range is retagged from + // `PromotedYoung` to plain `Old` here rather than in the walk above so that + // nothing observes a block that is simultaneously listed in `OLD_ARENA` and + // labelled young. + OLD_ARENA.with(|old| { + let old = unsafe { &mut *old.get() }; + for block in moved_blocks { + let base = block.data as usize; + let size = block.size; + let offset = block.offset; + super::block::old_gen_in_use_bytes_add(offset); + install_block_into(old, block); + retag_block_space(base, size, HeapGeneration::Old, HeapSpace::Old); + } + }); + + // Both young regions are empty now. Eden needs a usable current block for + // the inline bump allocator; the survivor flip keeps the semispace + // alternation the copying path maintains. + reset_young_after_promotion(); + stats +} + +/// Detach the block from its owning arena, leaving the `data = null, size = 0` +/// tombstone every arena path already tolerates (C4b-δ leaves the same shape), +/// so block indices stay stable across the cycle. +fn take_block(block: PromotedBlock) -> Option { + let detach = |arena: &mut Arena| -> Option { + // The index recorded at retag time is right in every case that can + // occur — no arena path reorders `blocks`, it only fills tombstones or + // pushes. Confirm it anyway and fall back to a search, because the + // alternative to finding this block is leaving an `Old`-registered + // block sitting in a young arena, where the next reset would zero live + // data. There is no safe "give up" branch here. + let found = match arena.blocks.get(block.index) { + Some(slot) if slot.data as usize == block.base => Some(block.index), + _ => arena + .blocks + .iter() + .position(|slot| slot.data as usize == block.base), + }; + let index = found.unwrap_or_else(|| { + panic!( + "in-place promotion cannot find the block it retagged \ + (base={:#x} size={}); it is registered as old-gen but still \ + owned by a young arena", + block.base, block.size + ) + }); + Some(std::mem::replace( + &mut arena.blocks[index], + ArenaBlock { + data: std::ptr::null_mut(), + size: 0, + offset: 0, + dead_cycles: 0, + }, + )) + }; + match block.source { + PromotionSource::Eden => ARENA.with(|arena| detach(unsafe { &mut *arena.get() })), + PromotionSource::Survivor(idx) => with_survivor_arena_mut(idx, detach), + } +} + +fn install_block_into(arena: &mut Arena, block: ArenaBlock) { + if let Some(slot) = arena.blocks.iter_mut().find(|slot| slot.data.is_null()) { + *slot = block; + return; + } + arena.blocks.push(block); +} + +/// Linear walk of one promoted block: stamp `GC_FLAG_TENURED` on every header, +/// and register the live ones with the old-gen page index in per-page bulk +/// runs. Returns `(objects, live_objects, live_bytes)`. +/// +/// The walk parses the block exactly the way `arena_walk_objects` and +/// `old_arena_walk_objects` do — hop by `GcHeader::size`, stop on an +/// implausible one. A block that does not parse to its own `offset` would leave +/// its tail unstamped and unindexed, which is a missed old→young edge; it is +/// also a block the existing sweep walkers could not parse either, so it is a +/// pre-existing whole-heap invariant rather than something this path can repair. +/// `debug_assert` catches it in CI instead of letting it be silent. +fn stamp_and_index_block(block: &ArenaBlock) -> (usize, usize, usize) { + use crate::gc::GcHeader; + + let mut objects = 0usize; + let mut live_objects = 0usize; + let mut live_bytes = 0usize; + + // One page's worth of live headers, flushed whenever the page changes. The + // walk is in address order, so a page's headers are contiguous in it. + let mut run_page: Option = None; + let mut run_headers: Vec = Vec::new(); + let mut run_bytes = 0usize; + + let mut offset = 0usize; + while offset < block.offset { + let aligned = (offset + 7) & !7; + if aligned >= block.offset { + break; + } + let header_ptr = unsafe { block.data.add(aligned) }; + let header = header_ptr as *mut GcHeader; + let total = unsafe { (*header).size } as usize; + if total < crate::gc::GC_HEADER_SIZE || total > block.size - aligned { + // Same guard the arena walkers use: an implausible size means we + // have run off the end of the initialised region. + break; + } + let obj_type = unsafe { (*header).obj_type }; + let flags = unsafe { (*header).gc_flags }; + // `Old ⟹ TENURED` (#7511): the generated barrier's fast path skips the + // whole remembering call when the parent header has no TENURED bit, so + // every object on a block that is about to be old-gen must carry it — + // dead ones included, since the walk cannot prove a header will never + // be looked at again. + unsafe { + crate::gc::stamp_header_promoted_in_place(header); + } + objects += 1; + + let live = flags & (crate::gc::GC_FLAG_MARKED | crate::gc::GC_FLAG_PINNED) != 0; + if live && crate::gc::gc_type_is_arena_walkable(obj_type) { + live_objects += 1; + live_bytes += total; + let header_addr = header_ptr as usize; + let object_end = header_addr + total; + let first_page = generation_page_for_addr(header_addr); + let last_page = generation_page_for_addr(object_end - 1); + for page in first_page..=last_page { + let page_base = generation_page_base(page); + let page_end = page_base + GENERATION_PAGE_SIZE; + let overlap_start = header_addr.max(page_base); + let overlap_end = object_end.min(page_end); + if overlap_start >= overlap_end { + continue; + } + if run_page != Some(page) { + if let Some(previous) = run_page { + register_promoted_page_run(previous, &run_headers, run_bytes); + } + run_page = Some(page); + run_headers.clear(); + run_bytes = 0; + } + run_headers.push(header_addr); + run_bytes += overlap_end - overlap_start; + } + } + offset = aligned + total; + } + if let Some(previous) = run_page { + register_promoted_page_run(previous, &run_headers, run_bytes); + } + debug_assert_eq!( + offset, block.offset, + "a promoted block did not parse to its own bump offset — its tail is \ + neither TENURED-stamped nor indexed, which is a missed old->young \ + edge (and a block the sweep walkers could not parse either)" + ); + (objects, live_objects, live_bytes) +} + +/// Put Eden back into a usable state and flip the (now both empty) survivor +/// semispaces, mirroring what `copying_reset_from_spaces_and_flip` leaves +/// behind. +/// +/// Eden is left with ONE usable block; the capacity the promotion gave away is +/// handed back as trigger headroom instead of as mapped blocks (see +/// `InPlacePromotion::reserved_bytes`), so the mutator maps only what it +/// actually allocates. +fn reset_young_after_promotion() { + crate::gc::ARENA_FREE_LIST.with(|fl| fl.borrow_mut().clear()); + crate::gc::ARENA_FREE_LIST_NONEMPTY.with(|c| c.set(false)); + + ARENA.with(|arena| unsafe { + let arena = &mut *arena.get(); + for block in arena.blocks.iter_mut() { + block.offset = 0; + block.dead_cycles = 0; + } + // `install_fresh_block` allocates through the NO-GC path + // (`alloc_block_no_gc`) — the collecting `reserve_arena_block` would + // start a second collection from inside this one. + if arena.blocks.iter().all(|block| block.data.is_null()) { + arena.install_fresh_block(BLOCK_SIZE); + } + arena.current = arena + .blocks + .iter() + .position(|block| !block.data.is_null()) + .unwrap_or(0); + INLINE_STATE.with(|s| { + let inline = &mut *s.get(); + if !inline.data.is_null() { + let block = &arena.blocks[arena.current]; + inline.data = block.data; + inline.offset = block.offset; + inline.size = block.size; + } + }); + }); + + for idx in [0usize, 1usize] { + with_survivor_arena_mut(idx, |arena| { + for block in arena.blocks.iter_mut() { + block.offset = 0; + block.dead_cycles = 0; + } + arena.current = 0; + }); + } + let active = ACTIVE_SURVIVOR.with(|active| active.get()); + ACTIVE_SURVIVOR.with(|active_cell| active_cell.set(1 - active)); +} diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index acc46fa762..76026f898c 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -1248,6 +1248,14 @@ fn deferred_registration_flush_sites() { must not get a repopulated one", ), ("defer_old_object_page_registration", "the producer"), + ( + "register_promoted_page_run", + "#7742: called once per PAGE from `finish_in_place_promotion`'s \ + single linear walk of a promoted block, which flushes once before \ + the whole walk. Flushing per call would be the same flush repeated \ + 256 times per 1 MiB block — and cannot be needed, because nothing \ + between the walk's start and its end allocates into old-gen", + ), ( "flush_deferred_old_page_registrations", "the flush entry point", diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 1d8afd9552..b6af6fab5e 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -8,6 +8,11 @@ pub(super) enum CopyingPointerKind { Longlived, Old, Malloc, + /// On a block this cycle is promoting whole, in place (#7742). Generation + /// is already `Old` — so every barrier predicate reads old-gen semantics — + /// but the object was young when the cycle began and therefore still owes + /// the collector exactly one field scan. + PromotedYoung, } #[derive(Clone, Copy)] @@ -81,18 +86,34 @@ impl CopyingPointerSet { if addr < GC_HEADER_SIZE { return None; } - let space = crate::arena::classify_heap_space(addr); - if matches!(space, crate::arena::HeapSpace::Unknown) { + // ONE range lookup answers both classifications this needs. The header + // sits `GC_HEADER_SIZE` below the user pointer and a block always + // begins with a header, so a real object's header is on the same + // registered range as its user pointer; `range_base` is the guard that + // keeps a garbage candidate sitting at the very start of a range from + // becoming a read of the unmapped page below it. Before #7742 this was + // two `classify_heap_space` calls for addresses 8 bytes apart, on + // EVERY visited slot. + let Some((space, range_base)) = crate::arena::classify_heap_space_in_range(addr) else { return None; - } + }; let header_addr = addr - GC_HEADER_SIZE; - if !matches!( + if header_addr < range_base { + return None; + } + debug_assert_eq!( crate::arena::classify_heap_space(header_addr), + space, + "an object's header and user pointer must classify identically" + ); + if !matches!( + space, crate::arena::HeapSpace::NurseryEden | crate::arena::HeapSpace::Survivor0 | crate::arena::HeapSpace::Survivor1 | crate::arena::HeapSpace::Longlived | crate::arena::HeapSpace::Old + | crate::arena::HeapSpace::PromotedYoung ) { return None; } @@ -104,6 +125,7 @@ impl CopyingPointerSet { let inactive_survivor = crate::arena::inactive_survivor_space(); let kind = match space { crate::arena::HeapSpace::NurseryEden => CopyingPointerKind::Eden, + crate::arena::HeapSpace::PromotedYoung => CopyingPointerKind::PromotedYoung, s if s == active_survivor => CopyingPointerKind::FromSurvivor, s if s == inactive_survivor => CopyingPointerKind::ToSurvivor, crate::arena::HeapSpace::Longlived => CopyingPointerKind::Longlived, @@ -404,6 +426,26 @@ pub(super) struct CopyingNurseryCollector { /// address-dependent — the gc-ratchet's bit-identical-counters contract /// caught it as a ±2-object jitter on the first heavy cycle. pub(super) tenuring_survivals: u8, + /// #7742: every remembered-set insertion this cycle could make is provably + /// impossible, so the passes that make them are skipped. + /// + /// A remembered-set entry is only ever created when + /// `remembered_child_needs_tracking(child)` says yes, and that is yes for + /// exactly two child populations: nursery-generation objects, and + /// malloc-registry objects. On a whole-block promoting cycle the first + /// population is EMPTY by construction — `retag_young_for_in_place_promotion` + /// takes every in-use Eden and survivor block, so after the retag nothing in + /// the heap classifies as `Nursery`. When the malloc registry was also empty + /// at cycle start the second population is empty too, and no mutator runs + /// mid-cycle to create one. + /// + /// So this is a proof, not a heuristic: three whole passes over the + /// surviving cohort's slots (`visit_slot_with_parent`'s re-decode + + /// remember, `rebuild_evacuated_old_to_young_remembered_set`, and + /// `restore_surviving_dirty_coverage`) can only insert nothing, and are + /// skipped. `debug_assert_no_remembering_needed` re-derives the premise at + /// runtime in debug builds. + pub(super) skip_remembering: bool, /// Weak target slots (WeakRef referent / WeakMap-WeakSet entry key / /// FinalizationRegistry record target) seen during the copy scan. The /// scan must NOT evacuate through them (that would strengthen the weak @@ -433,6 +475,7 @@ impl CopyingNurseryCollector { }, live_from_bytes: 0, tenuring_survivals, + skip_remembering: false, weak_slots: Vec::new(), } } @@ -561,7 +604,47 @@ impl CopyingNurseryCollector { } Some(addr) } + CopyingPointerKind::PromotedYoung => Some(unsafe { self.mark_promoted_young(ptr) }), + } + } + + /// #7742: the object's block is being promoted whole, in place. It does not + /// move, so this is a pure mark — the address it is already at is its final + /// address, and every slot in the heap that points at it is already + /// correct. + /// + /// It still goes on the worklist, and on `moved_headers`: it was young when + /// the cycle began, so it owes exactly one field scan (to evacuate any + /// child that is NOT on a promoted block, and to record the old→young and + /// old→malloc remembered-set edges its new generation implies), and the + /// mark has to be cleared at the end like any other. + pub(super) unsafe fn mark_promoted_young(&mut self, ptr: CopyingPointer) -> usize { + let header = ptr.header; + let user = (header as *mut u8).add(GC_HEADER_SIZE) as usize; + let flags = (*header).gc_flags; + if flags & GC_FLAG_FORWARDED != 0 { + // Array growth leaves a forwarding stub at the pre-grow address; + // follow it exactly as `move_young` does. + let forwarded = forwarding_address(header) as usize; + return self.mark_addr(forwarded).unwrap_or(forwarded); + } + if flags & GC_FLAG_MARKED == 0 { + (*header).gc_flags = flags | GC_FLAG_MARKED; + let total = (*header).size as usize; + self.worklist.push(header); + self.moved_headers.push(header); + self.stats.promoted_objects += 1; + self.stats.promoted_bytes += total; + self.stats.in_place_promoted_objects += 1; + self.live_from_bytes += total; + // Survivor-influx accounting: an in-place promotion consumes the + // whole young generation at once, so the split the adaptive + // tenuring loop reads has no meaning here. Everything is credited + // as Eden influx, which is what keeps `tenuring_survivals` pinned + // low for the workloads this path fires on. + self.stats.eden_live_bytes += total; } + user } pub(super) unsafe fn move_young(&mut self, ptr: CopyingPointer) -> usize { @@ -717,7 +800,7 @@ impl CopyingNurseryCollector { if let Some(new_bits) = self.visit_value_bits(bits) { *slot = new_bits; } - if !parent_header.is_null() { + if !parent_header.is_null() && !self.skip_remembering { let parent_user = (parent_header as *mut u8).add(GC_HEADER_SIZE) as usize; if barrier_parent_needs_remembering(parent_user, external) { if let Some((child_addr, _, _)) = self.ptrs.decode_bits(*slot) { @@ -1097,6 +1180,28 @@ impl CopiedMinorEligibility { } } +/// Re-derive `skip_remembering`'s premise from the heap itself, in debug +/// builds: no in-use young block survived the retag, and the malloc registry is +/// empty. Either being false would make three skipped passes non-empty and turn +/// a dropped remembered-set entry into a swept-live-object crash a cycle later, +/// so it is worth re-deriving rather than trusting the argument. +fn debug_assert_no_remembering_possible() { + #[cfg(debug_assertions)] + { + let young_in_use = crate::arena::young_in_use_bytes_after_retag(); + debug_assert_eq!( + young_in_use, 0, + "in-place promotion left {young_in_use} bytes of young generation in use; \ + `skip_remembering` would drop real old->young remembered-set entries" + ); + let malloc_objects = MALLOC_STATE.with(|s| s.borrow().objects.len()); + debug_assert_eq!( + malloc_objects, 0, + "malloc registry is non-empty; `skip_remembering` would drop old->malloc edges" + ); + } +} + pub(super) fn gc_collect_minor_copying_fast_path( trace: &mut Option, start: Instant, @@ -1152,11 +1257,34 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( let phase_start = trace_phase_start(trace); let from_space_bytes = crate::arena::copying_from_space_in_use_bytes(); + // #7742: decide BEFORE anything classifies, then retag the young blocks so + // every classification for the rest of this cycle already reads the + // generation those objects will have when it ends. The eligibility + // preflight above ran against the pre-retag labels, which is correct — it + // answers "may this cycle move objects at all", a question the retag does + // not change. + let promotion = if super::should_promote_young_in_place() { + crate::arena::retag_young_for_in_place_promotion() + } else { + crate::arena::InPlacePromotion::default() + }; + // An empty plan (nothing in use to promote) falls back to the ordinary + // path, so the from-space reset still runs. + let promoting_in_place = !promotion.is_empty(); let mut collector = CopyingNurseryCollector::new(ptrs); collector.stats.eligible = true; collector.stats.fallback_reason = CopiedMinorFallbackReason::None; collector.stats.malloc_sweep_due = malloc_sweep_due; collector.stats.preflight_skipped = preflight_skipped; + collector.stats.in_place_promotion = promoting_in_place; + collector.stats.in_place_promoted_blocks = promotion.block_count(); + // See `CopyingNurseryCollector::skip_remembering` for the proof. + collector.skip_remembering = + promoting_in_place && collector.ptrs.malloc_registry_empty_at_start; + if collector.skip_remembering { + debug_assert_no_remembering_possible(); + } + collector.stats.remembering_skipped = collector.skip_remembering; collector.stats.reset_blocks += crate::arena::copying_prepare_to_space(); let native_stack_walk = visit_mutable_root_slots(|slot| unsafe { @@ -1234,9 +1362,12 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( if let Some(trace) = trace.as_mut() { trace.remembered_set = remembered_stats; } - let promoted_sticky = rebuild_evacuated_old_to_young_remembered_set(&collector.moved_headers); - promoted_sticky.restore(); - collector.sticky.extend(promoted_sticky); + if !collector.skip_remembering { + let promoted_sticky = + rebuild_evacuated_old_to_young_remembered_set(&collector.moved_headers); + promoted_sticky.restore(); + collector.sticky.extend(promoted_sticky); + } if gc_verify_evacuation_enabled() { let phase_start = trace_phase_start(trace); let old_young_edge_verifier = verify_old_to_young_edges_covered(); @@ -1337,14 +1468,44 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( crate::promise::cleanup_copied_minor_promise_contexts_for_gc(); finalize_dead_copied_minor_from_space_side_allocations(); - let reset = crate::arena::copying_reset_from_spaces_and_flip(); + // #7742: on a promoting cycle the young blocks are handed to old-gen + // instead of being reset. This MUST stay before `clear_marks` — the finish + // walk reads `GC_FLAG_MARKED` to decide which objects to index — and it + // takes the place of, never runs alongside, the from-space reset: the + // blocks the reset would recycle are the blocks this keeps. + let (reset, promotion_stats) = if promoting_in_place { + let phase_start = trace_phase_start(trace); + super::note_promoted_young_capacity(promotion.reserved_bytes()); + let promotion_stats = crate::arena::finish_in_place_promotion(promotion); + trace_phase_record(trace, "in_place_promotion", phase_start); + ( + crate::arena::ArenaResetStats { + reset_blocks: 0, + reusable_bytes: 0, + deallocated_blocks: 0, + deallocated_bytes: 0, + }, + promotion_stats, + ) + } else { + ( + crate::arena::copying_reset_from_spaces_and_flip(), + crate::arena::InPlacePromotionStats::default(), + ) + }; collector.stats.reset_blocks += reset.reset_blocks; + collector.stats.in_place_dead_bytes = promotion_stats + .bytes + .saturating_sub(promotion_stats.live_bytes); + collector.stats.in_place_sparse_blocks = promotion_stats.sparse_blocks; if let Some(trace) = trace.as_mut() { trace.old_pages = crate::arena::old_page_summary(); } remembered_set_clear(); collector.sticky.restore(); - restore_surviving_dirty_coverage(&snapshot); + if !collector.skip_remembering { + restore_surviving_dirty_coverage(&snapshot); + } let malloc_freed_bytes = if malloc_sweep_due { let phase_start = trace_phase_start(trace); let freed = sweep_malloc_objects(); @@ -1358,7 +1519,26 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( } CONS_PINNED.with(|s| s.borrow_mut().clear()); - let nursery_freed_bytes = from_space_bytes.saturating_sub(collector.live_from_bytes) as u64; + // #7742: feed the policy its measurement. This runs on EVERY copying minor + // — promoting ones included, which is the whole reason a promoting cycle + // still traces — so the ratio the next decision reads is never stale. + super::note_young_survival(from_space_bytes, collector.live_from_bytes); + collector.stats.young_survival_permille = + super::last_young_survival_permille().unwrap_or_default(); + // A promoting cycle frees NOTHING: the dead young bytes were promoted + // along with the live ones and are reclaimable only by a full collection. + // Reporting them as freed would tell the pacer it had made progress it had + // not made. + let nursery_freed_bytes = if promoting_in_place { + super::note_in_place_promotion( + from_space_bytes, + collector.live_from_bytes, + collector.stats.in_place_promoted_objects, + ); + 0 + } else { + from_space_bytes.saturating_sub(collector.live_from_bytes) as u64 + }; let freed_bytes = nursery_freed_bytes.saturating_add(malloc_freed_bytes); collector.stats.malloc_validation_lookups = collector.ptrs.malloc_validation_lookups(); collector.stats.malloc_registry_rebuilds = collector.ptrs.malloc_registry_rebuilds(); @@ -1406,7 +1586,12 @@ pub(super) fn gc_collect_minor_copying_fast_path_with_eligibility( ); if std::env::var_os("PERRY_GC_DIAG").is_some() { eprintln!( - "[gc-copy-minor] ran copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} tenuring_survivals={} eden_live_bytes={} trigger={:?} declared_safepoint={}", + "[gc-copy-minor] ran in_place={} in_place_blocks={} in_place_dead_bytes={} sparse_blocks={} survival_permille={} copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} tenuring_survivals={} eden_live_bytes={} trigger={:?} declared_safepoint={}", + collector.stats.in_place_promotion, + collector.stats.in_place_promoted_blocks, + collector.stats.in_place_dead_bytes, + collector.stats.in_place_sparse_blocks, + collector.stats.young_survival_permille, collector.stats.copied_objects, collector.stats.copied_bytes, collector.stats.promoted_objects, diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index aa627329de..0422eb48fa 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -567,6 +567,30 @@ pub(super) fn reserved_with_copied_survival_age(reserved: u16, age: u8) -> u16 { (reserved & !GC_COPY_SURVIVAL_AGE_MASK) | (capped << GC_COPY_SURVIVAL_AGE_SHIFT) } +/// Stamp a header the way `move_young`'s promoting arm stamps its to-space +/// copy — except that whole-block promotion (#7742) has no copy, so the SAME +/// header is aged in place. +/// +/// Both halves matter. `GC_FLAG_TENURED` upholds the `Old ⟹ TENURED` +/// invariant the generated write barrier's fast path is gated on (#7511): +/// without it, a store into a promoted object would skip the remembering call +/// entirely and its young child would be swept alive. Clearing +/// `GC_FLAG_HAS_SURVIVED` and pinning the survival age to +/// `GC_COPY_PROMOTION_SURVIVALS` keeps `copied_survival_age` reading the same +/// value it would have read off an evacuated copy, so nothing downstream can +/// tell a promoted-in-place object from a promoted-by-copy one. +/// +/// # Safety +/// `header` must point at a live `GcHeader` inside a block that is being +/// promoted to old-gen this cycle. +#[inline] +pub(crate) unsafe fn stamp_header_promoted_in_place(header: *mut GcHeader) { + let flags = (*header).gc_flags; + (*header).gc_flags = (flags | GC_FLAG_TENURED) & !GC_FLAG_HAS_SURVIVED; + (*header)._reserved = + reserved_with_copied_survival_age((*header)._reserved, GC_COPY_PROMOTION_SURVIVALS); +} + #[inline] pub(super) fn strip_nanbox_user_ptr(bits: u64) -> usize { if (bits >> 48) >= 0x7FF8 { diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 0bf5995e3f..5386a917d5 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -121,6 +121,11 @@ use old_free::*; pub(crate) use old_free::{old_free_bytes, old_free_filter_range, old_free_take_exact}; mod tenuring; use tenuring::*; +/// #7742: the measured policy behind whole-block in-place promotion. The +/// mechanism is `arena/promote.rs`; this decides when to use it. +mod promote_in_place; +use promote_in_place::*; +pub use promote_in_place::{in_place_promoted_objects, in_place_promotion_cycles}; mod oldgen; use oldgen::*; mod oldgen_defrag; diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index d00d8dca9d..0cea490ff6 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1529,6 +1529,10 @@ pub(super) fn finish_full_old_reclaim_baseline() { GC_LAST_FULL_ARENA_IN_USE_BYTES.with(|bytes| bytes.set(post_in_use)); update_major_pacing_backoff(post_in_use); GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); + // #7742: the dead bytes that whole-block promotion parked in old-gen are + // exactly what this collection just reclaimed, so the running budget that + // caps them starts over. + super::note_full_collection_reclaimed_old_gen(); } /// Percent of the pre-full live set a full must reclaim to count as productive. @@ -1824,7 +1828,13 @@ fn gc_finish_arena_trigger_collection(pre_in_use: usize, outcome: GcCollectOutco let stepped = new_total.saturating_add(step); let capped = stepped.min(gc_trigger_absolute_ceiling_bytes()); let floor = new_total.saturating_add(gc_trigger_headroom_floor_bytes()); - let next_trigger = std::cmp::max(capped, floor); + // #7742: whole-block promotion hands Eden's blocks to old-gen instead of + // recycling them, so the free young capacity that would have carried the + // mutator to the next collection is gone from `new_total`. Give it back as + // headroom (consumed once) rather than by re-reserving the blocks, which + // would map memory the program may never reach. + let next_trigger = + std::cmp::max(capped, floor).saturating_add(super::take_promoted_young_capacity_credit()); GC_NEXT_TRIGGER_BYTES.with(|c| c.set(next_trigger)); GC_TRIGGER_ARMED.with(|a| a.set(true)); // Rebaseline the malloc-count trigger only if this collection diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs new file mode 100644 index 0000000000..f6f06275b4 --- /dev/null +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -0,0 +1,258 @@ +//! Policy for whole-block in-place promotion (#7742). +//! +//! The mechanism lives in `arena/promote.rs`; this is the decision layer, and +//! the decision is made from a **measurement**, not a guess. +//! +//! # The measurement +//! +//! Every copying minor already computes `live_from_bytes` — the bytes of the +//! young generation that survived — and the from-space size it started with. +//! Their ratio is the young-survival ratio. Measured on the GC benchmark set +//! (`gc-handoff/bench`, best-of-5 on the pinned M1 mini): +//! +//! | workload | copying minors | young-survival ratio | +//! |---|--:|--:| +//! | `retain`, `retain1`, `retain_wide` | 5–7 | **0.999 – 1.000** | +//! | `deeplist` | 3 | **1.000** | +//! | `churn`, `churn_alloc`, `push_cls` | 105 | 0.000 – 0.004 | +//! | `push_num`, `cycles` | 16–18 | 0.000 | +//! | `tree`, `tree_wide`, `churn_read` | 0 | — (no copying minor runs) | +//! +//! The population is bimodal with a gap of three orders of magnitude, so the +//! threshold is not a tuning knob in any interesting sense — anything in +//! `[0.01, 0.99]` classifies this set identically. It is set at +//! [`PROMOTE_SURVIVAL_THRESHOLD_PERMILLE`] = 95% purely to bound the footprint +//! cost, which is what the constant's value is actually justified against. +//! +//! # Why a per-cycle decision is safe +//! +//! A block's liveness is not knowable before the trace, and Eden blocks are +//! recycled at offset 0 so per-block history means nothing. The decision is +//! therefore per cycle, taken from the PREVIOUS cycle's ratio. The thing that +//! makes that sound rather than optimistic is that a promoting cycle **still +//! traces**, so it measures the ratio too: the feedback never goes stale, and a +//! workload that flips from live to garbage pays at most ONE nursery of +//! retained garbage before the policy turns itself off. +//! +//! Two further bounds: +//! +//! * [`PROMOTED_DEAD_BUDGET_BYTES`] caps the running total of dead bytes +//! promoted since the last full collection. Reaching it disables in-place +//! promotion until a full runs and actually reclaims them, so a workload +//! that sits just above the threshold forever cannot bleed footprint +//! indefinitely. +//! * `PERRY_GC_FORCE_EVACUATE` / `PERRY_GC_ZEAL` turn it off outright. Those +//! knobs exist to make objects MOVE; a promoting cycle moves nothing, and an +//! instrument that silently stops exercising its subject is exactly the +//! failure mode CLAUDE.md's "a gate must assert its subject was live" rule +//! is about. + +use super::*; + +/// Young-survival ratio, in permille, at or above which the next copying minor +/// promotes the whole young generation in place instead of evacuating it. +/// +/// Chosen for footprint, not for classification (see the module docs): at 95% +/// a mispredicted cycle retains at most 5% of the young generation as old-gen +/// garbage, which against the 64 MB young-cap ceiling is ≤ 3.2 MB before the +/// re-measured ratio turns the policy off. +pub(super) const PROMOTE_SURVIVAL_THRESHOLD_PERMILLE: u64 = 950; + +/// Running cap on dead bytes promoted in place since the last full collection. +/// +/// The per-cycle bound above is self-correcting; this bounds the pathological +/// steady state it does NOT cover — a workload whose ratio sits just above the +/// threshold every single cycle, where each cycle is individually "fine" and +/// the total still grows without limit. 32 MB is half the 64 MB young-cap +/// ceiling: it lets a fully-live workload promote unboundedly (it produces zero +/// dead bytes) while capping the marginal case at a fraction of one nursery +/// before a full collection has to justify continuing. +pub(super) const PROMOTED_DEAD_BUDGET_BYTES: usize = 32 * 1024 * 1024; + +thread_local! { + /// Young-survival ratio of the most recent copying minor, in permille. + /// `None` until one has run — the first copying minor is always a real + /// evacuation, because nothing has been measured yet. + static LAST_YOUNG_SURVIVAL_PERMILLE: Cell> = const { Cell::new(None) }; + /// Dead bytes promoted in place since the last full collection. + static PROMOTED_DEAD_BYTES: Cell = const { Cell::new(0) }; + /// Cycle counters, for the trace and for tests that need to prove the + /// subject actually ran. + static IN_PLACE_PROMOTION_CYCLES: Cell = const { Cell::new(0) }; + static IN_PLACE_PROMOTED_OBJECTS: Cell = const { Cell::new(0) }; + /// Young capacity the last promotion handed to old-gen, owed back to the + /// next arena-bytes trigger as headroom. See + /// `arena::InPlacePromotion::reserved_bytes`. + static YOUNG_CAPACITY_CREDIT: Cell = const { Cell::new(0) }; +} + +/// Record that `bytes` of young capacity became old-gen, so the next +/// arena-bytes trigger restores the allocation runway a copying minor would +/// have kept by recycling those same blocks. +pub(super) fn note_promoted_young_capacity(bytes: usize) { + YOUNG_CAPACITY_CREDIT.with(|c| c.set(c.get().saturating_add(bytes))); +} + +/// Consume the credit. Read exactly once, by the post-collection trigger +/// rebaseline — leaving it set would compound across cycles into unbounded +/// headroom, which is the failure `gc_bump_arena_trigger_target`'s own comment +/// records (a trigger that ratcheted hundreds of MB above the live set and +/// never fired again). +pub(super) fn take_promoted_young_capacity_credit() -> usize { + YOUNG_CAPACITY_CREDIT.with(|c| c.replace(0)) +} + +/// `PERRY_GC_PROMOTE_IN_PLACE=0|off|false` reverts to object-by-object +/// evacuation on every cycle. Bisection escape hatch for a change that alters +/// where every surviving object lives; its OFF state is asserted by +/// `gc::tests::promote_in_place::promote_in_place_knob_parses_both_states`. +pub(super) fn promote_in_place_enabled() -> bool { + parse_promote_in_place(std::env::var("PERRY_GC_PROMOTE_IN_PLACE").ok().as_deref()) +} + +/// Pure knob parse, so both states are asserted without touching the process +/// environment (see `gc/tests/fromspace_protect.rs` for why the live readers +/// are never poked directly from a test). +/// +/// Default is ON. Only the three explicit off-spellings turn it off — a typo +/// must not silently change which collector a bisect is measuring. +pub(super) fn parse_promote_in_place(raw: Option<&str>) -> bool { + match raw { + Some(v) => !matches!( + v.trim().to_ascii_lowercase().as_str(), + "0" | "off" | "false" + ), + None => true, + } +} + +/// Should this copying minor promote the young generation whole, in place? +pub(super) fn should_promote_young_in_place() -> bool { + // In test builds the path is opt-in per thread. The unit suite drives + // `gc_collect_minor` directly and asserts object IDENTITY across it + // ("the survivor is at a new address"), so a policy keyed on the whole + // thread's Eden liveness would make those assertions depend on which other + // test allocated first. `in_place_promotion_opt_in_for_tests` turns it on + // for the tests that are ABOUT this path — and those assert the live- + // subject counters, so the production behaviour is genuinely exercised, not + // merely compiled. + #[cfg(test)] + if !TEST_OPT_IN.with(Cell::get) { + return false; + } + if !promote_in_place_enabled() { + return false; + } + // Both of these exist to make objects move. Leave them a copier to drive. + if gc_force_evacuate_enabled() || gc_zeal_enabled() { + return false; + } + if PROMOTED_DEAD_BYTES.with(Cell::get) >= PROMOTED_DEAD_BUDGET_BYTES { + return false; + } + LAST_YOUNG_SURVIVAL_PERMILLE + .with(Cell::get) + .is_some_and(|permille| permille >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE) +} + +/// Record the young-survival ratio a copying minor just measured. Called on +/// EVERY copying minor, promoting or evacuating — that is what keeps the +/// predictor from going stale under repeated promotion. +pub(super) fn note_young_survival(young_bytes: usize, live_bytes: usize) { + if young_bytes == 0 { + return; + } + let permille = (live_bytes as u64) + .saturating_mul(1000) + .checked_div(young_bytes as u64) + .unwrap_or(0) + .min(1000); + LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille))); +} + +/// Charge the dead bytes an in-place promotion just moved into old-gen against +/// the running budget. +pub(super) fn note_in_place_promotion( + young_bytes: usize, + live_bytes: usize, + promoted_objects: usize, +) { + let dead = young_bytes.saturating_sub(live_bytes); + PROMOTED_DEAD_BYTES.with(|c| c.set(c.get().saturating_add(dead))); + IN_PLACE_PROMOTION_CYCLES.with(|c| c.set(c.get().saturating_add(1))); + IN_PLACE_PROMOTED_OBJECTS.with(|c| c.set(c.get().saturating_add(promoted_objects as u64))); +} + +/// A full collection reclaimed old-gen, so the retained-garbage budget starts +/// over. +pub(super) fn note_full_collection_reclaimed_old_gen() { + PROMOTED_DEAD_BYTES.with(|c| c.set(0)); +} + +/// How many cycles promoted in place, and how many objects they promoted. +/// The "did the subject actually run?" counters — a green benchmark that never +/// entered the path proves nothing. +pub fn in_place_promotion_cycles() -> u64 { + IN_PLACE_PROMOTION_CYCLES.with(Cell::get) +} + +pub fn in_place_promoted_objects() -> u64 { + IN_PLACE_PROMOTED_OBJECTS.with(Cell::get) +} + +/// Last measured young-survival ratio in permille, or `None` before the first +/// copying minor. Trace/test observability. +pub(crate) fn last_young_survival_permille() -> Option { + LAST_YOUNG_SURVIVAL_PERMILLE.with(Cell::get) +} + +pub(crate) fn promoted_dead_bytes_since_full() -> usize { + PROMOTED_DEAD_BYTES.with(Cell::get) +} + +#[cfg(test)] +thread_local! { + static TEST_OPT_IN: Cell = const { Cell::new(false) }; +} + +/// Opt this thread's copying minors into the in-place promotion path, and +/// restore the previous state (plus the whole policy state) on drop. +#[cfg(test)] +pub(super) struct InPlacePromotionTestGuard { + previous_opt_in: bool, + previous_survival: Option, + previous_dead: usize, +} + +#[cfg(test)] +impl InPlacePromotionTestGuard { + pub(super) fn enabled(survival_permille: u64) -> Self { + let guard = Self { + previous_opt_in: TEST_OPT_IN.replace(true), + previous_survival: LAST_YOUNG_SURVIVAL_PERMILLE.get(), + previous_dead: PROMOTED_DEAD_BYTES.get(), + }; + LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(survival_permille))); + PROMOTED_DEAD_BYTES.with(|c| c.set(0)); + guard + } +} + +#[cfg(test)] +impl Drop for InPlacePromotionTestGuard { + fn drop(&mut self) { + TEST_OPT_IN.with(|c| c.set(self.previous_opt_in)); + LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(self.previous_survival)); + PROMOTED_DEAD_BYTES.with(|c| c.set(self.previous_dead)); + } +} + +#[cfg(test)] +pub(super) fn seed_young_survival_for_tests(permille: u64) { + LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille))); +} + +#[cfg(test)] +pub(super) fn seed_promoted_dead_bytes_for_tests(bytes: usize) { + PROMOTED_DEAD_BYTES.with(|c| c.set(bytes)); +} diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index 525afcb6d4..e3cc182a02 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -229,6 +229,26 @@ pub(super) struct CopyingNurseryTraceStats { /// `eligible=true` and `preflight_skipped=false` did the old work. pub(super) preflight_skipped: bool, pub(super) fallback_reason: CopiedMinorFallbackReason, + /// #7742: this cycle promoted the young generation whole, in place — + /// nothing was copied and nothing moved. + pub(super) in_place_promotion: bool, + /// Objects promoted by that path. The "did the subject run?" counter: a + /// row with `in_place_promotion=true` and zero here promoted nothing and + /// proves nothing. + pub(super) in_place_promoted_objects: usize, + pub(super) in_place_promoted_blocks: usize, + /// Bytes on the promoted blocks that were NOT live — the footprint this + /// technique trades for the speed, retained until the next full. + pub(super) in_place_dead_bytes: usize, + /// Promoted blocks whose live fraction was under 50%: the shape in-place + /// promotion is the wrong answer for. Non-zero here means the policy + /// threshold is admitting cycles it should not. + pub(super) in_place_sparse_blocks: usize, + /// Young-survival ratio (permille) this cycle measured — the input the + /// NEXT cycle's promotion decision is taken from. + pub(super) young_survival_permille: u64, + /// #7742: the three remembered-set passes were provably empty and skipped. + pub(super) remembering_skipped: bool, } #[derive(Clone, Copy, Default)] @@ -1016,6 +1036,13 @@ impl GcCycleTrace { "malloc_registry_rebuilds": self.copying_nursery.malloc_registry_rebuilds, "malloc_sweep_due": self.copying_nursery.malloc_sweep_due, "fallback_reason": self.copying_nursery.fallback_reason.as_str(), + "in_place_promotion": self.copying_nursery.in_place_promotion, + "in_place_promoted_objects": self.copying_nursery.in_place_promoted_objects, + "in_place_promoted_blocks": self.copying_nursery.in_place_promoted_blocks, + "in_place_dead_bytes": self.copying_nursery.in_place_dead_bytes, + "in_place_sparse_blocks": self.copying_nursery.in_place_sparse_blocks, + "young_survival_permille": self.copying_nursery.young_survival_permille, + "remembering_skipped": self.copying_nursery.remembering_skipped, }); let evacuation_policy_json = serde_json::json!({ "allowed": self.evacuation_policy.allowed, diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index f54eb4fe7d..ea929659bd 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -29,6 +29,7 @@ mod lazy_intrinsic_towers; mod lazy_tape_side_alloc; mod oldgen; mod os_tag; +mod promote_in_place; mod root_words; mod roots; mod runtime_roots; diff --git a/crates/perry-runtime/src/gc/tests/promote_in_place.rs b/crates/perry-runtime/src/gc/tests/promote_in_place.rs new file mode 100644 index 0000000000..917097ce92 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/promote_in_place.rs @@ -0,0 +1,207 @@ +//! Teeth for whole-block in-place promotion (#7742). +//! +//! Two obligations, and the second is the one that matters: +//! +//! 1. The policy classifies the measured workload population correctly, and +//! every knob that can turn the path off is asserted in BOTH states (the GC +//! knob kill-policy in CLAUDE.md). +//! 2. A cycle that takes the path actually promotes something, and the +//! promoted object comes out the far side at the SAME address, in old-gen, +//! `GC_FLAG_TENURED`, and findable through the old-gen page index. A test +//! that merely observes "nothing threw" would pass against a promotion that +//! silently indexed nothing — which is precisely the shape that turns into a +//! swept-live-object crash one cycle later. + +use super::super::promote_in_place::{ + parse_promote_in_place, seed_promoted_dead_bytes_for_tests, InPlacePromotionTestGuard, + PROMOTED_DEAD_BUDGET_BYTES, PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, +}; +use super::super::*; +use super::support::*; + +// --------------------------------------------------------------------------- +// Knob + policy (pure) +// --------------------------------------------------------------------------- + +#[test] +fn promote_in_place_knob_parses_both_states() { + // ON is the default and every unrecognised spelling — a typo must not + // silently change which collector a bisect is measuring. + for raw in [None, Some("1"), Some("on"), Some("true"), Some("banana")] { + assert!( + parse_promote_in_place(raw), + "{raw:?} must leave in-place promotion ON" + ); + } + for raw in ["0", "off", "false", "OFF", " false "] { + assert!( + !parse_promote_in_place(Some(raw)), + "{raw} must turn in-place promotion OFF" + ); + } +} + +#[test] +fn threshold_separates_the_measured_workload_population() { + // The ratios actually measured on gc-handoff/bench (see the module docs on + // gc/promote_in_place.rs). This is the claim the constant rests on: the + // population is bimodal, so the threshold is not a tuning dial. + let fully_live = [999u64, 1000, 1000, 1000]; + let churny = [0u64, 1, 2, 3, 4]; + for r in fully_live { + assert!( + r >= PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "retain/deeplist-shaped ratio {r} must promote in place" + ); + } + for r in churny { + assert!( + r < PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + "churn-shaped ratio {r} must NOT promote in place" + ); + } +} + +#[test] +fn a_promoting_cycle_still_measures_so_the_predictor_cannot_go_stale() { + let _guard = InPlacePromotionTestGuard::enabled(1000); + assert!(should_promote_young_in_place()); + + // The promoting cycle traces, so it measures. A workload that flips to + // garbage turns the policy off on the very next decision — one nursery of + // retained garbage, not an unbounded run of them. + note_young_survival(16 * 1024 * 1024, 4 * 1024); + assert!( + !should_promote_young_in_place(), + "a measured collapse in survival must disable in-place promotion immediately" + ); + + note_young_survival(16 * 1024 * 1024, 16 * 1024 * 1024); + assert!(should_promote_young_in_place()); +} + +#[test] +fn an_unmeasured_thread_never_promotes() { + // 0 permille is what an unmeasured thread decides as (the `None` case takes + // the same branch): with no evidence there is no basis for promoting, so + // the first copying minor must evacuate and measure. + let _guard = InPlacePromotionTestGuard::enabled(0); + assert!(!should_promote_young_in_place()); +} + +#[test] +fn dead_byte_budget_stops_promotion_until_a_full_reclaims() { + let _guard = InPlacePromotionTestGuard::enabled(1000); + assert!(should_promote_young_in_place()); + + seed_promoted_dead_bytes_for_tests(PROMOTED_DEAD_BUDGET_BYTES); + assert!( + !should_promote_young_in_place(), + "the running dead-byte budget is the bound on the steady state the \ + per-cycle re-measurement does NOT cover" + ); + + note_full_collection_reclaimed_old_gen(); + assert!( + should_promote_young_in_place(), + "a full collection reclaimed the parked garbage, so the budget resets" + ); +} + +// --------------------------------------------------------------------------- +// End to end: the object does not move, and it is a first-class old-gen object +// afterwards. +// --------------------------------------------------------------------------- + +#[test] +fn in_place_promotion_leaves_the_object_at_its_address_in_old_gen() { + // NOTE: `CopyingNurseryTestGuard::new` takes the copying-nursery isolation + // lock itself — taking it again here is a self-deadlock. + let _guard = CopyingNurseryTestGuard::new(4); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _promote = InPlacePromotionTestGuard::enabled(1000); + // NO `reset_shadow_stack()` here: the guard has already pushed the frame + // whose slot 0 is written below, and resetting would drop it — the object + // would then have no root, die, and the test would read as "the in-place + // path promoted nothing". + let child = young_leaf(); + js_shadow_slot_set(0, ptr_bits(child)); + assert!(crate::arena::pointer_in_nursery(child)); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + + // Live subject: a green row that promoted nothing proves nothing. + assert!( + trace.copying_nursery.in_place_promotion, + "the cycle must have taken the in-place path" + ); + assert!( + trace.copying_nursery.in_place_promoted_objects > 0, + "the in-place path must have promoted at least one object" + ); + assert!( + trace.copying_nursery.in_place_promoted_blocks > 0, + "the in-place path must have taken at least one block" + ); + assert_eq!( + trace.copying_nursery.copied_objects, 0, + "an in-place promotion copies nothing" + ); + + // The whole point: the address is unchanged, and every slot that pointed + // at it is therefore still correct without any rewrite. + let after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_eq!(after, child, "in-place promotion must not move the object"); + assert!( + crate::arena::pointer_in_old_gen(child), + "the promoted object must classify as old-gen afterwards" + ); + + // `Old ⟹ TENURED` (#7511): the generated write barrier's fast path skips + // the remembering call outright when this bit is missing. + let header = unsafe { header_from_user_ptr(child as *const u8) }; + assert_ne!( + unsafe { (*header).gc_flags } & GC_FLAG_TENURED, + 0, + "a promoted object must carry GC_FLAG_TENURED" + ); + + // Findable through the old-gen page index — this is what the remembered-set + // dirty scan uses to reach it, so an unindexed promoted object is a + // missed old→young edge waiting to happen. + let page = crate::arena::generation_page_for_addr(header as usize); + let meta = crate::arena::old_page_meta_for_tests(page) + .expect("the promoted object's page must have old-gen metadata"); + assert!( + meta.live_object_count > 0 && meta.live_bytes > 0, + "the promoted object must be indexed as live on its old page, got {meta:?}" + ); +} + +#[test] +fn a_low_survival_cycle_still_evacuates_and_moves_the_object() { + // NOTE: `CopyingNurseryTestGuard::new` takes the copying-nursery isolation + // lock itself — taking it again here is a self-deadlock. + let _guard = CopyingNurseryTestGuard::new(4); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + // Opted in, but the measurement says "mostly garbage" — the OFF arm of the + // policy, driven through the same entry point as the ON arm above. + let _promote = InPlacePromotionTestGuard::enabled(10); + let child = young_leaf(); + js_shadow_slot_set(0, ptr_bits(child)); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + assert!( + !trace.copying_nursery.in_place_promotion, + "a 1% survival reading must evacuate, not promote" + ); + + let after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!( + after, child, + "the ordinary copying path must still relocate" + ); + assert!(crate::arena::pointer_in_nursery(after)); +} diff --git a/scripts/addr_class_allowlist.txt b/scripts/addr_class_allowlist.txt index 4365a30627..205647490a 100644 --- a/scripts/addr_class_allowlist.txt +++ b/scripts/addr_class_allowlist.txt @@ -25,6 +25,7 @@ crates/perry-runtime/src/arena/quarantine.rs | let header = data.add(pos) as *co crates/perry-runtime/src/string/mod.rs | let header = raw.sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; | #7647 zero_alignment_padding_tail: `raw` is the pointer arena_alloc_gc just returned to string_storage_alloc/_longlived a few lines above, never a NaN-box payload -- same discipline as arena/allocators.rs's own grandfathered entry, reading `.size` back to zero the alignment pad the allocator introduced crates/perry-runtime/src/arena/tests.rs | * | arena allocator/walker internals: header addresses come from block iteration or fresh allocation, never from NaN-box payloads crates/perry-runtime/src/arena/walk.rs | * | arena allocator/walker internals: header addresses come from block iteration or fresh allocation, never from NaN-box payloads +crates/perry-runtime/src/arena/promote.rs | let header = header_ptr as *mut GcHeader; | #7742 whole-block promotion: `header_ptr` is `block.data + aligned` from the same linear block iteration arena/walk.rs performs (its grandfathered sibling entry above), never a NaN-box payload, so no handle band can reach it; the walk stops at the first header whose size does not cover the remaining bytes crates/perry-runtime/src/array/alloc.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/concat_reverse.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/from_concat.rs | * | #4994 split of the pre-existing concat GcHeader probe; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up From f581ef327fe39fe0f63829680a407634c94790d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 10 Aug 2026 06:05:12 +0200 Subject: [PATCH 2/4] review: name the right assert helper, both semispaces in the doc, a real unmeasured-state test --- .../7744-gc-whole-block-in-place-promotion.md | 2 +- crates/perry-runtime/src/arena/promote.rs | 3 ++- crates/perry-runtime/src/gc/copying.rs | 2 +- .../perry-runtime/src/gc/promote_in_place.rs | 9 +++++++ .../src/gc/tests/promote_in_place.rs | 26 ++++++++++++++----- 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/changelog.d/7744-gc-whole-block-in-place-promotion.md b/changelog.d/7744-gc-whole-block-in-place-promotion.md index ee1501e915..27e3ab0c0d 100644 --- a/changelog.d/7744-gc-whole-block-in-place-promotion.md +++ b/changelog.d/7744-gc-whole-block-in-place-promotion.md @@ -2,7 +2,7 @@ When a copying minor's nursery is (near-)entirely live, the collector now relabels its blocks as old-gen instead of evacuating them object by object. -`retain.ts` 0.80 s → **0.53 s**, `retain_wide.ts` 1.33 s → **1.07 s**, +`retain.ts` 0.81 s → **0.53 s**, `retain_wide.ts` 1.33 s → **1.07 s**, `retain1` 0.38 → 0.29, `retain_wide1` 0.38 → 0.27, `deeplist` 0.30 → 0.24 (peak RSS 117 MB → 97 MB). Promotion cost per object **243 ns → 101 ns**, derived from the trace's own promoted-object count and pause, not estimated. diff --git a/crates/perry-runtime/src/arena/promote.rs b/crates/perry-runtime/src/arena/promote.rs index 0d13752d12..b37ec9a834 100644 --- a/crates/perry-runtime/src/arena/promote.rs +++ b/crates/perry-runtime/src/arena/promote.rs @@ -34,7 +34,8 @@ //! The two halves: //! //! * [`retag_young_for_in_place_promotion`] runs *before* the trace. It flips -//! every in-use Eden and active-survivor block's page range to generation +//! every in-use Eden and survivor block's page range — BOTH semispaces, see +//! the comment on the loop — to generation //! `Old`, space [`HeapSpace::PromotedYoung`]. From that instant the barrier //! predicates (`barrier_parent_needs_remembering`, //! `remembered_child_needs_tracking`) treat those objects as old — which is diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index b6af6fab5e..52761c04aa 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -443,7 +443,7 @@ pub(super) struct CopyingNurseryCollector { /// surviving cohort's slots (`visit_slot_with_parent`'s re-decode + /// remember, `rebuild_evacuated_old_to_young_remembered_set`, and /// `restore_surviving_dirty_coverage`) can only insert nothing, and are - /// skipped. `debug_assert_no_remembering_needed` re-derives the premise at + /// skipped. `debug_assert_no_remembering_possible` re-derives the premise at /// runtime in debug builds. pub(super) skip_remembering: bool, /// Weak target slots (WeakRef referent / WeakMap-WeakSet entry key / diff --git a/crates/perry-runtime/src/gc/promote_in_place.rs b/crates/perry-runtime/src/gc/promote_in_place.rs index f6f06275b4..03f09ba538 100644 --- a/crates/perry-runtime/src/gc/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/promote_in_place.rs @@ -247,6 +247,15 @@ impl Drop for InPlacePromotionTestGuard { } } +/// Put the thread back in the state it boots in: no copying minor has run, so +/// nothing has been measured. Distinct from seeding 0 permille — that is a +/// MEASUREMENT of "almost nothing survived", and only this exercises the +/// `None` arm of the decision. +#[cfg(test)] +pub(super) fn clear_young_survival_for_tests() { + LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(None)); +} + #[cfg(test)] pub(super) fn seed_young_survival_for_tests(permille: u64) { LAST_YOUNG_SURVIVAL_PERMILLE.with(|c| c.set(Some(permille))); diff --git a/crates/perry-runtime/src/gc/tests/promote_in_place.rs b/crates/perry-runtime/src/gc/tests/promote_in_place.rs index 917097ce92..d35c86cf68 100644 --- a/crates/perry-runtime/src/gc/tests/promote_in_place.rs +++ b/crates/perry-runtime/src/gc/tests/promote_in_place.rs @@ -13,8 +13,9 @@ //! swept-live-object crash one cycle later. use super::super::promote_in_place::{ - parse_promote_in_place, seed_promoted_dead_bytes_for_tests, InPlacePromotionTestGuard, - PROMOTED_DEAD_BUDGET_BYTES, PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, + clear_young_survival_for_tests, parse_promote_in_place, seed_promoted_dead_bytes_for_tests, + seed_young_survival_for_tests, InPlacePromotionTestGuard, PROMOTED_DEAD_BUDGET_BYTES, + PROMOTE_SURVIVAL_THRESHOLD_PERMILLE, }; use super::super::*; use super::support::*; @@ -82,11 +83,22 @@ fn a_promoting_cycle_still_measures_so_the_predictor_cannot_go_stale() { #[test] fn an_unmeasured_thread_never_promotes() { - // 0 permille is what an unmeasured thread decides as (the `None` case takes - // the same branch): with no evidence there is no basis for promoting, so - // the first copying minor must evacuate and measure. - let _guard = InPlacePromotionTestGuard::enabled(0); - assert!(!should_promote_young_in_place()); + // Both no-promote arms, and they are genuinely different states: `None` is + // "no copying minor has run on this thread", 0 permille is a MEASUREMENT of + // "almost nothing survived". The first copying minor of a process is in the + // former, and it must evacuate and measure rather than promote on no + // evidence — so the `None` arm needs asserting in its own right. + let _guard = InPlacePromotionTestGuard::enabled(1000); + clear_young_survival_for_tests(); + assert!( + !should_promote_young_in_place(), + "an unmeasured thread has no basis for promoting" + ); + seed_young_survival_for_tests(0); + assert!( + !should_promote_young_in_place(), + "a measured 0 permille must not promote either" + ); } #[test] From 15b553fda005a7bcdd19532761c55cb8c53cc75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 10 Aug 2026 06:26:38 +0200 Subject: [PATCH 3/4] changelog: measured ns/object for both retain variants --- changelog.d/7744-gc-whole-block-in-place-promotion.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.d/7744-gc-whole-block-in-place-promotion.md b/changelog.d/7744-gc-whole-block-in-place-promotion.md index 27e3ab0c0d..1aeb850e7a 100644 --- a/changelog.d/7744-gc-whole-block-in-place-promotion.md +++ b/changelog.d/7744-gc-whole-block-in-place-promotion.md @@ -4,8 +4,9 @@ When a copying minor's nursery is (near-)entirely live, the collector now relabels its blocks as old-gen instead of evacuating them object by object. `retain.ts` 0.81 s → **0.53 s**, `retain_wide.ts` 1.33 s → **1.07 s**, `retain1` 0.38 → 0.29, `retain_wide1` 0.38 → 0.27, `deeplist` 0.30 → 0.24 -(peak RSS 117 MB → 97 MB). Promotion cost per object **243 ns → 101 ns**, -derived from the trace's own promoted-object count and pause, not estimated. +(peak RSS 117 MB → 97 MB). Promotion cost per object **243 ns → 105 ns** on `retain` +and **264 ns → 116 ns** on `retain_wide`, derived from the trace's own +promoted-object count and pause summed over the promoting cycles, not estimated. All measurements are best-of-5 wall clock on the pinned quiet M1 mini, outputs byte-verified against `node --experimental-strip-types` before timing. From 7d8980016c7b50526cb4e5a9b792b6fe41809269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 10 Aug 2026 07:52:55 +0200 Subject: [PATCH 4/4] chore: bump version to 0.5.1434 Claude-Session: https://claude.ai/code/session_01Y1QZ5wUP9gRSwpiweT4Wix --- CLAUDE.md | 2 +- Cargo.lock | 152 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 3 files changed, 78 insertions(+), 78 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a26a9bd656..0230db5fbb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1433 +**Current Version:** 0.5.1434 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index d6931f6a61..34362ea65b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5547,7 +5547,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "base64", @@ -5607,14 +5607,14 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "serde", ] [[package]] name = "perry-audio-miniaudio" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "cc", "libc", @@ -5622,7 +5622,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "inkwell", @@ -5639,7 +5639,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-hir", @@ -5647,7 +5647,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-hir", @@ -5655,7 +5655,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-dispatch", @@ -5664,7 +5664,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-hir", @@ -5672,7 +5672,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "base64", @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-hir", @@ -5692,7 +5692,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "async-trait", @@ -5721,14 +5721,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "serde", "serde_json", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1433" +version = "0.5.1434" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5747,7 +5747,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "clap", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "block2", "objc2", @@ -5772,7 +5772,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "argon2", "perry-ffi", @@ -5780,7 +5780,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "reqwest", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "bcrypt", "perry-ffi", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "rusqlite", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "scraper", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "perry-runtime", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "chrono", "cron", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "chrono", "perry-ffi", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "rust_decimal", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "serde_json", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "rand 0.10.1", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "perry-runtime", @@ -5871,14 +5871,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "bytes", "http-body-util", @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "bytes", "lazy_static", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "bytes", "h2", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "lazy_static", "perry-ffi", @@ -5943,7 +5943,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "jsonwebtoken", @@ -5954,7 +5954,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "lru", "perry-ffi", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "chrono", "perry-ffi", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "bson", "futures-util", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "chrono", "perry-ffi", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "nanoid", "perry-ffi", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "bytes", "perry-ffi", @@ -6015,7 +6015,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "const-oid 0.9.6", "der 0.7.10", @@ -6034,7 +6034,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "lettre", "perry-ffi", @@ -6044,7 +6044,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "printpdf", @@ -6052,7 +6052,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "sqlx", @@ -6061,7 +6061,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "governor", "perry-ffi", @@ -6069,7 +6069,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "fast_image_resize", "image", @@ -6079,14 +6079,14 @@ dependencies = [ [[package]] name = "perry-ext-slugify" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-streams" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "lazy_static", "perry-ffi", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "perry-runtime", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "uuid", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ffi", "regex", @@ -6122,7 +6122,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "futures-util", "lazy_static", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "brotli", "flate2", @@ -6145,7 +6145,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "dashmap", "once_cell", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-api-manifest", @@ -6172,7 +6172,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-diagnostics", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "base64", @@ -6226,14 +6226,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6328,14 +6328,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "perry-hir", @@ -6344,14 +6344,14 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ui-model", ] [[package]] name = "perry-ui-android" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "itoa", @@ -6368,7 +6368,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "rand 0.10.1", "serde", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "cairo-rs 0.22.0", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "block2", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "block2", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1433" +version = "0.5.1434" [[package]] name = "perry-ui-test" @@ -6443,11 +6443,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1433" +version = "0.5.1434" [[package]] name = "perry-ui-tvos" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "block2", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "block2", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "block2", "libc", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "base64", "libc", @@ -6509,14 +6509,14 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "perry-ui-windows", ] [[package]] name = "perry-updater" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "anyhow", "base64", @@ -6532,7 +6532,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1433" +version = "0.5.1434" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index af641b50d0..43a0760ca1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ codegen-units = 16 codegen-units = 16 [workspace.package] -version = "0.5.1433" +version = "0.5.1434" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry"