diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index e0d9453..5e33898 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -15,5 +15,5 @@ jobs: uses: jcfangc/ci/.github/workflows/rust-codspeed.yml@v0 with: rust_toolchain: stable - build_command: cargo codspeed build -m simulation + build_command: RUSTFLAGS="-C target-feature=+avx2" cargo codspeed build -m simulation run_command: cargo codspeed run -m simulation ours_ diff --git a/Cargo.lock b/Cargo.lock index 0618e7e..038a9fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,12 +49,11 @@ dependencies = [ [[package]] name = "bit-string" -version = "0.4.5" +version = "0.5.0" dependencies = [ "bitvec_simd", "codspeed-divan-compat", "int-interval", - "once_cell", "proptest", "witnessed", ] diff --git a/Cargo.toml b/Cargo.toml index 493da5c..87207e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string" -version = "0.4.5" +version = "0.5.0" edition = "2024" description = "A compact owned bit string type with editing, matching, and bitwise operations." readme = "README.md" @@ -11,13 +11,8 @@ categories = ["data-structures", "no-std"] exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] -[features] -default = [] -compile-time-dispatch = [] - [dependencies] int-interval = "0.9.6" -once_cell = "1" witnessed = "0.8.0" [lib] diff --git a/README.md b/README.md index ffe226d..ebf2e78 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/gh/jcfangc/bit-string) [![Coverage](https://codecov.io/gh/jcfangc/bit-string/branch/main/graph/badge.svg)](https://codecov.io/gh/jcfangc/bit-string) -A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — all accelerated by runtime SIMD dispatch (AVX2, SSSE3, NEON). +A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — accelerated by compile-time-selected SIMD backends (AVX2, SSSE3, NEON). ## Quick start @@ -200,7 +200,7 @@ assert_eq!(bits, vec![true, false, false]); ## SIMD backends -At runtime (or compile time with the `compile-time-dispatch` feature) the crate selects the best available SIMD backend: +The crate selects the fastest SIMD backend enabled for the compilation target: | Backend | Target | Width | |---------|--------|-------| @@ -209,7 +209,7 @@ At runtime (or compile time with the `compile-time-dispatch` feature) the crate | NEON | aarch64 | 128-bit (2×u64) | | Scalar | all targets | fallback | -For maximum local performance, copy the example config: +There is no runtime CPU detection. For maximum local performance, compile for the host CPU by copying the example config: ```bash cp .cargo/config.toml.example .cargo/config.toml diff --git a/benches/bit_ops_trailing.rs b/benches/bit_ops_trailing.rs index db5300c..ec64823 100644 --- a/benches/bit_ops_trailing.rs +++ b/benches/bit_ops_trailing.rs @@ -14,108 +14,131 @@ enum Pattern { } // ═══════════════════════════════════════════════════════════════════════ -// BitStr::leading_zeros (optimised trait path) — reference baseline -// for trailing_zeros. Both go through WordsScan; trailing adds the -// reverse-scan overhead. Goal: make trailing as close as possible. +// Leading references are named under the same case hierarchy as their +// trailing counterparts, so Divan prints each comparison together. // ═══════════════════════════════════════════════════════════════════════ -#[divan::bench(name = "leading_zeros/len_65/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_65/all_zeros/leading/ours_str")] fn lead_65_zeros_str_ref(b: Bencher) { bench_lead_str(b, 65, Pattern::Zeros); } -#[divan::bench(name = "leading_zeros/len_65/dense/ours_str")] +#[divan::bench(name = "count_zeros/len_65/all_zeros/leading/ours_string")] +fn lead_65_zeros_string_ref(b: Bencher) { + bench_lead_string(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "count_zeros/len_65/dense/leading/ours_str")] fn lead_65_dense_str_ref(b: Bencher) { bench_lead_str(b, 65, Pattern::Dense); } -#[divan::bench(name = "leading_zeros/len_4096/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_65/dense/leading/ours_string")] +fn lead_65_dense_string_ref(b: Bencher) { + bench_lead_string(b, 65, Pattern::Dense); +} +#[divan::bench(name = "count_zeros/len_4096/all_zeros/leading/ours_str")] fn lead_4096_zeros_str_ref(b: Bencher) { bench_lead_str(b, 4096, Pattern::Zeros); } -#[divan::bench(name = "leading_zeros/len_65536/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_4096/all_zeros/leading/ours_string")] +fn lead_4096_zeros_string_ref(b: Bencher) { + bench_lead_string(b, 4096, Pattern::Zeros); +} +#[divan::bench(name = "count_zeros/len_65536/all_zeros/leading/ours_str")] fn lead_65536_zeros_str_ref(b: Bencher) { bench_lead_str(b, 65536, Pattern::Zeros); } -#[divan::bench(name = "leading_ones/len_65/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_65536/all_zeros/leading/ours_string")] +fn lead_65536_zeros_string_ref(b: Bencher) { + bench_lead_string(b, 65536, Pattern::Zeros); +} +#[divan::bench(name = "count_ones/len_65/all_zeros/leading/ours_str")] fn lead_ones_65_zeros_str_ref(b: Bencher) { bench_lead_ones_str(b, 65, Pattern::Zeros); } -#[divan::bench(name = "leading_ones/len_4096/all_zeros/ours_str")] +#[divan::bench(name = "count_ones/len_65/all_zeros/leading/ours_string")] +fn lead_ones_65_zeros_string_ref(b: Bencher) { + bench_lead_ones_string(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "count_ones/len_4096/all_zeros/leading/ours_str")] fn lead_ones_4096_zeros_str_ref(b: Bencher) { bench_lead_ones_str(b, 4096, Pattern::Zeros); } +#[divan::bench(name = "count_ones/len_4096/all_zeros/leading/ours_string")] +fn lead_ones_4096_zeros_string_ref(b: Bencher) { + bench_lead_ones_string(b, 4096, Pattern::Zeros); +} // ═══════════════════════════════════════════════════════════════════════ // trailing_zeros // ═══════════════════════════════════════════════════════════════════════ -#[divan::bench(name = "trailing_zeros/len_65/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_65/all_zeros/trailing/ours_str")] fn trailing_65_zeros_str(b: Bencher) { bench_str(b, 65, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/len_65/all_zeros/ours_string")] +#[divan::bench(name = "count_zeros/len_65/all_zeros/trailing/ours_string")] fn trailing_65_zeros_string(b: Bencher) { bench_string(b, 65, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/len_65/alternating/ours_str")] +#[divan::bench(name = "count_zeros/len_65/alternating/trailing/ours_str")] fn trailing_65_alternating_str(b: Bencher) { bench_str(b, 65, Pattern::Alternating); } -#[divan::bench(name = "trailing_zeros/len_65/alternating/ours_string")] +#[divan::bench(name = "count_zeros/len_65/alternating/trailing/ours_string")] fn trailing_65_alternating_string(b: Bencher) { bench_string(b, 65, Pattern::Alternating); } -#[divan::bench(name = "trailing_zeros/len_65/dense/ours_str")] +#[divan::bench(name = "count_zeros/len_65/dense/trailing/ours_str")] fn trailing_65_dense_str(b: Bencher) { bench_str(b, 65, Pattern::Dense); } -#[divan::bench(name = "trailing_zeros/len_65/dense/ours_string")] +#[divan::bench(name = "count_zeros/len_65/dense/trailing/ours_string")] fn trailing_65_dense_string(b: Bencher) { bench_string(b, 65, Pattern::Dense); } -#[divan::bench(name = "trailing_zeros/len_4096/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_4096/all_zeros/trailing/ours_str")] fn trailing_4096_zeros_str(b: Bencher) { bench_str(b, 4096, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/len_4096/all_zeros/ours_string")] +#[divan::bench(name = "count_zeros/len_4096/all_zeros/trailing/ours_string")] fn trailing_4096_zeros_string(b: Bencher) { bench_string(b, 4096, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/len_4096/dense/ours_str")] +#[divan::bench(name = "count_zeros/len_4096/dense/trailing/ours_str")] fn trailing_4096_dense_str(b: Bencher) { bench_str(b, 4096, Pattern::Dense); } -#[divan::bench(name = "trailing_zeros/len_4096/dense/ours_string")] +#[divan::bench(name = "count_zeros/len_4096/dense/trailing/ours_string")] fn trailing_4096_dense_string(b: Bencher) { bench_string(b, 4096, Pattern::Dense); } -#[divan::bench(name = "trailing_zeros/len_65536/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_65536/all_zeros/trailing/ours_str")] fn trailing_65536_zeros_str(b: Bencher) { bench_str(b, 65536, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/len_65536/all_zeros/ours_string")] +#[divan::bench(name = "count_zeros/len_65536/all_zeros/trailing/ours_string")] fn trailing_65536_zeros_string(b: Bencher) { bench_string(b, 65536, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/len_65536/dense/ours_str")] +#[divan::bench(name = "count_zeros/len_65536/dense/trailing/ours_str")] fn trailing_65536_dense_str(b: Bencher) { bench_str(b, 65536, Pattern::Dense); } -#[divan::bench(name = "trailing_zeros/len_65536/dense/ours_string")] +#[divan::bench(name = "count_zeros/len_65536/dense/trailing/ours_string")] fn trailing_65536_dense_string(b: Bencher) { bench_string(b, 65536, Pattern::Dense); } -#[divan::bench(name = "trailing_zeros/unaligned_3/len_4096/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_4096/all_zeros/trailing_unaligned_3/ours_str")] fn trailing_unaligned_3_4096_zeros_str(b: Bencher) { bench_unaligned_str(b, 4096, 3, Pattern::Zeros); } -#[divan::bench(name = "trailing_zeros/unaligned_63/len_4096/all_zeros/ours_str")] +#[divan::bench(name = "count_zeros/len_4096/all_zeros/trailing_unaligned_63/ours_str")] fn trailing_unaligned_63_4096_zeros_str(b: Bencher) { bench_unaligned_str(b, 4096, 63, Pattern::Zeros); } @@ -124,29 +147,29 @@ fn trailing_unaligned_63_4096_zeros_str(b: Bencher) { // trailing_ones // ═══════════════════════════════════════════════════════════════════════ -#[divan::bench(name = "trailing_ones/len_65/all_zeros/ours_str")] +#[divan::bench(name = "count_ones/len_65/all_zeros/trailing/ours_str")] fn trailing_ones_65_zeros_str(b: Bencher) { bench_str_trailing_ones(b, 65, Pattern::Zeros); } -#[divan::bench(name = "trailing_ones/len_65/all_zeros/ours_string")] +#[divan::bench(name = "count_ones/len_65/all_zeros/trailing/ours_string")] fn trailing_ones_65_zeros_string(b: Bencher) { bench_string_trailing_ones(b, 65, Pattern::Zeros); } -#[divan::bench(name = "trailing_ones/len_65/dense/ours_str")] +#[divan::bench(name = "count_ones/len_65/dense/trailing/ours_str")] fn trailing_ones_65_dense_str(b: Bencher) { bench_str_trailing_ones(b, 65, Pattern::Dense); } -#[divan::bench(name = "trailing_ones/len_65/dense/ours_string")] +#[divan::bench(name = "count_ones/len_65/dense/trailing/ours_string")] fn trailing_ones_65_dense_string(b: Bencher) { bench_string_trailing_ones(b, 65, Pattern::Dense); } -#[divan::bench(name = "trailing_ones/len_4096/all_zeros/ours_str")] +#[divan::bench(name = "count_ones/len_4096/all_zeros/trailing/ours_str")] fn trailing_ones_4096_zeros_str(b: Bencher) { bench_str_trailing_ones(b, 4096, Pattern::Zeros); } -#[divan::bench(name = "trailing_ones/len_4096/all_zeros/ours_string")] +#[divan::bench(name = "count_ones/len_4096/all_zeros/trailing/ours_string")] fn trailing_ones_4096_zeros_string(b: Bencher) { bench_string_trailing_ones(b, 4096, Pattern::Zeros); } @@ -158,11 +181,19 @@ fn bench_lead_str(b: Bencher, len: usize, p: Pattern) { let v = bits.as_bit_str(); b.bench(|| black_box(&v).leading_zeros()); } +fn bench_lead_string(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + b.bench(|| black_box(&bits).leading_zeros()); +} fn bench_lead_ones_str(b: Bencher, len: usize, p: Pattern) { let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); let v = bits.as_bit_str(); b.bench(|| black_box(&v).leading_ones()); } +fn bench_lead_ones_string(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + b.bench(|| black_box(&bits).leading_ones()); +} // ── trailing_zeros helpers ──────────────────────────────────────────── diff --git a/benches/hash.rs b/benches/hash.rs index bc6c78e..8f33d2c 100644 --- a/benches/hash.rs +++ b/benches/hash.rs @@ -79,6 +79,25 @@ fn hash_len_64_alternating_str(b: Bencher) { bench_str(b, 64, Pattern::Alternating); } +// --------------------------------------------------------------------------- +// len = 65 (one full word plus a partial-word tail) +// --------------------------------------------------------------------------- + +#[divan::bench(name = "hash/len_65/dense/ours_string")] +fn hash_len_65_dense_bit_string(b: Bencher) { + bench_bit_string(b, 65, Pattern::Dense); +} + +#[divan::bench(name = "hash/len_65/dense/ours_str")] +fn hash_len_65_dense_bit_str(b: Bencher) { + bench_bit_str(b, 65, Pattern::Dense); +} + +#[divan::bench(name = "hash/len_65/dense/ours_str_unaligned")] +fn hash_len_65_dense_bit_str_unaligned(b: Bencher) { + bench_bit_str_unaligned(b, 65, Pattern::Dense); +} + // --------------------------------------------------------------------------- // len = 4096 // --------------------------------------------------------------------------- @@ -230,6 +249,16 @@ fn bench_bit_str(bencher: Bencher, len: usize, pattern: Pattern) { }); } +fn bench_bit_str_unaligned(bencher: Bencher, len: usize, pattern: Pattern) { + let bits = make_bit_string(len + 3, pattern); + let view = bits.as_bit_str().slice_from(3).slice_until(3 + len); + bencher.bench(|| { + let mut h = DefaultHasher::new(); + black_box(&view).hash(&mut h); + black_box(h.finish()) + }); +} + fn bench_string(bencher: Bencher, len: usize, pattern: Pattern) { let s = make_string(len, pattern); bencher.bench(|| { diff --git a/benches/ord.rs b/benches/ord.rs index 17af5be..78a7596 100644 --- a/benches/ord.rs +++ b/benches/ord.rs @@ -54,6 +54,30 @@ fn cmp_len_64_diff_last_string(b: Bencher) { bench_string(b, 64, CmpCase::DifferAtLast); } +// --------------------------------------------------------------------------- +// len = 65 (one full word plus a partial-word tail) +// --------------------------------------------------------------------------- + +#[divan::bench(name = "cmp/len_65/identical/ours_string")] +fn cmp_len_65_identical_bit_string(b: Bencher) { + bench_bit_string(b, 65, CmpCase::Identical); +} + +#[divan::bench(name = "cmp/len_65/identical/bit_str_unaligned")] +fn cmp_len_65_identical_bit_str_unaligned(b: Bencher) { + bench_bit_str_unaligned(b, 65, CmpCase::Identical); +} + +#[divan::bench(name = "cmp/len_65/diff_last/ours_string")] +fn cmp_len_65_diff_last_bit_string(b: Bencher) { + bench_bit_string(b, 65, CmpCase::DifferAtLast); +} + +#[divan::bench(name = "cmp/len_65/diff_last/bit_str_unaligned")] +fn cmp_len_65_diff_last_bit_str_unaligned(b: Bencher) { + bench_bit_str_unaligned(b, 65, CmpCase::DifferAtLast); +} + // --------------------------------------------------------------------------- // len = 4096 // --------------------------------------------------------------------------- diff --git a/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs b/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs index 44efbb0..7998532 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs @@ -1,3 +1,6 @@ +//! `WORD_ALIGNED = true` is a caller guarantee; `false` makes no alignment +//! guarantee and retains the general path. + use crate::traits::WordsScan; use crate::{WORD_BITS, low_mask}; @@ -7,6 +10,7 @@ impl<'bs> BitStr<'bs> { /// `count_ones` with compile-time alignment signal. #[inline] pub(crate) fn count_ones_inner(&self) -> usize { + debug_assert!(!WORD_ALIGNED || self.start.is_multiple_of(WORD_BITS)); if self.bit_len == 0 { return 0; } diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs index cfd45e0..8ae4b41 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs @@ -1,3 +1,6 @@ +//! `WORD_ALIGNED = true` is a caller guarantee; `false` makes no alignment +//! guarantee and retains the general path. + use crate::BitStr; use crate::WORD_BITS; use crate::traits::WordsScan; @@ -7,6 +10,7 @@ impl<'bs> BitStr<'bs> { pub(crate) fn leading_value_bits_inner( &self, ) -> usize { + debug_assert!(!WORD_ALIGNED || self.start.is_multiple_of(WORD_BITS)); if self.bit_len == 0 { return 0; } diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs index 953ceb9..0eab59b 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs @@ -1,3 +1,6 @@ +//! `WORD_ALIGNED = true` is a caller guarantee; `false` makes no alignment +//! guarantee and retains the general path. + use crate::BitStr; use crate::WORD_BITS; use crate::traits::WordsScan; @@ -7,6 +10,7 @@ impl<'bs> BitStr<'bs> { pub(crate) fn trailing_value_bits_inner( &self, ) -> usize { + debug_assert!(!WORD_ALIGNED || self.start.is_multiple_of(WORD_BITS)); if self.bit_len == 0 { return 0; } @@ -17,6 +21,33 @@ impl<'bs> BitStr<'bs> { let words_ptr = unsafe { all_words.as_ptr().add(word_start) }; let start_offset = (self.start % WORD_BITS) as u32; + // Aligned one- and two-word views avoid rebuilding a slice and + // entering the generic reverse scanner. + if WORD_ALIGNED && self.bit_len <= WORD_BITS * 2 { + let last_wi = (self.bit_len - 1) / WORD_BITS; + let used = self.bit_len - last_wi * WORD_BITS; + let mask = if used == WORD_BITS { + u64::MAX + } else { + (1u64 << used) - 1 + }; + // SAFETY: BitStr invariants guarantee storage for last_wi. + let last = unsafe { *words_ptr.add(last_wi) }; + let mismatch = (last ^ FILL) & mask; + if mismatch != 0 { + return (mismatch << (WORD_BITS - used)).leading_zeros() as usize; + } + if last_wi == 0 { + return self.bit_len; + } + // SAFETY: last_wi == 1, so the first word exists as well. + let first_mismatch = unsafe { *words_ptr } ^ FILL; + if first_mismatch == 0 { + return self.bit_len; + } + return used + first_mismatch.leading_zeros() as usize; + } + // Fast-path: check the rightmost word(s) before the trait call. // Mirrors the shortcuts in BitString's trailing section. diff --git a/src/bit_str/impls_for_hash/inner.rs b/src/bit_str/impls_for_hash/inner.rs index 7ef9ded..0388737 100644 --- a/src/bit_str/impls_for_hash/inner.rs +++ b/src/bit_str/impls_for_hash/inner.rs @@ -1,3 +1,6 @@ +//! `WORD_ALIGNED = true` is a caller guarantee; `false` makes no alignment +//! guarantee and retains the general path. + use crate::BitStr; use crate::traits::WordsEdit; use crate::{WORD_BITS, low_mask}; @@ -7,6 +10,7 @@ impl<'bs> BitStr<'bs> { /// Hash with compile-time alignment signal. #[inline] pub(crate) fn hash_inner(&self, state: &mut H) { + debug_assert!(!WORD_ALIGNED || self.start.is_multiple_of(WORD_BITS)); self.bit_len.hash(state); if self.bit_len == 0 { return; @@ -28,7 +32,7 @@ impl<'bs> BitStr<'bs> { } if rem > 0 { let tail_start = self.start + full_words * WORD_BITS; - (words.read_word_at::(tail_start) & low_mask(rem)).hash(state); + (words.read_word_at::(tail_start) & low_mask(rem)).hash(state); } } } diff --git a/src/bit_str/impls_for_matching/impls_for_find/inner.rs b/src/bit_str/impls_for_matching/impls_for_find/inner.rs index 9f12285..2cf4455 100644 --- a/src/bit_str/impls_for_matching/impls_for_find/inner.rs +++ b/src/bit_str/impls_for_matching/impls_for_find/inner.rs @@ -1,3 +1,6 @@ +//! Alignment consts are caller guarantees when `true`; `false` makes no +//! alignment guarantee and retains the general path. + use crate::BitStr; use crate::traits::*; use crate::{SMALL_WORDS, WORD_BITS}; @@ -17,6 +20,8 @@ impl<'bs> BitStr<'bs> { let words = self.source.words(); let sw = self.start / WORD_BITS; let so = self.start % WORD_BITS; + debug_assert!(!WORD_ALIGNED || so == 0); + debug_assert!(!ND_WORD_ALIGNED || needle.start.is_multiple_of(WORD_BITS)); let needle_words = needle.source.words(); let needle_len = needle.bit_len; if WORD_ALIGNED || so == 0 { @@ -69,6 +74,8 @@ impl<'bs> BitStr<'bs> { let words = self.source.words(); let sw = self.start / WORD_BITS; let so = self.start % WORD_BITS; + debug_assert!(!WORD_ALIGNED || so == 0); + debug_assert!(!ND_WORD_ALIGNED || needle.start.is_multiple_of(WORD_BITS)); let needle_words = needle.source.words(); let needle_len = needle.bit_len; if WORD_ALIGNED || so == 0 { @@ -122,6 +129,8 @@ impl<'bs> BitStr<'bs> { let words = self.source.words(); let sw = self.start / WORD_BITS; let so = self.start % WORD_BITS; + debug_assert!(!HS_WORD_ALIGNED || so == 0); + debug_assert!(!ND_WORD_ALIGNED || needle.start.is_multiple_of(WORD_BITS)); let needle_words = needle.source.words(); let needle_len = needle.bit_len; if !HS_WORD_ALIGNED && so != 0 { diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs b/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs index 8ebc59a..f5ca1e7 100644 --- a/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs +++ b/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs @@ -1,3 +1,6 @@ +//! Alignment consts are caller guarantees when `true`; `false` makes no +//! alignment guarantee and retains the general path. + use crate::BitStr; use crate::traits::*; use crate::{WORD_BITS, low_mask}; @@ -15,12 +18,14 @@ impl<'bs> BitStr<'bs> { } let hs_base = self.start + offset; let nd_base = needle.start; + debug_assert!(!HS_WORD_ALIGNED || hs_base.is_multiple_of(WORD_BITS)); + debug_assert!(!ND_WORD_ALIGNED || nd_base.is_multiple_of(WORD_BITS)); let hs_words = self.source.words(); let nd_words = needle.source.words(); if n <= WORD_BITS { let mask = low_mask(n); - let h = hs_words.read_word_at::(hs_base); - let nd = nd_words.read_word_at::(nd_base); + let h = hs_words.read_word_at::(hs_base); + let nd = nd_words.read_word_at::(nd_base); return (h & mask) == (nd & mask); } let nd_is_aligned = ND_WORD_ALIGNED || nd_base % WORD_BITS == 0; @@ -39,7 +44,7 @@ impl<'bs> BitStr<'bs> { let rem = n % WORD_BITS; if rem > 0 { let mask = low_mask(rem); - let h = hs_words.read_word_at::(hs_base + full_words * WORD_BITS); + let h = hs_words.read_word_at::(hs_base + full_words * WORD_BITS); if (h & mask) != (nd_aligned[full_words] & mask) { return false; } @@ -49,8 +54,8 @@ impl<'bs> BitStr<'bs> { let full_words = n / WORD_BITS; for i in 0..full_words { let pos = i * WORD_BITS; - let h = hs_words.read_word_at::(hs_base + pos); - let nd = nd_words.read_word_at::(nd_base + pos); + let h = hs_words.read_word_at::(hs_base + pos); + let nd = nd_words.read_word_at::(nd_base + pos); if h != nd { return false; } @@ -59,8 +64,8 @@ impl<'bs> BitStr<'bs> { if rem > 0 { let mask = low_mask(rem); let pos = full_words * WORD_BITS; - let h = hs_words.read_word_at::(hs_base + pos); - let nd = nd_words.read_word_at::(nd_base + pos); + let h = hs_words.read_word_at::(hs_base + pos); + let nd = nd_words.read_word_at::(nd_base + pos); if (h & mask) != (nd & mask) { return false; } diff --git a/src/bit_str/impls_for_ord/inner.rs b/src/bit_str/impls_for_ord/inner.rs index 0a562ba..96e2b40 100644 --- a/src/bit_str/impls_for_ord/inner.rs +++ b/src/bit_str/impls_for_ord/inner.rs @@ -1,3 +1,6 @@ +//! Alignment consts are caller guarantees when `true`; `false` makes no +//! alignment guarantee and retains the general path. + use crate::BitStr; use crate::traits::*; use crate::{WORD_BITS, low_mask}; @@ -18,6 +21,8 @@ impl<'bs> BitStr<'bs> { let nd_words = other.source.words(); let hs_base = self.start; let nd_base = other.start; + debug_assert!(!HS_WORD_ALIGNED || hs_base.is_multiple_of(WORD_BITS)); + debug_assert!(!ND_WORD_ALIGNED || nd_base.is_multiple_of(WORD_BITS)); let full = common / WORD_BITS; let nd_is_aligned = ND_WORD_ALIGNED || nd_base % WORD_BITS == 0; if nd_is_aligned { @@ -51,8 +56,8 @@ impl<'bs> BitStr<'bs> { if rem > 0 { let pos = full * WORD_BITS; let mask = low_mask(rem); - let a = hs_words.read_word_at::(hs_base + pos) & mask; - let b = nd_words.read_word_at::(nd_base + pos) & mask; + let a = hs_words.read_word_at::(hs_base + pos) & mask; + let b = nd_words.read_word_at::(nd_base + pos) & mask; if a != b { return a.bitwise_cmp(b); } diff --git a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs index 7289ee3..4a17438 100644 --- a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs +++ b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs @@ -23,22 +23,34 @@ impl BitString { return (w0.trailing_zeros() as usize).min(bit_len); } - // ── Tiny inputs — dispatch to BMI1 when available ──────── + // One- and two-word strings are common enough to avoid the generic scan. + if bit_len <= WORD_BITS { + return bit_len; + } + if bit_len <= WORD_BITS * 2 { + let used = bit_len - WORD_BITS; + // SAFETY: bit_len > WORD_BITS, so the second backing word exists. + let w1 = unsafe { *words_ptr.add(1) }; + let mask = if used == WORD_BITS { + u64::MAX + } else { + (1u64 << used) - 1 + }; + let w1 = w1 & mask; + if w1 == 0 { + return bit_len; + } + return WORD_BITS + w1.trailing_zeros() as usize; + } + + // ── Tiny inputs ─────────────────────────────────── let last_wi = (bit_len - 1) / WORD_BITS; let end_rem = bit_len % WORD_BITS; let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; if mid_end < SMALL_WORDS { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - if !cfg!(target_feature = "bmi1") && crate::cpuid::features().bmi1 { - // SAFETY: BMI1 confirmed by CPUID. `words_ptr` is valid - // for at least `mid_end + (end_rem != 0) as usize` u64 - // reads per BitString invariants. - return unsafe { leading_zeros_scalar_bmi(bit_len, words_ptr, mid_end, end_rem) }; - } let mut scanned = WORD_BITS; // word 0 already checked above - // SAFETY: `i` ranges in `1..mid_end`. `words` contains at - // least `mid_end + (end_rem != 0) as usize` elements by the - // BitString invariant (backing storage covers all bits). + // SAFETY: i ranges in 1..mid_end. words contains every + // backing word covered by bit_len. for i in 1..mid_end { let w = unsafe { *words_ptr.add(i) }; if w != 0 { @@ -47,9 +59,7 @@ impl BitString { scanned += WORD_BITS; } if end_rem != 0 { - // SAFETY: `mid_end` is the index of the last partial word; - // it is within bounds because `end_rem != 0` implies an - // extra word exists beyond `mid_end - 1`. + // SAFETY: a partial final word exists at mid_end. let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); if last == 0 { return bit_len; @@ -119,6 +129,10 @@ impl BitString { } let words_ptr = self.words.as_ptr(); + if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { + return count; + } + // ── Last partial word ──────────────────────────────────── let end_rem = bit_len % WORD_BITS; if end_rem != 0 { @@ -161,6 +175,10 @@ impl BitString { } let words_ptr = self.words.as_ptr(); + if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { + return count; + } + let end_rem = bit_len % WORD_BITS; if end_rem != 0 { let last_wi = (bit_len - 1) / WORD_BITS; @@ -193,40 +211,40 @@ impl BitString { } } -// ── BMI1-accelerated scalar path ─────────────────────────────────────── - -/// BMI1 variant of the tiny-input scalar loop. Uses `tzcnt` instead of -/// `bsf` for `trailing_zeros()`, eliminating the false output dependency. +/// Handles owned strings that fit in at most two words without entering the +/// generic reverse scanner. /// /// # Safety /// -/// `words_ptr` must be valid for at least `mid_end + (end_rem != 0) as -/// usize` u64 reads. Caller must guarantee BMI1 is available per CPUID. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "bmi1")] -unsafe fn leading_zeros_scalar_bmi( - bit_len: usize, - words_ptr: *const u64, - mid_end: usize, - end_rem: usize, -) -> usize { - let mut scanned = crate::WORD_BITS; // word 0 already checked by caller - // SAFETY: caller guarantees `words_ptr` is valid for the indices used. - for i in 1..mid_end { - let w = unsafe { *words_ptr.add(i) }; - if w != 0 { - return (scanned + w.trailing_zeros() as usize).min(bit_len); - } - scanned += crate::WORD_BITS; +/// `words_ptr` must point to the backing storage for `bit_len` bits. +#[inline] +unsafe fn trailing_small(words_ptr: *const u64, bit_len: usize) -> Option { + if bit_len > WORD_BITS * 2 { + return None; } - if end_rem != 0 { - // SAFETY: `mid_end` is the index of the last partial word; - // bounds guaranteed by caller. - let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); - if last == 0 { - return bit_len; - } - return (scanned + last.trailing_zeros() as usize).min(bit_len); + + let last_wi = (bit_len - 1) / WORD_BITS; + let used = bit_len - last_wi * WORD_BITS; + let mask = if used == WORD_BITS { + u64::MAX + } else { + (1u64 << used) - 1 + }; + // SAFETY: the caller guarantees backing storage for every covered word. + let last = unsafe { *words_ptr.add(last_wi) }; + let mismatch = (last ^ FILL) & mask; + if mismatch != 0 { + return Some((mismatch << (WORD_BITS - used)).leading_zeros() as usize); + } + if last_wi == 0 { + return Some(bit_len); + } + + // SAFETY: last_wi == 1, so the first backing word also exists. + let first_mismatch = unsafe { *words_ptr } ^ FILL; + if first_mismatch == 0 { + Some(bit_len) + } else { + Some(used + first_mismatch.leading_zeros() as usize) } - bit_len } diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs index 13f78ec..a395135 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs @@ -34,72 +34,38 @@ pub(super) fn bools_core(src: *const u8, bit_len: usize) -> Vec { /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) { - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. AVX2 availability was confirmed by CPUID. - unsafe { avx2::words(dst, src, bit_len) }; - return; - } - if f.sse2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. SSE2 availability was confirmed by CPUID. - unsafe { sse2::words(dst, src, bit_len) }; - return; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. NEON availability was confirmed by `#[target_feature]`. - unsafe { neon::words(dst, src, bit_len) }; - return; - } - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - #[allow(unused)] - unsafe { - scalar::words(dst, src, bit_len); - } + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. AVX2 availability was confirmed by `#[target_feature]`. + unsafe { avx2::words(dst, src, bit_len) }; + return; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. AVX2 availability was confirmed by `#[target_feature]`. - unsafe { avx2::words(dst, src, bit_len) }; - return; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. SSE2 availability was confirmed by `#[target_feature]`. - unsafe { sse2::words(dst, src, bit_len) }; - return; - } + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. SSE2 availability was confirmed by `#[target_feature]`. + unsafe { sse2::words(dst, src, bit_len) }; + return; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. NEON availability was confirmed by `#[target_feature]`. - unsafe { neon::words(dst, src, bit_len) }; - return; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. NEON availability was confirmed by `#[target_feature]`. + unsafe { neon::words(dst, src, bit_len) }; + return; + } - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - #[allow(unused)] - unsafe { - scalar::words(dst, src, bit_len); - } + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + #[allow(unused)] + unsafe { + scalar::words(dst, src, bit_len); } } diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs index 652ea50..0a60129 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs @@ -2,6 +2,9 @@ use crate::BitString; use alloc::vec::Vec; impl BitString { + // Keep the concrete iterator adapter visible together with the packing + // path so LLVM can optimize the temporary byte collection at the call site. + #[inline] pub(crate) fn from_bool_iter(iter: I) -> Self where I: IntoIterator, diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs index 7b99b5c..d4dd417 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs @@ -43,72 +43,38 @@ pub(super) fn str_core(src: *const u8, bit_len: usize) -> Result, (usiz /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) -> Option<(usize, u8)> { - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. - // AVX2 availability was confirmed by CPUID check above. - return unsafe { avx2::words(dst, src, bit_len) }; - } - if f.sse2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. - // SSE2 availability was confirmed by CPUID check above. - return unsafe { sse2::words(dst, src, bit_len) }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. - // NEON is guaranteed by `#[cfg(target_feature = "neon")]`. - return unsafe { neon::words(dst, src, bit_len) }; - } - #[allow(unused)] - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - unsafe { - scalar::words(dst, src, bit_len) - } + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // AVX2 is guaranteed by `#[cfg(target_feature = "avx2")]`. + return unsafe { avx2::words(dst, src, bit_len) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. - // AVX2 is guaranteed by `#[cfg(target_feature = "avx2")]`. - return unsafe { avx2::words(dst, src, bit_len) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. - // SSE2 is guaranteed by `#[cfg(target_feature = "sse2")]`. - return unsafe { sse2::words(dst, src, bit_len) }; - } + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // SSE2 is guaranteed by `#[cfg(target_feature = "sse2")]`. + return unsafe { sse2::words(dst, src, bit_len) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. - // NEON is guaranteed by `#[cfg(target_feature = "neon")]`. - return unsafe { neon::words(dst, src, bit_len) }; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // NEON is guaranteed by `#[cfg(target_feature = "neon")]`. + return unsafe { neon::words(dst, src, bit_len) }; + } - #[allow(unused)] - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - unsafe { - scalar::words(dst, src, bit_len) - } + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words(dst, src, bit_len) } } diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index e78ef13..58f9ba7 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -5,34 +5,17 @@ use super::*; impl BitString { /// Returns `true` if `pattern` matches the bits starting at `index`. /// - /// `BitString` is always word-aligned, so `HS_WORD_ALIGNED = true`. + /// The backing storage is word-aligned, but the comparison window is + /// aligned only when `index` is word-aligned. #[inline] pub fn matches_at_str(&self, index: usize, pattern: crate::BitStr<'_>) -> bool { - let view = self.as_bit_str(); - if index > view.bit_len { - return false; - } - if pattern.bit_len > view.bit_len - index { - return false; - } - if pattern.start % WORD_BITS == 0 { - view.matches_at_inner::(index, pattern) - } else { - view.matches_at_inner::(index, pattern) - } + self.as_bit_str().matches_at_str(index, pattern) } - /// `matches_at_str` when `pattern` is a `BitString` (both aligned). + /// `matches_at_str` when `pattern` is a `BitString`. #[inline] pub fn matches_at_string(&self, index: usize, pattern: &BitString) -> bool { - let view = self.as_bit_str(); - if index > view.bit_len { - return false; - } - if pattern.bit_len > view.bit_len - index { - return false; - } - view.matches_at_inner::(index, pattern.as_bit_str()) + self.as_bit_str().matches_at_string(index, pattern) } // ------------------------------------------------------------------- diff --git a/src/bit_string/impls_for_matching/impls_for_strip.rs b/src/bit_string/impls_for_matching/impls_for_strip.rs index 16de53c..8c09358 100644 --- a/src/bit_string/impls_for_matching/impls_for_strip.rs +++ b/src/bit_string/impls_for_matching/impls_for_strip.rs @@ -27,22 +27,14 @@ impl BitString { /// Strips `suffix` from the end, returning the remaining `BitString`. #[inline] pub fn strip_suffix_str(&self, suffix: crate::BitStr<'_>) -> Option { - let view = self.as_bit_str(); - let offset = view.bit_len - suffix.bit_len; - let ok = if suffix.start % WORD_BITS == 0 { - view.ends_with_inner::(suffix, offset) - } else { - view.ends_with_inner::(suffix, offset) - }; - ok.then(|| self.slice_until(self.bit_len - suffix.bit_len)) + self.ends_with_str(suffix) + .then(|| self.slice_until(self.bit_len - suffix.bit_len)) } /// `strip_suffix_str` when both sides are `BitString`. #[inline] pub fn strip_suffix_string(&self, suffix: &BitString) -> Option { - let view = self.as_bit_str(); - let offset = view.bit_len - suffix.as_bit_str().bit_len; - let ok = view.ends_with_inner::(suffix.as_bit_str(), offset); - ok.then(|| self.slice_until(self.bit_len - suffix.as_bit_str().bit_len)) + self.ends_with_string(suffix) + .then(|| self.slice_until(self.bit_len - suffix.bit_len)) } } diff --git a/src/cpuid.rs b/src/cpuid.rs deleted file mode 100644 index 3734c16..0000000 --- a/src/cpuid.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! CPU feature detection — once per process, shared by all SIMD dispatch. -//! -//! Runs CPUID exactly once on first access via `OnceCell`. All runtime -//! dispatch sites read from this single cache via `crate::cpuid::features()`. - -use once_cell::sync::OnceCell; - -/// Cached CPU feature flags. All fields are `false` on non-x86 targets. -#[allow(dead_code)] -pub(crate) struct CpuFeatures { - pub(crate) avx2: bool, - pub(crate) bmi1: bool, - pub(crate) sse41: bool, - pub(crate) ssse3: bool, - pub(crate) sse2: bool, -} - -static FEATURES: OnceCell = OnceCell::new(); - -/// Returns a reference to the process-lifetime CPU feature cache. -/// -/// CPUID runs at most once, on the very first call. -#[inline] -pub(crate) fn features() -> &'static CpuFeatures { - FEATURES.get_or_init(|| { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - // SAFETY: `__cpuid_count` is always safe — read-only instruction. - #[cfg(target_arch = "x86_64")] - let (leaf1, leaf7) = unsafe { - ( - core::arch::x86_64::__cpuid_count(1, 0), - core::arch::x86_64::__cpuid_count(7, 0), - ) - }; - #[cfg(target_arch = "x86")] - let (leaf1, leaf7) = unsafe { - ( - core::arch::x86::__cpuid_count(1, 0), - core::arch::x86::__cpuid_count(7, 0), - ) - }; - CpuFeatures { - avx2: leaf7.ebx & (1 << 5) != 0, - bmi1: leaf7.ebx & (1 << 3) != 0, - sse41: leaf1.ecx & (1 << 19) != 0, - ssse3: leaf1.ecx & (1 << 9) != 0, - sse2: leaf1.edx & (1 << 26) != 0, - } - } - #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] - CpuFeatures { - avx2: false, - bmi1: false, - sse41: false, - ssse3: false, - sse2: false, - } - }) -} diff --git a/src/lib.rs b/src/lib.rs index d2248c3..e2864ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,6 @@ extern crate alloc; mod consts_for_bits; -mod cpuid; mod funcs_for_bits; pub(crate) mod traits; diff --git a/src/traits/words_arith/funcs_for_binary_core.rs b/src/traits/words_arith/funcs_for_binary_core.rs index 7b5c256..6b5a546 100644 --- a/src/traits/words_arith/funcs_for_binary_core.rs +++ b/src/traits/words_arith/funcs_for_binary_core.rs @@ -56,79 +56,44 @@ pub(super) fn assign(lhs: &mut [u64], rhs: &[u64]) { /// - `dst` must not overlap `rhs`. #[inline] unsafe fn dispatch(dst: *mut u64, lhs: *const u64, rhs: *const u64, len: usize) { - // ── Default: runtime SIMD detection ────────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. - unsafe { avx2::words::(dst, lhs, rhs, len) }; - return; - } - if f.sse2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. - unsafe { sse2::words::(dst, lhs, rhs, len) }; - return; - } - } - // Non-x86 fallbacks: NEON (aarch64), scalar. - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. - unsafe { neon::words::(dst, lhs, rhs, len) }; - return; - } - #[allow(unused)] - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - unsafe { - scalar::words::(dst, lhs, rhs, len); - } + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words::(dst, lhs, rhs, len) }; + return; } - // ── compile-time-dispatch: existing #[cfg] cascade ─────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words::(dst, lhs, rhs, len) }; - return; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words::(dst, lhs, rhs, len) }; - return; - } + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words::(dst, lhs, rhs, len) }; + return; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words::(dst, lhs, rhs, len) }; - return; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words::(dst, lhs, rhs, len) }; + return; + } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words::(dst, lhs, rhs, len); - } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words::(dst, lhs, rhs, len); } } diff --git a/src/traits/words_arith/funcs_for_not_core.rs b/src/traits/words_arith/funcs_for_not_core.rs index edf5088..d619286 100644 --- a/src/traits/words_arith/funcs_for_not_core.rs +++ b/src/traits/words_arith/funcs_for_not_core.rs @@ -52,74 +52,42 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize) { /// - be exactly equal to `src`. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u64, len: usize) { - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. - unsafe { avx2::words(dst, src, len) }; - return; - } - if f.sse2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. - unsafe { sse2::words(dst, src, len) }; - return; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. - unsafe { neon::words(dst, src, len) }; - return; - } - #[allow(unused)] - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - unsafe { - scalar::words(dst, src, len) - }; + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words(dst, src, len) }; + return; } - - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words(dst, src, len) }; - return; - } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words(dst, src, len) }; - return; - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words(dst, src, len) }; - return; - } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, len) - }; + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words(dst, src, len) }; + return; } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words(dst, src, len) }; + return; + } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words(dst, src, len) + }; } #[allow(unused)] diff --git a/src/traits/words_arith/funcs_for_shl_core.rs b/src/traits/words_arith/funcs_for_shl_core.rs index 75ca055..42ab3dc 100644 --- a/src/traits/words_arith/funcs_for_shl_core.rs +++ b/src/traits/words_arith/funcs_for_shl_core.rs @@ -59,74 +59,42 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize, amount: usize) { /// - be exactly equal to `src`. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. - unsafe { avx2::words(dst, src, word_len, amount) }; - return; - } - if f.sse2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. - unsafe { sse2::words(dst, src, word_len, amount) }; - return; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. - unsafe { neon::words(dst, src, word_len, amount) }; - return; - } - #[allow(unused)] - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - unsafe { - scalar::words(dst, src, word_len, amount) - }; + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words(dst, src, word_len, amount) }; + return; } - - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words(dst, src, word_len, amount) }; - return; - } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words(dst, src, word_len, amount) }; - return; - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words(dst, src, word_len, amount) }; - return; - } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, word_len, amount) - }; + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words(dst, src, word_len, amount) }; + return; } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words(dst, src, word_len, amount) }; + return; + } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words(dst, src, word_len, amount) + }; } #[inline] diff --git a/src/traits/words_arith/funcs_for_shr_core.rs b/src/traits/words_arith/funcs_for_shr_core.rs index 020e2fa..6086500 100644 --- a/src/traits/words_arith/funcs_for_shr_core.rs +++ b/src/traits/words_arith/funcs_for_shr_core.rs @@ -59,74 +59,42 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize, amount: usize) { /// - be exactly equal to `src`. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. - unsafe { avx2::words(dst, src, word_len, amount) }; - return; - } - if f.sse2 { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. - unsafe { sse2::words(dst, src, word_len, amount) }; - return; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. - unsafe { neon::words(dst, src, word_len, amount) }; - return; - } - #[allow(unused)] - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - unsafe { - scalar::words(dst, src, word_len, amount) - }; + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words(dst, src, word_len, amount) }; + return; } - - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words(dst, src, word_len, amount) }; - return; - } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words(dst, src, word_len, amount) }; - return; - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words(dst, src, word_len, amount) }; - return; - } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, word_len, amount) - }; + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words(dst, src, word_len, amount) }; + return; } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words(dst, src, word_len, amount) }; + return; + } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words(dst, src, word_len, amount) + }; } #[inline] diff --git a/src/traits/words_edit.rs b/src/traits/words_edit.rs index b5a28eb..9e984f8 100644 --- a/src/traits/words_edit.rs +++ b/src/traits/words_edit.rs @@ -25,6 +25,7 @@ pub(crate) trait WordsEdit { /// Bits past the end of `self` are silently treated as zero. /// When `WORD_ALIGNED` is `true`, `bit_start % WORD_BITS == 0` is /// guaranteed — the cross-word stitch is eliminated at compile time. + /// When it is `false`, no alignment guarantee is made. fn read_word_at(&self, bit_start: usize) -> u64; /// Writes the low `len` bits of `value` into `self` starting at `bit_start`. @@ -35,6 +36,7 @@ pub(crate) trait WordsEdit { /// the value is split across two consecutive words. /// When `WORD_ALIGNED` is `true`, `bit_start % WORD_BITS == 0` is /// guaranteed — the cross-word spill is eliminated at compile time. + /// When it is `false`, no alignment guarantee is made. fn write_word_at(&mut self, bit_start: usize, value: u64, len: usize); /// Captures a snapshot of `len` bits starting at `start` for deferred diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs index 4cb86d8..600af9a 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs @@ -30,73 +30,38 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi return; } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). AVX2 availability was confirmed by CPUID. - unsafe { avx2::copy_words_shifted(dst, src, count, shift) }; - return; - } - if f.sse2 { - // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). SSE2 availability was confirmed by CPUID. - unsafe { sse2::copy_words_shifted(dst, src, count, shift) }; - return; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). NEON availability is confirmed at compile time by `target_feature = "neon"`. - unsafe { neon::copy_words_shifted(dst, src, count, shift) }; - return; - } - #[allow(unused)] - { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - } - } + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). AVX2 availability is confirmed at compile time. + unsafe { avx2::copy_words_shifted(dst, src, count, shift) }; + return; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). AVX2 availability is confirmed at compile time. - unsafe { avx2::copy_words_shifted(dst, src, count, shift) }; - return; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). SSE2 availability is confirmed at compile time. - unsafe { sse2::copy_words_shifted(dst, src, count, shift) }; - return; - } + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). SSE2 availability is confirmed at compile time. + unsafe { sse2::copy_words_shifted(dst, src, count, shift) }; + return; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). NEON availability is confirmed at compile time. - unsafe { neon::copy_words_shifted(dst, src, count, shift) }; - return; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). NEON availability is confirmed at compile time. + unsafe { neon::copy_words_shifted(dst, src, count, shift) }; + return; + } - #[allow(unused)] - { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - } + #[allow(unused)] + { + for i in 0..count { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); } } } diff --git a/src/traits/words_edit/impls_for_u64_slice.rs b/src/traits/words_edit/impls_for_u64_slice.rs index 91739cb..8384b91 100644 --- a/src/traits/words_edit/impls_for_u64_slice.rs +++ b/src/traits/words_edit/impls_for_u64_slice.rs @@ -45,6 +45,7 @@ impl WordsEdit for [u64] { /// is stitched in from the next word. #[inline] fn read_word_at(&self, bit_start: usize) -> u64 { + debug_assert!(!WORD_ALIGNED || bit_start.is_multiple_of(WORD_BITS)); let word = bit_start / WORD_BITS; if WORD_ALIGNED { @@ -72,6 +73,7 @@ impl WordsEdit for [u64] { value: u64, len: usize, ) { + debug_assert!(!WORD_ALIGNED || bit_start.is_multiple_of(WORD_BITS)); let value = value & low_mask(len); let word = bit_start / WORD_BITS; diff --git a/src/traits/words_eq.rs b/src/traits/words_eq.rs index f35829d..f654922 100644 --- a/src/traits/words_eq.rs +++ b/src/traits/words_eq.rs @@ -19,6 +19,7 @@ pub(crate) trait WordsEq { /// `haystack_shift` is `original_offset % WORD_BITS`. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. + /// When it is `false`, no alignment guarantee is made. fn eq_words( &self, needle: &[u64], diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs index d048f40..3c2a9cb 100644 --- a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs @@ -22,74 +22,39 @@ pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool return true; } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. - return unsafe { avx2::eq_words(src, other, count) }; - } - if f.sse41 { - // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. - return unsafe { sse41::eq_words(src, other, count) }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: `src`/`other` are valid for `count` words. Backend feature is enabled by the `#[cfg]` gate on this block. - return unsafe { neon::eq_words(src, other, count) }; - } - #[allow(unused)] - { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - true - } + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { avx2::eq_words(src, other, count) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: `src`/`other` are valid for `count` words. Backend feature is guaranteed by compile-time `#[cfg]` gate. - return unsafe { avx2::eq_words(src, other, count) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: `src`/`other` are valid for `count` words. Backend feature is guaranteed by compile-time `#[cfg]` gate. - return unsafe { sse41::eq_words(src, other, count) }; - } + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { sse41::eq_words(src, other, count) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: `src`/`other` are valid for `count` words. Backend feature is enabled by the `#[cfg]` gate on this block. - return unsafe { neon::eq_words(src, other, count) }; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is enabled by the `#[cfg]` gate on this block. + return unsafe { neon::eq_words(src, other, count) }; + } - #[allow(unused)] - { - for i in 0..count { - if src[i] != other[i] { - return false; - } + #[allow(unused)] + { + for i in 0..count { + if src[i] != other[i] { + return false; } - true } + true } } diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs index 4e9e885..2a9fd69 100644 --- a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs @@ -26,78 +26,41 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift return true; } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. - return unsafe { avx2::eq_words_unaligned(src, other, count, shift) }; - } - if f.sse41 { - // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. - return unsafe { sse41::eq_words_unaligned(src, other, count, shift) }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is enabled by the `#[cfg]` gate on this block. - return unsafe { neon::eq_words_unaligned(src, other, count, shift) }; - } - #[allow(unused)] - { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - } - true - } + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { avx2::eq_words_unaligned(src, other, count, shift) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is guaranteed by compile-time `#[cfg]` gate. - return unsafe { avx2::eq_words_unaligned(src, other, count, shift) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is guaranteed by compile-time `#[cfg]` gate. - return unsafe { sse41::eq_words_unaligned(src, other, count, shift) }; - } + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { sse41::eq_words_unaligned(src, other, count, shift) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is enabled by the `#[cfg]` gate on this block. - return unsafe { neon::eq_words_unaligned(src, other, count, shift) }; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is enabled by the `#[cfg]` gate on this block. + return unsafe { neon::eq_words_unaligned(src, other, count, shift) }; + } - #[allow(unused)] - { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } + #[allow(unused)] + { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; } - true } + true } } diff --git a/src/traits/words_eq/impls_for_u64_slice.rs b/src/traits/words_eq/impls_for_u64_slice.rs index 9e13544..aab8473 100644 --- a/src/traits/words_eq/impls_for_u64_slice.rs +++ b/src/traits/words_eq/impls_for_u64_slice.rs @@ -8,6 +8,7 @@ impl WordsEq for [u64] { full_words: usize, haystack_shift: usize, ) -> bool { + debug_assert!(!HS_WORD_ALIGNED || haystack_shift == 0); if HS_WORD_ALIGNED || haystack_shift == 0 { super::funcs_for_eq_words_aligned_core::eq_words_aligned(self, needle, full_words) } else { diff --git a/src/traits/words_find/funcs_for_contains_core.rs b/src/traits/words_find/funcs_for_contains_core.rs index 28d64f0..dea6291 100644 --- a/src/traits/words_find/funcs_for_contains_core.rs +++ b/src/traits/words_find/funcs_for_contains_core.rs @@ -50,133 +50,70 @@ where ); } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { - avx2::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) - }; - } - if f.sse41 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { - sse41::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) - }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { - neon::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) - }; - } - #[allow(unused)] - scalar( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + avx2::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { - avx2::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) - }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { - sse41::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) - }; - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { - neon::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) - }; - } + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + sse41::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; + } - #[allow(unused)] - scalar( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + neon::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; } + + #[allow(unused)] + scalar( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) } // --------------------------------------------------------------------------- diff --git a/src/traits/words_find/funcs_for_find_core.rs b/src/traits/words_find/funcs_for_find_core.rs index 33c3ca9..f2d3674 100644 --- a/src/traits/words_find/funcs_for_find_core.rs +++ b/src/traits/words_find/funcs_for_find_core.rs @@ -29,69 +29,36 @@ where return scalar_find(haystack, needle_first, needle_mask, last_start, verify); } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { - avx2::find(haystack, needle_first, needle_mask, last_start, verify) - }; - } - if f.sse41 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { - sse41::find(haystack, needle_first, needle_mask, last_start, verify) - }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::find(haystack, needle_first, needle_mask, last_start, verify) }; - } - #[allow(unused)] - scalar_find(haystack, needle_first, needle_mask, last_start, verify) + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::find(haystack, needle_first, needle_mask, last_start, verify) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { avx2::find(haystack, needle_first, needle_mask, last_start, verify) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { sse41::find(haystack, needle_first, needle_mask, last_start, verify) }; - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::find(haystack, needle_first, needle_mask, last_start, verify) }; - } + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::find(haystack, needle_first, needle_mask, last_start, verify) }; + } - #[allow(unused)] - scalar_find(haystack, needle_first, needle_mask, last_start, verify) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::find(haystack, needle_first, needle_mask, last_start, verify) }; } + + #[allow(unused)] + scalar_find(haystack, needle_first, needle_mask, last_start, verify) } // --------------------------------------------------------------------------- diff --git a/src/traits/words_find/funcs_for_rfind_core.rs b/src/traits/words_find/funcs_for_rfind_core.rs index 761087e..68ecbe1 100644 --- a/src/traits/words_find/funcs_for_rfind_core.rs +++ b/src/traits/words_find/funcs_for_rfind_core.rs @@ -31,69 +31,36 @@ where return scalar_rfind(haystack, needle_key, needle_mask, last_start, verify); } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { - avx2::rfind(haystack, needle_key, needle_mask, last_start, verify) - }; - } - if f.sse41 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { - sse41::rfind(haystack, needle_key, needle_mask, last_start, verify) - }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::rfind(haystack, needle_key, needle_mask, last_start, verify) }; - } - #[allow(unused)] - scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::rfind(haystack, needle_key, needle_mask, last_start, verify) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { avx2::rfind(haystack, needle_key, needle_mask, last_start, verify) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { sse41::rfind(haystack, needle_key, needle_mask, last_start, verify) }; - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::rfind(haystack, needle_key, needle_mask, last_start, verify) }; - } + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::rfind(haystack, needle_key, needle_mask, last_start, verify) }; + } - #[allow(unused)] - scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::rfind(haystack, needle_key, needle_mask, last_start, verify) }; } + + #[allow(unused)] + scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) } // --------------------------------------------------------------------------- diff --git a/src/traits/words_ord.rs b/src/traits/words_ord.rs index ad99a48..97e7bf9 100644 --- a/src/traits/words_ord.rs +++ b/src/traits/words_ord.rs @@ -23,6 +23,7 @@ pub(crate) trait WordsOrd { /// `haystack_shift` is the intra-word offset within the first word. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. + /// When it is `false`, no alignment guarantee is made. fn cmp_words( &self, needle: &[u64], diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs index 10f54b6..d27def7 100644 --- a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs @@ -14,65 +14,36 @@ pub(super) fn cmp_aligned_words(src: &[u64], other: &[u64], count: usize) -> Opt return scalar_cmp_aligned(src, other, count); } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { avx2::cmp_aligned(src, other, count) }; - } - if f.sse41 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { sse41::cmp_aligned(src, other, count) }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::cmp_aligned(src, other, count) }; - } - #[allow(unreachable_code)] - scalar_cmp_aligned(src, other, count) + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::cmp_aligned(src, other, count) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { avx2::cmp_aligned(src, other, count) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { sse41::cmp_aligned(src, other, count) }; - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::cmp_aligned(src, other, count) }; - } + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::cmp_aligned(src, other, count) }; + } - #[allow(unreachable_code)] - scalar_cmp_aligned(src, other, count) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::cmp_aligned(src, other, count) }; } + + #[allow(unreachable_code)] + scalar_cmp_aligned(src, other, count) } #[inline] diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs index 104b8d3..cbb6fa4 100644 --- a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs @@ -24,65 +24,36 @@ pub(super) fn cmp_unaligned_words( return scalar_cmp_unaligned(src, other, count, shift); } - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { avx2::cmp_unaligned(src, other, count, shift) }; - } - if f.sse41 { - // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. - return unsafe { sse41::cmp_unaligned(src, other, count, shift) }; - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::cmp_unaligned(src, other, count, shift) }; - } - #[allow(unreachable_code)] - scalar_cmp_unaligned(src, other, count, shift) + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::cmp_unaligned(src, other, count, shift) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { avx2::cmp_unaligned(src, other, count, shift) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { sse41::cmp_unaligned(src, other, count, shift) }; - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: pointer validity guaranteed by caller. Backend is always safe / - // enabled by `#[target_feature]` at compile time. - return unsafe { neon::cmp_unaligned(src, other, count, shift) }; - } + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::cmp_unaligned(src, other, count, shift) }; + } - #[allow(unreachable_code)] - scalar_cmp_unaligned(src, other, count, shift) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::cmp_unaligned(src, other, count, shift) }; } + + #[allow(unreachable_code)] + scalar_cmp_unaligned(src, other, count, shift) } #[inline] diff --git a/src/traits/words_ord/impls_for_u64_slice.rs b/src/traits/words_ord/impls_for_u64_slice.rs index 5cf2780..802e923 100644 --- a/src/traits/words_ord/impls_for_u64_slice.rs +++ b/src/traits/words_ord/impls_for_u64_slice.rs @@ -12,6 +12,7 @@ impl WordsOrd for [u64] { full_words: usize, haystack_shift: usize, ) -> Option { + debug_assert!(!HS_WORD_ALIGNED || haystack_shift == 0); if HS_WORD_ALIGNED || haystack_shift == 0 { funcs_for_cmp_aligned_core::cmp_aligned_words(self, needle, full_words) } else { diff --git a/src/traits/words_scan.rs b/src/traits/words_scan.rs index 741c3d5..df550b0 100644 --- a/src/traits/words_scan.rs +++ b/src/traits/words_scan.rs @@ -9,6 +9,7 @@ pub(crate) trait WordsScan { /// `start_offset` is `physical_start % WORD_BITS`. /// When `WORD_ALIGNED` is `true`, `start_offset` is guaranteed to be 0 /// and the first-word phase is eliminated at compile time. + /// When it is `false`, no alignment guarantee is made. fn leading_value_bits( &self, start_offset: u32, diff --git a/src/traits/words_scan/funcs_for_count_ones.rs b/src/traits/words_scan/funcs_for_count_ones.rs index baf625d..e23ac4f 100644 --- a/src/traits/words_scan/funcs_for_count_ones.rs +++ b/src/traits/words_scan/funcs_for_count_ones.rs @@ -47,78 +47,41 @@ unsafe fn dispatch(src: *const u64, len: usize) -> usize { // Small inputs: skip SIMD setup overhead, go straight to scalar popcnt. // Threshold equals the backend's LANES count. - // ── Default: runtime SIMD detection ───────────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let f = crate::cpuid::features(); - if f.avx2 { - if len >= 4 { - // SAFETY: `src` is valid for `len` words. Backend selected via CPUID verification. - return unsafe { avx2::count_words(src, len) }; - } - } - if f.ssse3 { - if len >= 2 { - // SAFETY: `src` is valid for `len` words. Backend selected via CPUID verification. - return unsafe { ssse3::count_words(src, len) }; - } - } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - if len >= 2 { - // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` gate on NEON availability. - return unsafe { neon::count_words(src, len) }; - } - } - #[allow(unused)] - // SAFETY: pointer validity guaranteed by caller. Scalar backend is always safe. - unsafe { - scalar::count_words(src, len) + if len >= 4 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. + return unsafe { avx2::count_words(src, len) }; } } - // ── compile-time-dispatch: pure #[cfg] cascade ────────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "ssse3", + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - if len >= 4 { - // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. - return unsafe { avx2::count_words(src, len) }; - } - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "ssse3", - not(target_feature = "avx2") - ))] - { - if len >= 2 { - // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. - return unsafe { ssse3::count_words(src, len) }; - } + if len >= 2 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. + return unsafe { ssse3::count_words(src, len) }; } + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - if len >= 2 { - // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. - return unsafe { neon::count_words(src, len) }; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + if len >= 2 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. + return unsafe { neon::count_words(src, len) }; } + } - #[allow(unused)] - // SAFETY: pointer validity guaranteed by caller. Scalar backend is always safe. - unsafe { - scalar::count_words(src, len) - } + #[allow(unused)] + // SAFETY: pointer validity guaranteed by caller. Scalar backend is always safe. + unsafe { + scalar::count_words(src, len) } } diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs index a5cb6c5..a53f490 100644 --- a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs @@ -2,19 +2,9 @@ //! //! Parameterised by `const FILL: u64` and `const WORD_ALIGNED: bool`. +use super::count_matching; use crate::{SMALL_WORDS, WORD_BITS, low_mask}; -// ── Scalar helper ────────────────────────────────────────────────────── - -#[inline] -fn count_trailing(val: u64) -> usize { - if FILL == 0 { - val.trailing_zeros() as usize - } else { - (!val).trailing_zeros() as usize - } -} - // ── Dispatch ─────────────────────────────────────────────────────────── #[inline] @@ -23,6 +13,7 @@ pub(crate) fn leading( start_offset: u32, bit_len: usize, ) -> usize { + debug_assert!(!WORD_ALIGNED || start_offset == 0); if bit_len == 0 { return 0; } @@ -38,7 +29,7 @@ pub(crate) fn leading( if !WORD_ALIGNED && start_offset != 0 { let first_val = bits[0] >> start_offset; let first_limit = (WORD_BITS - start_offset as usize).min(bit_len); - let first_count = count_trailing::(first_val).min(first_limit); + let first_count = count_matching::(first_val).min(first_limit); if first_count < first_limit { return first_count; } @@ -55,7 +46,7 @@ pub(crate) fn leading( for i in 0..total { let w = bits[wi + i]; if w != FILL { - return (scanned + count_trailing::(w)).min(bit_len); + return (scanned + count_matching::(w)).min(bit_len); } scanned += WORD_BITS; } @@ -73,74 +64,44 @@ pub(crate) fn leading( // branch), so `base` is valid for at least one u64 read. let w0 = unsafe { *base }; if w0 != FILL { - return (scanned + count_trailing::(w0)).min(bit_len); + return (scanned + count_matching::(w0)).min(bit_len); } // Start SIMD from `base` (not base+1). Word 0 is // double-checked (fast path + SIMD) but this keeps the // iteration count a clean multiple of the SIMD stride. let mut p = base; - // ── Default: runtime SIMD detection ───────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - if crate::cpuid::features().avx2 { - // SAFETY: CPUID confirmed AVX2 is available. - // `p` through `end` are within the input slice. - p = unsafe { avx2::leading_scan::(p, end, base, total) }; - } else { - // SAFETY: SSE2 is baseline on x86-64. - // `p` through `end` are within the input slice. - p = unsafe { sse2::leading_scan::(p, end, total) }; - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: NEON is available per `#[cfg]` gate. - // `p` through `end` are within the input slice. - p = unsafe { neon::leading_scan::(p, end, total) }; - } - #[allow(unused)] - { - // Scalar fallback: `p` stays at base; shared tail - // below scans word-by-word. - } + // SAFETY: AVX2 is guaranteed by compile-time + // `#[cfg]` gate. + p = unsafe { avx2::leading_scan::(p, end, base, total) }; } - // ── compile-time-dispatch: pure #[cfg] cascade ────────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + any(target_feature = "sse2", target_feature = "ssse3"), + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: AVX2 is guaranteed by compile-time - // `#[cfg]` gate. - p = unsafe { avx2::leading_scan::(p, end, base, total) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - any(target_feature = "sse2", target_feature = "ssse3"), - not(target_feature = "avx2") - ))] - { - // SAFETY: SSE2 is guaranteed by compile-time - // `#[cfg]` gate. - p = unsafe { sse2::leading_scan::(p, end, total) }; - } + // SAFETY: SSE2 is guaranteed by compile-time + // `#[cfg]` gate. + p = unsafe { sse2::leading_scan::(p, end, total) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: NEON is guaranteed by compile-time - // `#[cfg]` gate. - p = unsafe { neon::leading_scan::(p, end, total) }; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: NEON is guaranteed by compile-time + // `#[cfg]` gate. + p = unsafe { neon::leading_scan::(p, end, total) }; + } - #[allow(unused)] - { - // Scalar fallback: `p` stays at base. - } + #[allow(unused)] + { + // Scalar fallback: `p` stays at base. } // ── Post-SIMD: shared scalar remainder ───────────────── @@ -157,7 +118,7 @@ pub(crate) fn leading( for _ in 0..rem { unsafe { if *p != FILL { - scanned += count_trailing::(*p); + scanned += count_matching::(*p); return (scanned).min(bit_len); } scanned += WORD_BITS; @@ -170,7 +131,7 @@ pub(crate) fn leading( if end_rem != 0 && wi == last_wi { let last_val = bits[wi] & low_mask(end_rem); - scanned += count_trailing::(last_val).min(end_rem); + scanned += count_matching::(last_val).min(end_rem); } scanned.min(bit_len) @@ -185,17 +146,17 @@ pub(crate) fn leading( mod avx2 { #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, - _mm256_xor_si256, + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, + _mm256_testz_si256, _mm256_xor_si256, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, - _mm256_xor_si256, + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, + _mm256_testz_si256, _mm256_xor_si256, }; const LANES: usize = 4; - const STRIDE: usize = 8; // 2 × LANES for unrolled iteration + const STRIDE: usize = 16; // 4 × LANES for unrolled iteration const ALIGN_THRESHOLD: usize = 128; /// AVX2 forward scan: advances `p` past all-FILL 256-bit chunks. @@ -207,7 +168,7 @@ mod avx2 { /// /// # Safety /// - /// Caller must ensure AVX2 is available (checked via CPUID). + /// Caller must ensure AVX2 is enabled for the compilation target. /// `p` through `end` must be valid for u64 reads. #[target_feature(enable = "avx2")] pub(super) unsafe fn leading_scan( @@ -216,8 +177,8 @@ mod avx2 { base: *const u64, total: usize, ) -> *const u64 { - // SAFETY: only callable when AVX2 is available (caller verified - // via CPUID). All pointer arithmetic stays within bounds. + // SAFETY: AVX2 is enabled for this compilation target. All pointer + // arithmetic stays within bounds. unsafe { if total >= ALIGN_THRESHOLD { // Distance in words to the next 32-byte boundary. @@ -237,7 +198,12 @@ mod avx2 { if FILL == 0 { let d0 = _mm256_load_si256(p.cast::<__m256i>()); let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); - if _mm256_testz_si256(d0, d0) == 0 || _mm256_testz_si256(d1, d1) == 0 { + let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); + let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { break; } } else { @@ -246,7 +212,14 @@ mod avx2 { let x0 = _mm256_xor_si256(d0, fill_vec); let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); let x1 = _mm256_xor_si256(d1, fill_vec); - if _mm256_testz_si256(x0, x0) == 0 || _mm256_testz_si256(x1, x1) == 0 { + let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { break; } } @@ -254,13 +227,18 @@ mod avx2 { iters -= 1; } } else { - // 2×-unrolled unaligned path. + // 4×-unrolled unaligned path. let mut iters = total / STRIDE; while iters > 0 { if FILL == 0 { let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); - if _mm256_testz_si256(d0, d0) == 0 || _mm256_testz_si256(d1, d1) == 0 { + let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); + let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { break; } } else { @@ -269,7 +247,14 @@ mod avx2 { let x0 = _mm256_xor_si256(d0, fill_vec); let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); let x1 = _mm256_xor_si256(d1, fill_vec); - if _mm256_testz_si256(x0, x0) == 0 || _mm256_testz_si256(x1, x1) == 0 { + let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { break; } } @@ -351,8 +336,7 @@ mod sse2 { end: *const u64, total: usize, ) -> *const u64 { - // SAFETY: only callable when SSE2 is available (caller verified - // via CPUID, or SSE2 is baseline). + // SAFETY: SSE2 is enabled for this compilation target. unsafe { let mut iters = total / LANES_2X; while iters > 0 { diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs index 24e6f2f..884d8c2 100644 --- a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs @@ -4,20 +4,9 @@ //! When `WORD_ALIGNED` is `true` the caller guarantees `start_offset == 0`, //! allowing the compiler to eliminate the first-word LZCNT phase. +use super::count_matching; use crate::{SMALL_WORDS, WORD_BITS}; -// ── Scalar helper ────────────────────────────────────────────────────── - -/// Counts leading bits within a single u64 word that match `FILL`. -#[inline] -fn count_leading(val: u64) -> usize { - if FILL == 0 { - val.leading_zeros() as usize - } else { - (!val).leading_zeros() as usize - } -} - // ── Dispatch ─────────────────────────────────────────────────────────── #[inline] @@ -26,6 +15,7 @@ pub(crate) fn trailing( start_offset: u32, bit_len: usize, ) -> usize { + debug_assert!(!WORD_ALIGNED || start_offset == 0); if bit_len == 0 { return 0; } @@ -44,7 +34,7 @@ pub(crate) fn trailing( end_rem }; let shifted = bits[last_wi] << (WORD_BITS - end_rem); - let last_count = count_leading::(shifted).min(last_limit); + let last_count = count_matching::(shifted).min(last_limit); if last_count < last_limit { return last_count; } @@ -74,7 +64,7 @@ pub(crate) fn trailing( { let w = bits[wi_end]; if w != FILL { - scanned += count_leading::(w); + scanned += count_matching::(w); return scanned.min(bit_len); } } @@ -84,7 +74,7 @@ pub(crate) fn trailing( while done < total_words { let wi = wi_end - done; if bits[wi] != FILL { - scanned += count_leading::(bits[wi]); + scanned += count_matching::(bits[wi]); return scanned.min(bit_len); } scanned += WORD_BITS; @@ -92,75 +82,40 @@ pub(crate) fn trailing( } // All full words match FILL — skip SIMD. } else { - // ── Default: runtime SIMD detection ─────────────────── - #[cfg(not(feature = "compile-time-dispatch"))] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let done_before = done; - if crate::cpuid::features().avx2 { - // SAFETY: CPUID confirmed AVX2 is available. - done = - unsafe { avx2::trailing_scan::(ptr, wi_end, done, total_words) }; - } else { - // SAFETY: SSE2 is baseline on x86-64. - done = - unsafe { sse2::trailing_scan::(ptr, wi_end, done, total_words) }; - } - scanned += (done - done_before) * WORD_BITS; - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - let done_before = done; - // SAFETY: NEON is available per `#[cfg]` gate. - done = unsafe { neon::trailing_scan::(ptr, wi_end, done, total_words) }; - scanned += (done - done_before) * WORD_BITS; - } - #[allow(unused)] - { - // Scalar fallback: `done` stays unchanged; shared - // tail below scans word-by-word. - } + let done_before = done; + // SAFETY: AVX2 is guaranteed by compile-time gate. + done = unsafe { avx2::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; } - // ── compile-time-dispatch: pure #[cfg] cascade ──────── - #[cfg(feature = "compile-time-dispatch")] + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + any(target_feature = "sse2", target_feature = "ssse3"), + not(target_feature = "avx2") + ))] { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - let done_before = done; - // SAFETY: AVX2 is guaranteed by compile-time gate. - done = unsafe { avx2::trailing_scan::(ptr, wi_end, done, total_words) }; - scanned += (done - done_before) * WORD_BITS; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - any(target_feature = "sse2", target_feature = "ssse3"), - not(target_feature = "avx2") - ))] - { - let done_before = done; - // SAFETY: SSE2 is guaranteed by compile-time gate. - done = unsafe { sse2::trailing_scan::(ptr, wi_end, done, total_words) }; - scanned += (done - done_before) * WORD_BITS; - } + let done_before = done; + // SAFETY: SSE2 is guaranteed by compile-time gate. + done = unsafe { sse2::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - let done_before = done; - // SAFETY: NEON is guaranteed by compile-time gate. - done = unsafe { neon::trailing_scan::(ptr, wi_end, done, total_words) }; - scanned += (done - done_before) * WORD_BITS; - } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + let done_before = done; + // SAFETY: NEON is guaranteed by compile-time gate. + done = unsafe { neon::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; + } - #[allow(unused)] - { - // Scalar fallback. - } + #[allow(unused)] + { + // Scalar fallback. } } // else (SIMD path) @@ -168,7 +123,7 @@ pub(crate) fn trailing( while done < total_words { let wi = wi_end - done; if bits[wi] != FILL { - scanned += count_leading::(bits[wi]); + scanned += count_matching::(bits[wi]); return scanned.min(bit_len); } scanned += WORD_BITS; @@ -179,7 +134,7 @@ pub(crate) fn trailing( // ── First-word partial (trailing side) ─────────────────────── if !WORD_ALIGNED && start_offset > 0 { let first_limit = WORD_BITS - start_offset as usize; - let first_count = count_leading::(bits[0]).min(first_limit); + let first_count = count_matching::(bits[0]).min(first_limit); scanned += first_count; } @@ -187,7 +142,7 @@ pub(crate) fn trailing( } // ═══════════════════════════════════════════════════════════════════════ -// AVX2 backend — 256-bit / 4-lane, unaligned loads only. +// AVX2 backend — 256-bit / 4-lane reverse scan. // ═══════════════════════════════════════════════════════════════════════ #[allow(unused)] @@ -195,15 +150,17 @@ pub(crate) fn trailing( mod avx2 { #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m256i, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, _mm256_xor_si256, + __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m256i, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, _mm256_xor_si256, + __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, }; const LANES: usize = 4; - const STRIDE: usize = 8; + const STRIDE: usize = 16; /// AVX2 reverse scan: scans backwards from `wi_end` and advances `done` /// past all-FILL 256-bit chunks. @@ -212,7 +169,7 @@ mod avx2 { /// /// # Safety /// - /// Caller must ensure AVX2 is available (checked via CPUID). + /// Caller must ensure AVX2 is enabled for the compilation target. /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. #[target_feature(enable = "avx2")] pub(super) unsafe fn trailing_scan( @@ -221,32 +178,41 @@ mod avx2 { mut done: usize, total_words: usize, ) -> usize { - // SAFETY: only callable when AVX2 is available (caller verified - // via CPUID). All pointer arithmetic stays within bounds. + // SAFETY: AVX2 is enabled for this compilation target. All pointer + // arithmetic stays within bounds. unsafe { - // 2×‑unrolled + // Four vectors are combined before testing. This keeps the + // all-matching hot path to one branch per 16 words, matching + // the forward scanner's throughput-oriented layout. while done + STRIDE <= total_words { let chunk_start = wi_end + 1 - (done + STRIDE); - let d0_ok = if FILL == 0 { - let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - _mm256_testz_si256(d, d) != 0 - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let x = _mm256_xor_si256(d, fill_vec); - _mm256_testz_si256(x, x) != 0 - }; - let d1_ok = if FILL == 0 { - let d = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); - _mm256_testz_si256(d, d) != 0 + if FILL == 0 { + let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); + let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } } else { let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); - let x = _mm256_xor_si256(d, fill_vec); - _mm256_testz_si256(x, x) != 0 - }; - if !d0_ok || !d1_ok { - break; + let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } } done += STRIDE; } @@ -326,8 +292,7 @@ mod sse2 { mut done: usize, total_words: usize, ) -> usize { - // SAFETY: only callable when SSE2 is available (caller verified - // via CPUID, or SSE2 is baseline). + // SAFETY: SSE2 is enabled for this compilation target. unsafe { while done + LANES_2X <= total_words { let chunk_start = wi_end + 1 - (done + LANES_2X); diff --git a/src/traits/words_scan/funcs_for_ends/mod.rs b/src/traits/words_scan/funcs_for_ends/mod.rs index fb50b0e..bfa6932 100644 --- a/src/traits/words_scan/funcs_for_ends/mod.rs +++ b/src/traits/words_scan/funcs_for_ends/mod.rs @@ -3,3 +3,16 @@ mod funcs_for_trailing_core; pub(crate) use funcs_for_leading_core::leading; pub(crate) use funcs_for_trailing_core::trailing; + +/// Counts consecutive bits matching `FILL` from either end of one word. +/// +/// `FROM_MSB` selects `leading_zeros`; otherwise the count starts at the LSB. +#[inline] +fn count_matching(word: u64) -> usize { + let mismatches = word ^ FILL; + if FROM_MSB { + mismatches.leading_zeros() as usize + } else { + mismatches.trailing_zeros() as usize + } +} diff --git a/tests/adversarial/tests_for_matching.rs b/tests/adversarial/tests_for_matching.rs index 6a9864a..0ebc21e 100644 --- a/tests/adversarial/tests_for_matching.rs +++ b/tests/adversarial/tests_for_matching.rs @@ -1,6 +1,88 @@ use super::*; use int_interval::UsizeCO; +fn alignment_pattern(len: usize) -> String { + (0..len) + .map(|i| if (i * 17 + i / 3) % 11 < 5 { '0' } else { '1' }) + .collect() +} + +fn padded_bits(text: &str, offset: usize) -> BitString { + bs(&cat(&[ + "1".repeat(offset).as_str(), + text, + "0".repeat(7).as_str(), + ])) +} + +#[test] +fn attack_matching_alignment_matrix_at_word_boundaries() { + for len in [0, 1, 63, 64, 65, 127, 128, 129] { + let text = alignment_pattern(len); + let owned = bs(&text); + + for hs_offset in [0, 3] { + let hs_source = padded_bits(&text, hs_offset); + let hs = hs_source + .as_bit_str() + .slice_from(hs_offset) + .slice_until(len); + + assert!(hs.matches_at_string(0, &owned), "len={len} hs={hs_offset}"); + assert!(hs.starts_with_string(&owned), "len={len} hs={hs_offset}"); + assert!(hs.ends_with_string(&owned), "len={len} hs={hs_offset}"); + + for nd_offset in [0, 5] { + let nd_source = padded_bits(&text, nd_offset); + let nd = nd_source + .as_bit_str() + .slice_from(nd_offset) + .slice_until(len); + + assert!( + hs.matches_at_str(0, nd), + "matches_at len={len} hs={hs_offset} nd={nd_offset}" + ); + assert!( + hs.starts_with_str(nd), + "starts_with len={len} hs={hs_offset} nd={nd_offset}" + ); + assert!( + hs.ends_with_str(nd), + "ends_with len={len} hs={hs_offset} nd={nd_offset}" + ); + + if len > 0 { + let mut first_diff = text.clone().into_bytes(); + first_diff[0] = if first_diff[0] == b'0' { b'1' } else { b'0' }; + let first_diff = String::from_utf8(first_diff).unwrap(); + let first_source = padded_bits(&first_diff, nd_offset); + let first = first_source + .as_bit_str() + .slice_from(nd_offset) + .slice_until(len); + assert!(!hs.matches_at_str(0, first)); + assert!(!hs.starts_with_str(first)); + assert!(!hs.ends_with_str(first)); + + let mut last_diff = text.clone().into_bytes(); + let last = len - 1; + last_diff[last] = if last_diff[last] == b'0' { b'1' } else { b'0' }; + let last_diff = String::from_utf8(last_diff).unwrap(); + let last_source = padded_bits(&last_diff, nd_offset); + let last_view = last_source + .as_bit_str() + .slice_from(nd_offset) + .slice_until(len); + assert!(!hs.matches_at_str(0, last_view)); + assert!(!hs.starts_with_str(last_view)); + assert!(!hs.ends_with_str(last_view)); + } + } + } + } +} + #[test] fn attack_matches_at_oob() { let bits = bs("10101"); @@ -22,6 +104,29 @@ fn attack_matches_at_oob() { assert!(bits.matches_at_str(5, empty)); } +#[test] +fn attack_bitstring_matches_at_long_pattern_unaligned_offsets() { + let pattern_text = "10110010".repeat(17); // 136 bits: forces the multi-word path. + let pattern = bs(&pattern_text); + + for index in [1, 3, 63, 65] { + let haystack = bs(&cat(&[ + "0".repeat(index).as_str(), + pattern_text.as_str(), + "111", + ])); + + assert!( + haystack.matches_at_string(index, &pattern), + "matches_at_string failed at unaligned index {index}" + ); + assert!( + haystack.matches_at_str(index, pattern.as_bit_str()), + "matches_at_str failed at unaligned index {index}" + ); + } +} + #[test] fn attack_starts_with_ends_with_edge() { let bits = bs("10101"); @@ -136,6 +241,34 @@ fn attack_strip_prefix_suffix() { assert_eq!(stripped.to_string(), "101"); } +#[test] +fn attack_bitstring_strip_suffix_long_pattern_at_unaligned_offset() { + let prefix = "011"; + let suffix_text = "10110010".repeat(17); // 136 bits: forces the multi-word path. + let suffix = bs(&suffix_text); + let bits = bs(&cat(&[prefix, suffix_text.as_str()])); + + assert_eq!( + bits.strip_suffix_string(&suffix).unwrap().to_string(), + prefix + ); + assert_eq!( + bits.strip_suffix_str(suffix.as_bit_str()) + .unwrap() + .to_string(), + prefix + ); +} + +#[test] +fn attack_bitstring_strip_suffix_longer_returns_none() { + let bits = bs("10101"); + let longer = bs("101010"); + + assert!(bits.strip_suffix_string(&longer).is_none()); + assert!(bits.strip_suffix_str(longer.as_bit_str()).is_none()); +} + // =========================================================================== // B. Unaligned find / contains / rfind // =========================================================================== diff --git a/tests/adversarial/tests_for_ord_hash.rs b/tests/adversarial/tests_for_ord_hash.rs index fada50e..8235582 100644 --- a/tests/adversarial/tests_for_ord_hash.rs +++ b/tests/adversarial/tests_for_ord_hash.rs @@ -2,6 +2,80 @@ use super::*; use core::cmp::Ordering; use int_interval::UsizeCO; +fn alignment_pattern(len: usize) -> String { + (0..len) + .map(|i| if (i * 17 + i / 3) % 11 < 5 { '0' } else { '1' }) + .collect() +} + +fn padded_bits(text: &str, offset: usize) -> BitString { + bs(&cat(&[ + "1".repeat(offset).as_str(), + text, + "0".repeat(7).as_str(), + ])) +} + +#[test] +fn attack_eq_ord_hash_alignment_matrix_at_word_boundaries() { + for len in [0, 1, 63, 64, 65, 127, 128, 129] { + let text = alignment_pattern(len); + + for lhs_offset in [0, 3] { + let lhs_source = padded_bits(&text, lhs_offset); + let lhs = lhs_source + .as_bit_str() + .slice_from(lhs_offset) + .slice_until(len); + + for rhs_offset in [0, 5] { + let rhs_source = padded_bits(&text, rhs_offset); + let rhs = rhs_source + .as_bit_str() + .slice_from(rhs_offset) + .slice_until(len); + + assert_eq!(lhs, rhs, "eq len={len} lhs={lhs_offset} rhs={rhs_offset}"); + assert_eq!( + lhs.cmp_str(&rhs), + Ordering::Equal, + "cmp len={len} lhs={lhs_offset} rhs={rhs_offset}" + ); + assert_eq!( + hash(&lhs), + hash(&rhs), + "hash len={len} lhs={lhs_offset} rhs={rhs_offset}" + ); + + if len > 0 { + for differing_index in [0, len - 1] { + let mut different = text.clone().into_bytes(); + different[differing_index] = if different[differing_index] == b'0' { + b'1' + } else { + b'0' + }; + let different = String::from_utf8(different).unwrap(); + let different_source = padded_bits(&different, rhs_offset); + let different_view = different_source + .as_bit_str() + .slice_from(rhs_offset) + .slice_until(len); + let expected = text.cmp(&different); + + assert_ne!(lhs, different_view); + assert_eq!( + lhs.cmp_str(&different_view), + expected, + "different bit {differing_index}, len={len}, lhs={lhs_offset}, rhs={rhs_offset}" + ); + } + } + } + } + } +} + #[test] fn attack_ord_different_lengths() { // Shorter is less than longer when common prefix equal